I have an easy to reproduce problem I'm
experiencing with a Live image generated from Fedora 18 bases. If
too much gets written to the writable tmpfs overlay, the host goes all
wonky in that most all commands will fail to run and immediately return
one of the above error messages. For example:
root@aos-61:46 # cat /proc/meminfo
MemTotal: 995020 kB
MemFree: 199348
kB
Buffers: 91192
kB
Cached: 637920
kB
SwapCached: 0
kB
Active: 221796
kB
Inactive: 549340 kB
Active(anon): 43120 kB
Inactive(anon): 3220 kB
Active(file): 178676 kB
Inactive(file): 546120 kB
Unevictable: 0
kB
Mlocked:
0 kB
HighTotal: 110344 kB
HighFree: 220
kB
LowTotal: 884676 kB
LowFree: 199128
kB
SwapTotal:
0 kB
SwapFree:
0 kB
Dirty:
0 kB
Writeback:
0 kB
AnonPages: 42036 kB
Mapped: 21392
kB
Shmem: 4324
kB
Slab: 16696
kB
SReclaimable: 11736 kB
SUnreclaim: 4960 kB
KernelStack: 584 kB
PageTables: 612
kB
NFS_Unstable: 0
kB
Bounce:
0 kB
WritebackTmp: 0
kB
CommitLimit: 497508 kB
Committed_AS: 92280 kB
VmallocTotal: 122880 kB
VmallocUsed: 35324 kB
VmallocChunk: 87296 kB
HardwareCorrupted: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 4096 kB
DirectMap4k: 12280 kB
DirectMap4M: 892928 kB
root@aos-61:46 # df -h
Filesystem Size
Used Avail Use% Mounted on
devtmpfs
481M 0 481M 0% /dev
tmpfs
486M 0 486M 0% /dev/shm
tmpfs
486M 296K 486M 1% /run
tmpfs
486M 0 486M 0% /sys/fs/cgroup
/dev/sda1 7.4G
1.2G 5.9G 17% /run/initramfs/live
/dev/mapper/live-rw 2.0G 1.3G 658M
67% /
tmpfs
486M 1.5M 485M 1% /tmp
root@aos-61:46 # mount
proc on /proc type proc (rw,relatime)
sysfs on /sys type sysfs (rw,relatime)
devtmpfs on /dev type devtmpfs (rw,nosuid,size=492160k,nr_inodes=123040,mode=755)
securityfs on /sys/kernel/security type securityfs
(rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,relatime)
devpts on /dev/pts type devpts (rw,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755)
tmpfs on /sys/fs/cgroup type tmpfs (rw,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/net_cls type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
/dev/sda1 on /run/initramfs/live type ext3 (ro,relatime)
/dev/mapper/live-rw on / type ext3 (rw,noatime,data=ordered)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs
(rw,relatime,fd=29,pgrp=1,timeout=300,minproto=5,maxproto=5,direct)
mqueue on /dev/mqueue type mqueue (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
tmpfs on /tmp type tmpfs (rw)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
configfs on /sys/kernel/config type configfs (rw,relatime)
none on /var/lib/stateless/writable type tmpfs (rw,relatime)
none on /var/cache/man type tmpfs (rw,relatime)
none on /var/lib/xkb type tmpfs (rw,relatime)
none on /var/log type tmpfs (rw,relatime)
none on /var/lib/dbus type tmpfs (rw,relatime)
none on /var/lib/nfs type tmpfs (rw,relatime)
none on /tmp type tmpfs (rw,relatime)
none on /var/lib/dhclient type tmpfs (rw,relatime)
none on /var/tmp type tmpfs (rw,relatime)
none on /var/lib/logrotate.status type tmpfs (rw,relatime)
none on /var/lib/random-seed type tmpfs (rw,relatime)
none on /var/spool type tmpfs (rw,relatime)
/dev/sda1 on /mnt/live type ext3 (rw,noatime)
root@aos-61:46 #
root@aos-61:46 #
root@aos-61:46 # # Lets now make it all go wonky:
root@aos-61:46 # time dd if=/dev/zero of=/foo
Bus error
real 1m15.775s
user 0m2.818s
sys 0m24.129s
root@aos-61:46 #
root@aos-61:46 # ls /root
-bash: /bin/ls: Input/output error
root@aos-61:46 # df -h
-bash: /usr/bin/df: Input/output error
root@aos-61:46 # mount
-bash: /usr/bin/mount: Input/output error
root@aos-61:46 # cat /proc/meminfo
-bash: /usr/bin/cat: Input/output error
Is this expected? Is there anything
I can do, e.g., configuration-wise, that can prevent this? Ideally
this would fail much like any other full disk situation. I understand
that the overlay consumes space, i.e., memory, for this file growth, including
file removals, but I'd at least like to be able to remotely reboot a system
when in this state, however I can't even do that because the reboot command
will either return the same I/O error or it may succeed but get the I/O
error when systemd tries to read /usr/lib/systemd/system/reboot.target.
I dug around in bugzilla, but found
nothing there. I can file a bug, but which package is likely at fault
here?
--
John Florian
--Fred