I have an easy to reproduce problem I'm experiencing with a Live image
generated from Fedora 18 bases. If too much gets written to the writable
tmpfs overlay, the host goes all wonky in that most all commands will fail
to run and immediately return one of the above error messages. For
example:
root@aos-61:46 # cat /proc/meminfo
MemTotal: 995020 kB
MemFree: 199348 kB
Buffers: 91192 kB
Cached: 637920 kB
SwapCached: 0 kB
Active: 221796 kB
Inactive: 549340 kB
Active(anon): 43120 kB
Inactive(anon): 3220 kB
Active(file): 178676 kB
Inactive(file): 546120 kB
Unevictable: 0 kB
Mlocked: 0 kB
HighTotal: 110344 kB
HighFree: 220 kB
LowTotal: 884676 kB
LowFree: 199128 kB
SwapTotal: 0 kB
SwapFree: 0 kB
Dirty: 0 kB
Writeback: 0 kB
AnonPages: 42036 kB
Mapped: 21392 kB
Shmem: 4324 kB
Slab: 16696 kB
SReclaimable: 11736 kB
SUnreclaim: 4960 kB
KernelStack: 584 kB
PageTables: 612 kB
NFS_Unstable: 0 kB
Bounce: 0 kB
WritebackTmp: 0 kB
CommitLimit: 497508 kB
Committed_AS: 92280 kB
VmallocTotal: 122880 kB
VmallocUsed: 35324 kB
VmallocChunk: 87296 kB
HardwareCorrupted: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 4096 kB
DirectMap4k: 12280 kB
DirectMap4M: 892928 kB
root@aos-61:46 # df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 481M 0 481M 0% /dev
tmpfs 486M 0 486M 0% /dev/shm
tmpfs 486M 296K 486M 1% /run
tmpfs 486M 0 486M 0% /sys/fs/cgroup
/dev/sda1 7.4G 1.2G 5.9G 17% /run/initramfs/live
/dev/mapper/live-rw 2.0G 1.3G 658M 67% /
tmpfs 486M 1.5M 485M 1% /tmp
root@aos-61:46 # mount
proc on /proc type proc (rw,relatime)
sysfs on /sys type sysfs (rw,relatime)
devtmpfs on /dev type devtmpfs
(rw,nosuid,size=492160k,nr_inodes=123040,mode=755)
securityfs on /sys/kernel/security type securityfs
(rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,relatime)
devpts on /dev/pts type devpts (rw,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755)
tmpfs on /sys/fs/cgroup type tmpfs (rw,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup
(rw,nosuid,nodev,noexec,relatime,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/cpuset type cgroup
(rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup
(rw,nosuid,nodev,noexec,relatime,cpuacct,cpu)
cgroup on /sys/fs/cgroup/memory type cgroup
(rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/devices type cgroup
(rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/freezer type cgroup
(rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/net_cls type cgroup
(rw,nosuid,nodev,noexec,relatime,net_cls)
cgroup on /sys/fs/cgroup/blkio type cgroup
(rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/perf_event type cgroup
(rw,nosuid,nodev,noexec,relatime,perf_event)
/dev/sda1 on /run/initramfs/live type ext3 (ro,relatime)
/dev/mapper/live-rw on / type ext3 (rw,noatime,data=ordered)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs
(rw,relatime,fd=29,pgrp=1,timeout=300,minproto=5,maxproto=5,direct)
mqueue on /dev/mqueue type mqueue (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
tmpfs on /tmp type tmpfs (rw)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
configfs on /sys/kernel/config type configfs (rw,relatime)
none on /var/lib/stateless/writable type tmpfs (rw,relatime)
none on /var/cache/man type tmpfs (rw,relatime)
none on /var/lib/xkb type tmpfs (rw,relatime)
none on /var/log type tmpfs (rw,relatime)
none on /var/lib/dbus type tmpfs (rw,relatime)
none on /var/lib/nfs type tmpfs (rw,relatime)
none on /tmp type tmpfs (rw,relatime)
none on /var/lib/dhclient type tmpfs (rw,relatime)
none on /var/tmp type tmpfs (rw,relatime)
none on /var/lib/logrotate.status type tmpfs (rw,relatime)
none on /var/lib/random-seed type tmpfs (rw,relatime)
none on /var/spool type tmpfs (rw,relatime)
/dev/sda1 on /mnt/live type ext3 (rw,noatime)
root@aos-61:46 #
root@aos-61:46 #
root@aos-61:46 # # Lets now make it all go wonky:
root@aos-61:46 # time dd if=/dev/zero of=/foo
Bus error
real 1m15.775s
user 0m2.818s
sys 0m24.129s
root@aos-61:46 #
root@aos-61:46 # ls /root
-bash: /bin/ls: Input/output error
root@aos-61:46 # df -h
-bash: /usr/bin/df: Input/output error
root@aos-61:46 # mount
-bash: /usr/bin/mount: Input/output error
root@aos-61:46 # cat /proc/meminfo
-bash: /usr/bin/cat: Input/output error
Is this expected? Is there anything I can do, e.g., configuration-wise,
that can prevent this? Ideally this would fail much like any other full
disk situation. I understand that the overlay consumes space, i.e.,
memory, for this file growth, including file removals, but I'd at least
like to be able to remotely reboot a system when in this state, however I
can't even do that because the reboot command will either return the same
I/O error or it may succeed but get the I/O error when systemd tries to
read /usr/lib/systemd/system/reboot.target.
I dug around in bugzilla, but found nothing there. I can file a bug, but
which package is likely at fault here?
--
John Florian