Extract functions from kdump.sh, and construct kdump-lib-initramfs.sh as kdump common functions/varaibles library.
kdump-lib-initramfs.sh will include kdump-lib.sh, because it will use the functions from there. IOW, kdump-lib-initramfs.sh will be a superset of kdump-lib.sh
So after this cleanup:
- scripts running in 1st kernel only have to include kdump-lib.sh - scripts running in 2nd kernel only have to include kdump-lib-initramfs.sh
Signed-off-by: WANG Chao chaowang@redhat.com --- dracut-kdump.sh | 145 ++---------------------------------------------- dracut-module-setup.sh | 1 + kdump-lib-initramfs.sh | 146 +++++++++++++++++++++++++++++++++++++++++++++++++ kexec-tools.spec | 2 + 4 files changed, 154 insertions(+), 140 deletions(-) create mode 100755 kdump-lib-initramfs.sh
diff --git a/dracut-kdump.sh b/dracut-kdump.sh index cb13d92..08f4322 100755 --- a/dracut-kdump.sh +++ b/dracut-kdump.sh @@ -2,30 +2,14 @@
exec &> /dev/console . /lib/dracut-lib.sh -. /lib/kdump-lib.sh +. /lib/kdump-lib-initramfs.sh
if [ -f "$initdir/lib/dracut/no-emergency-shell" ]; then rm -f -- $initdir/lib/dracut/no-emergency-shell fi
set -o pipefail -KDUMP_PATH="/var/crash" -CORE_COLLECTOR="" -DEFAULT_CORE_COLLECTOR="makedumpfile -l --message-level 1 -d 31" -DMESG_COLLECTOR="/sbin/vmcore-dmesg" -DEFAULT_ACTION="reboot -f" -DATEDIR=`date +%Y.%m.%d-%T` -HOST_IP='127.0.0.1' -DUMP_INSTRUCTION="" -SSH_KEY_LOCATION="/root/.ssh/kdump_id_rsa" -KDUMP_SCRIPT_DIR="/kdumpscripts" -DD_BLKSIZE=512 -FINAL_ACTION="reboot -f" DUMP_RETVAL=0 -conf_file="/etc/kdump.conf" -KDUMP_PRE="" -KDUMP_POST="" -MOUNTS=""
export PATH=$PATH:$KDUMP_SCRIPT_DIR
@@ -43,27 +27,6 @@ do_dump() return $_ret }
-do_umount() -{ - if [ -n "$MOUNTS" ]; then - for mount in $MOUNTS; do - ismounted $mount && umount -R $mount - done - fi -} - -do_final_action() -{ - do_umount - eval $FINAL_ACTION -} - -do_default_action() -{ - wait_for_loginit - eval $DEFAULT_ACTION -} - do_kdump_pre() { if [ -n "$KDUMP_PRE" ]; then @@ -83,39 +46,6 @@ add_dump_code() DUMP_INSTRUCTION=$1 }
-# dump_fs <mount point| device> -dump_fs() -{ - local _dev=$(findmnt -k -f -n -r -o SOURCE $1) - local _mp=$(findmnt -k -f -n -r -o TARGET $1) - - echo "kdump: dump target is $_dev" - - if [ -z "$_mp" ]; then - echo "kdump: error: Dump target $_dev is not mounted." - return 1 - fi - MOUNTS="$MOUNTS $_mp" - - # Remove -F in makedumpfile case. We don't want a flat format dump here. - [[ $CORE_COLLECTOR = *makedumpfile* ]] && CORE_COLLECTOR=`echo $CORE_COLLECTOR | sed -e "s/-F//g"` - - echo "kdump: saving to $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/" - - mount -o remount,rw $_mp || return 1 - mkdir -p $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR || return 1 - - save_vmcore_dmesg_fs ${DMESG_COLLECTOR} "$_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/" - - echo "kdump: saving vmcore" - $CORE_COLLECTOR /proc/vmcore $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete || return 1 - mv $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore - sync - - echo "kdump: saving vmcore complete" - return 0 -} - dump_raw() { local _raw=$1 @@ -165,21 +95,6 @@ dump_ssh() return 0 }
-save_vmcore_dmesg_fs() { - local _dmesg_collector=$1 - local _path=$2 - - echo "kdump: saving vmcore-dmesg.txt" - $_dmesg_collector /proc/vmcore > ${_path}/vmcore-dmesg-incomplete.txt - _exitcode=$? - if [ $_exitcode -eq 0 ]; then - mv ${_path}/vmcore-dmesg-incomplete.txt ${_path}/vmcore-dmesg.txt - echo "kdump: saving vmcore-dmesg.txt complete" - else - echo "kdump: saving vmcore-dmesg.txt failed" - fi -} - save_vmcore_dmesg_ssh() { local _dmesg_collector=$1 local _path=$2 @@ -198,7 +113,6 @@ save_vmcore_dmesg_ssh() { fi }
- get_host_ip() { local _host @@ -218,61 +132,12 @@ get_host_ip()
read_kdump_conf() { - if [ ! -f "$conf_file" ]; then - echo "kdump: $conf_file not found" + if [ ! -f "$KDUMP_CONF" ]; then + echo "kdump: $KDUMP_CONF not found" return fi
- # first get the necessary variables - while read config_opt config_val; - do - # remove inline comments after the end of a directive. - config_val=$(strip_comments $config_val) - case "$config_opt" in - path) - KDUMP_PATH="$config_val" - ;; - core_collector) - [ -n "$config_val" ] && CORE_COLLECTOR="$config_val" - ;; - sshkey) - if [ -f "$config_val" ]; then - SSH_KEY_LOCATION=$config_val - fi - ;; - kdump_pre) - KDUMP_PRE="$config_val" - ;; - kdump_post) - KDUMP_POST="$config_val" - ;; - fence_kdump_args) - FENCE_KDUMP_ARGS="$config_val" - ;; - fence_kdump_nodes) - FENCE_KDUMP_NODES="$config_val" - ;; - default) - case $config_val in - shell) - DEFAULT_ACTION="_emergency_shell kdump" - ;; - reboot) - DEFAULT_ACTION="do_umount; reboot -f" - ;; - halt) - DEFAULT_ACTION="do_umount; halt -f" - ;; - poweroff) - DEFAULT_ACTION="do_umount; poweroff -f" - ;; - dump_to_rootfs) - DEFAULT_ACTION="dump_fs $NEWROOT" - ;; - esac - ;; - esac - done < $conf_file + get_kdump_confs
# rescan for add code for dump target while read config_opt config_val; @@ -290,7 +155,7 @@ read_kdump_conf() add_dump_code "dump_ssh $SSH_KEY_LOCATION $config_val" ;; esac - done < $conf_file + done < $KDUMP_CONF }
fence_kdump_notify() diff --git a/dracut-module-setup.sh b/dracut-module-setup.sh index 87ad072..84bf975 100755 --- a/dracut-module-setup.sh +++ b/dracut-module-setup.sh @@ -555,6 +555,7 @@ install() { inst "/sbin/vmcore-dmesg" "/sbin/vmcore-dmesg" inst_hook pre-pivot 9999 "$moddir/kdump.sh" inst "/lib/kdump/kdump-lib.sh" "/lib/kdump-lib.sh" + inst "/lib/kdump/kdump-lib-initramfs.sh" "/lib/kdump-lib-initramfs.sh"
# Check for all the devices and if any device is iscsi, bring up iscsi # target. Ideally all this should be pushed into dracut iscsi module diff --git a/kdump-lib-initramfs.sh b/kdump-lib-initramfs.sh new file mode 100755 index 0000000..e2807c7 --- /dev/null +++ b/kdump-lib-initramfs.sh @@ -0,0 +1,146 @@ +# These variables and functions are useful in 2nd kernel + +. /lib/dracut-lib.sh +. /lib/kdump-lib.sh + +KDUMP_PATH="/var/crash" +CORE_COLLECTOR="" +DEFAULT_CORE_COLLECTOR="makedumpfile -l --message-level 1 -d 31" +DMESG_COLLECTOR="/sbin/vmcore-dmesg" +DEFAULT_ACTION="reboot -f" +DATEDIR=`date +%Y.%m.%d-%T` +HOST_IP='127.0.0.1' +DUMP_INSTRUCTION="" +SSH_KEY_LOCATION="/root/.ssh/kdump_id_rsa" +KDUMP_SCRIPT_DIR="/kdumpscripts" +DD_BLKSIZE=512 +FINAL_ACTION="reboot -f" +KDUMP_CONF="/etc/kdump.conf" +KDUMP_PRE="" +KDUMP_POST="" +NEWROOT="/sysroot" +MOUNTS="" + +get_kdump_confs() +{ + local config_opt config_val + + while read config_opt config_val; + do + # remove inline comments after the end of a directive. + config_val=$(strip_comments $config_val) + case "$config_opt" in + path) + KDUMP_PATH="$config_val" + ;; + core_collector) + [ -n "$config_val" ] && CORE_COLLECTOR="$config_val" + ;; + sshkey) + if [ -f "$config_val" ]; then + SSH_KEY_LOCATION=$config_val + fi + ;; + kdump_pre) + KDUMP_PRE="$config_val" + ;; + kdump_post) + KDUMP_POST="$config_val" + ;; + fence_kdump_args) + FENCE_KDUMP_ARGS="$config_val" + ;; + fence_kdump_nodes) + FENCE_KDUMP_NODES="$config_val" + ;; + default) + case $config_val in + shell) + DEFAULT_ACTION="_emergency_shell kdump" + ;; + reboot) + DEFAULT_ACTION="do_umount; reboot -f" + ;; + halt) + DEFAULT_ACTION="do_umount; halt -f" + ;; + poweroff) + DEFAULT_ACTION="do_umount; poweroff -f" + ;; + dump_to_rootfs) + DEFAULT_ACTION="dump_fs $NEWROOT" + ;; + esac + ;; + esac + done < $KDUMP_CONF +} + +# dump_fs <mount point| device> +dump_fs() +{ + + local _dev=$(findmnt -k -f -n -r -o SOURCE $1) + local _mp=$(findmnt -k -f -n -r -o TARGET $1) + + echo "kdump: dump target is $_dev" + + if [ -z "$_mp" ]; then + echo "kdump: error: Dump target $_dev is not mounted." + return 1 + fi + MOUNTS="$MOUNTS $_mp" + + # Remove -F in makedumpfile case. We don't want a flat format dump here. + [[ $CORE_COLLECTOR = *makedumpfile* ]] && CORE_COLLECTOR=`echo $CORE_COLLECTOR | sed -e "s/-F//g"` + + echo "kdump: saving to $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/" + + mount -o remount,rw $_mp || return 1 + mkdir -p $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR || return 1 + + save_vmcore_dmesg_fs ${DMESG_COLLECTOR} "$_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/" + + echo "kdump: saving vmcore" + $CORE_COLLECTOR /proc/vmcore $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete || return 1 + mv $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore + sync + + echo "kdump: saving vmcore complete" +} + +save_vmcore_dmesg_fs() { + local _dmesg_collector=$1 + local _path=$2 + + echo "kdump: saving vmcore-dmesg.txt" + $_dmesg_collector /proc/vmcore > ${_path}/vmcore-dmesg-incomplete.txt + _exitcode=$? + if [ $_exitcode -eq 0 ]; then + mv ${_path}/vmcore-dmesg-incomplete.txt ${_path}/vmcore-dmesg.txt + echo "kdump: saving vmcore-dmesg.txt complete" + else + echo "kdump: saving vmcore-dmesg.txt failed" + fi +} + +do_umount() +{ + if [ -n "$MOUNTS" ]; then + for mount in $MOUNTS; do + ismounted $mount && umount -R $mount + done + fi +} + +do_default_action() +{ + wait_for_loginit + eval $DEFAULT_ACTION +} + +do_final_action() +{ + do_umount + eval $FINAL_ACTION +} diff --git a/kexec-tools.spec b/kexec-tools.spec index 25dae2b..a613b43 100644 --- a/kexec-tools.spec +++ b/kexec-tools.spec @@ -27,6 +27,7 @@ Source20: kdump-lib.sh Source21: kdump-in-cluster-environment.txt Source22: kdump-dep-generator.sh Source23: kdump-anaconda-addon-20140721.tar.gz +Source24: kdump-lib-initramfs.sh
####################################### # These are sources for mkdumpramfs @@ -178,6 +179,7 @@ install -m 644 kexec/kexec.8 $RPM_BUILD_ROOT%{_mandir}/man8/kexec.8 install -m 755 %{SOURCE11} $RPM_BUILD_ROOT%{_datadir}/kdump/firstboot_kdump.py install -m 644 %{SOURCE12} $RPM_BUILD_ROOT%{_mandir}/man8/mkdumprd.8 install -m 755 %{SOURCE20} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-lib.sh +install -m 755 %{SOURCE24} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-lib-initramfs.sh %ifnarch s390x # For s390x the ELF header is created in the kdump kernel and therefore kexec # udev rules are not required
This patch does the following change in 2nd kernel: - dump target is mounted under /sysroot
With this change, we don't need to track what we've mounted in 2nd kernel. We can just umount recursively every mount in /sysroot by command:
umount -R /sysroot
It's very convenient to do so, because it's hard to track what we've mounted when we're in error handling path (later patches). So mount everything under /sysroot is reasonable and practical for us.
Also clean up a bit along with this patch.
Signed-off-by: WANG Chao chaowang@redhat.com --- kdump-lib-initramfs.sh | 8 +------- mkdumprd | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 19 deletions(-)
diff --git a/kdump-lib-initramfs.sh b/kdump-lib-initramfs.sh index e2807c7..b7f0e61 100755 --- a/kdump-lib-initramfs.sh +++ b/kdump-lib-initramfs.sh @@ -19,7 +19,6 @@ KDUMP_CONF="/etc/kdump.conf" KDUMP_PRE="" KDUMP_POST="" NEWROOT="/sysroot" -MOUNTS=""
get_kdump_confs() { @@ -89,7 +88,6 @@ dump_fs() echo "kdump: error: Dump target $_dev is not mounted." return 1 fi - MOUNTS="$MOUNTS $_mp"
# Remove -F in makedumpfile case. We don't want a flat format dump here. [[ $CORE_COLLECTOR = *makedumpfile* ]] && CORE_COLLECTOR=`echo $CORE_COLLECTOR | sed -e "s/-F//g"` @@ -126,11 +124,7 @@ save_vmcore_dmesg_fs() {
do_umount() { - if [ -n "$MOUNTS" ]; then - for mount in $MOUNTS; do - ismounted $mount && umount -R $mount - done - fi + umount -Rf $NEWROOT }
do_default_action() diff --git a/mkdumprd b/mkdumprd index b49b74f..ba35800 100644 --- a/mkdumprd +++ b/mkdumprd @@ -97,19 +97,19 @@ target_is_root() {
# caller should ensure $1 is valid and mounted in 1st kernel to_mount() { - local _dev=$1 _s _t _o _mntopts _pdev - - _s=$(findmnt -k -f -n -r -o SOURCE $_dev) - _t=$(findmnt -k -f -n -r -o TARGET,FSTYPE $_dev) - _o=$(findmnt -k -f -n -r -o OPTIONS $_dev) - _o=${_o/#ro/rw} #mount fs target as rw in 2nd kernel - # "nofail" mount could be run later than kdump.sh. So we don't pass nofail - # for short term. - #_o="${_o},nofail" #with nofail set, systemd won't block for mount failure - _mntopts="$_t $_o" + local _dev=$1 _source _target _fstype _options _mntopts _pdev + + _source=$(findmnt -k -f -n -r -o SOURCE $_dev) + _target=$(findmnt -k -f -n -r -o TARGET $_dev) + # mount under /sysroot in 2nd kernel, and we umount -R /sysroot before exit + _target="/sysroot$_target" + _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) + _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) + _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel + _mntopts="$_target $_fstype $_options" #for non-nfs _dev converting to use udev persistent name - if [ -b "$_s" ]; then - _pdev="$(get_persistent_dev $_s)" + if [ -b "$_source" ]; then + _pdev="$(get_persistent_dev $_source)" if [ $? -ne 0 ]; then return 1 fi
On Tue, Jul 22, 2014 at 03:17:59PM +0800, WANG Chao wrote:
This patch does the following change in 2nd kernel:
- dump target is mounted under /sysroot
With this change, we don't need to track what we've mounted in 2nd kernel. We can just umount recursively every mount in /sysroot by command:
umount -R /sysroot
It's very convenient to do so, because it's hard to track what we've mounted when we're in error handling path (later patches). So mount everything under /sysroot is reasonable and practical for us.
Also clean up a bit along with this patch.
Signed-off-by: WANG Chao chaowang@redhat.com
Acked-by: Vivek Goyal vgoyal@redhat.com
Vivek
kdump-lib-initramfs.sh | 8 +------- mkdumprd | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 19 deletions(-)
diff --git a/kdump-lib-initramfs.sh b/kdump-lib-initramfs.sh index e2807c7..b7f0e61 100755 --- a/kdump-lib-initramfs.sh +++ b/kdump-lib-initramfs.sh @@ -19,7 +19,6 @@ KDUMP_CONF="/etc/kdump.conf" KDUMP_PRE="" KDUMP_POST="" NEWROOT="/sysroot" -MOUNTS=""
get_kdump_confs() { @@ -89,7 +88,6 @@ dump_fs() echo "kdump: error: Dump target $_dev is not mounted." return 1 fi
MOUNTS="$MOUNTS $_mp"
# Remove -F in makedumpfile case. We don't want a flat format dump here. [[ $CORE_COLLECTOR = *makedumpfile* ]] && CORE_COLLECTOR=`echo $CORE_COLLECTOR | sed -e "s/-F//g"`
@@ -126,11 +124,7 @@ save_vmcore_dmesg_fs() {
do_umount() {
- if [ -n "$MOUNTS" ]; then
for mount in $MOUNTS; doismounted $mount && umount -R $mountdone- fi
- umount -Rf $NEWROOT
}
do_default_action() diff --git a/mkdumprd b/mkdumprd index b49b74f..ba35800 100644 --- a/mkdumprd +++ b/mkdumprd @@ -97,19 +97,19 @@ target_is_root() {
# caller should ensure $1 is valid and mounted in 1st kernel to_mount() {
- local _dev=$1 _s _t _o _mntopts _pdev
- _s=$(findmnt -k -f -n -r -o SOURCE $_dev)
- _t=$(findmnt -k -f -n -r -o TARGET,FSTYPE $_dev)
- _o=$(findmnt -k -f -n -r -o OPTIONS $_dev)
- _o=${_o/#ro/rw} #mount fs target as rw in 2nd kernel
- # "nofail" mount could be run later than kdump.sh. So we don't pass nofail
- # for short term.
- #_o="${_o},nofail" #with nofail set, systemd won't block for mount failure
- _mntopts="$_t $_o"
- local _dev=$1 _source _target _fstype _options _mntopts _pdev
- _source=$(findmnt -k -f -n -r -o SOURCE $_dev)
- _target=$(findmnt -k -f -n -r -o TARGET $_dev)
- # mount under /sysroot in 2nd kernel, and we umount -R /sysroot before exit
- _target="/sysroot$_target"
- _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev)
- _options=$(findmnt -k -f -n -r -o OPTIONS $_dev)
- _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel
- _mntopts="$_target $_fstype $_options" #for non-nfs _dev converting to use udev persistent name
- if [ -b "$_s" ]; then
_pdev="$(get_persistent_dev $_s)"
- if [ -b "$_source" ]; then
_pdev="$(get_persistent_dev $_source)" if [ $? -ne 0 ]; then return 1 fi-- 1.9.3
Now when mount in /etc/fstab fails, systemd would not consider it as critical and it would continue to boot. In fact, emergency service is triggered, but not in a isolation mode, and it results in the emergency service getting shutdown at some point later of the boot process. We need isolation otherwise we won't see any emergency service.
That is because in kdump initramfs, mount units specified in /etc/fstab are required by "local-fs.target". When any of these mounts fails, local-fs.target fails.
For kdump initramfs, we need to isolate to emergency service on any of the mount failure, that said, every service should be stopped and onlu emergency service would run. But local-fs.target won't trigger that on its failure. That means in case of mount failure, local-fs.target also enters failure state, but all the service will continue without any interruption.
After digging looking into source code of systemd-fstab-generator. I find "x-initrd.mount" using in initramfs mount, will make the mount units required by "initrd-root-fs.target" rather than it's used to be "local-fs.target".
"initrd-root-fs.target" is suitable to us because if it fails, it will isolate to emergency service. That means in case of any mount failure, the emergeny service will start and everything else will stop. We want this effect because we need to take kdump fail-safe action when there's a mount failure.
From systemd unit point of view, "initrd-root-fs.target" has
OnFailureIsolate=yes, but "local-fs.target" doesn't. From systemd.unit(5):
OnFailureIsolate= Takes a boolean argument. If true, the unit listed in OnFailure= will be enqueued in isolation mode, i.e. all units that are not its dependency will be stopped. If this is set, only a single unit may be listed in OnFailure=. Defaults to false.
NOTE: Harald who contributed "x-initrd.mount" in systemd, confirmed that this feature will stay.
Signed-off-by: WANG Chao chaowang@redhat.com --- mkdumprd | 3 +++ 1 file changed, 3 insertions(+)
diff --git a/mkdumprd b/mkdumprd index ba35800..c908b00 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,9 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel + # "x-initrd.mount" mount failure will trigger isolate emergency service + # W/o this, systemd won't isolate, thus we won't get to emergency + _options="$_options,x-initrd.mount" _mntopts="$_target $_fstype $_options" #for non-nfs _dev converting to use udev persistent name if [ -b "$_source" ]; then
On Tue, Jul 22, 2014 at 03:18:00PM +0800, WANG Chao wrote:
[..]
diff --git a/mkdumprd b/mkdumprd index ba35800..c908b00 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,9 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel
- # "x-initrd.mount" mount failure will trigger isolate emergency service
- # W/o this, systemd won't isolate, thus we won't get to emergency
- _options="$_options,x-initrd.mount"
Hi Chao,
Say, I am dumping to an NFS target and mounting NFS fails. Assume "default" in kdump.conf is "dump_to_rootfs". Now by the time we hit kdump emergency service, has rootfs been mounted so that dump can be saved on root fs?
Thanks Vivek
On 07/24/14 at 08:58am, Vivek Goyal wrote:
On Tue, Jul 22, 2014 at 03:18:00PM +0800, WANG Chao wrote:
[..]
diff --git a/mkdumprd b/mkdumprd index ba35800..c908b00 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,9 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel
- # "x-initrd.mount" mount failure will trigger isolate emergency service
- # W/o this, systemd won't isolate, thus we won't get to emergency
- _options="$_options,x-initrd.mount"
Hi Chao,
Say, I am dumping to an NFS target and mounting NFS fails. Assume "default" in kdump.conf is "dump_to_rootfs". Now by the time we hit kdump emergency service, has rootfs been mounted so that dump can be saved on root fs?
Yes.
If dump_to_rootfs(), we will start dracut-initqueue service and sysroot.mount again. So yes, the /sysroot is ok to access.
But I find a problem, when thinking about your question. nfs mount is not supposed to be with "x-initrd.mount". "x-initrd.mount" is only useful for local mount, because nfs mount should be associated with remote-fs.target. I'll send an update to fix it.
Thanks WANG Chao
Now when mount in /etc/fstab fails, systemd would not consider it as critical and it would continue to boot. In fact, emergency service is triggered, but not in a isolation mode, and it results in the emergency service getting shutdown at some point later of the boot process. We need isolation otherwise we won't see any emergency service.
That is because in kdump initramfs, mount units specified in /etc/fstab are required by "local-fs.target". When any of these mounts fails, local-fs.target fails.
For kdump initramfs, we need to isolate to emergency service on any of the mount failure, that said, every service should be stopped and onlu emergency service would run. But local-fs.target won't trigger that on its failure. That means in case of mount failure, local-fs.target also enters failure state, but all the service will continue without any interruption.
After digging looking into source code of systemd-fstab-generator. I find "x-initrd.mount" using in initramfs mount, will make the mount units required by "initrd-root-fs.target" rather than it's used to be "local-fs.target".
"initrd-root-fs.target" is suitable to us because if it fails, it will isolate to emergency service. That means in case of any mount failure, the emergeny service will start and everything else will stop. We want this effect because we need to take kdump fail-safe action when there's a mount failure.
From systemd unit point of view, "initrd-root-fs.target" has
OnFailureIsolate=yes, but "local-fs.target" doesn't. From systemd.unit(5):
OnFailureIsolate= Takes a boolean argument. If true, the unit listed in OnFailure= will be enqueued in isolation mode, i.e. all units that are not its dependency will be stopped. If this is set, only a single unit may be listed in OnFailure=. Defaults to false.
NOTE: Harald who contributed "x-initrd.mount" in systemd, confirmed that this feature will stay.
Signed-off-by: WANG Chao chaowang@redhat.com --- mkdumprd | 6 ++++++ 1 file changed, 6 insertions(+)
diff --git a/mkdumprd b/mkdumprd index ba35800..9ee2f0b 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,12 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel + # "x-initrd.mount" mount failure will trigger isolate emergency service + # W/o this, systemd won't isolate, thus we won't get to emergency + # This is only applicable to local fs mount + if ! is_nfs_target; then + _options="$_options,x-initrd.mount" + fi _mntopts="$_target $_fstype $_options" #for non-nfs _dev converting to use udev persistent name if [ -b "$_source" ]; then
Now when mount in /etc/fstab fails, systemd would not consider it as critical and it would continue to boot. In fact, emergency service is triggered, but not in a isolation mode, and it results in the emergency service getting shutdown at some point later of the boot process. We need isolation otherwise we won't see any emergency service.
That is because in kdump initramfs, mount units specified in /etc/fstab are required by "local-fs.target". When any of these mounts fails, local-fs.target fails.
For kdump initramfs, we need to isolate to emergency service on any of the mount failure, that said, every service should be stopped and onlu emergency service would run. But local-fs.target won't trigger that on its failure. That means in case of mount failure, local-fs.target also enters failure state, but all the service will continue without any interruption.
After digging looking into source code of systemd-fstab-generator. I find "x-initrd.mount" using in initramfs mount, will make the mount units required by "initrd-root-fs.target" rather than it's used to be "local-fs.target".
"initrd-root-fs.target" is suitable to us because if it fails, it will isolate to emergency service. That means in case of any mount failure, the emergeny service will start and everything else will stop. We want this effect because we need to take kdump fail-safe action when there's a mount failure.
From systemd unit point of view, "initrd-root-fs.target" has
OnFailureIsolate=yes, but "local-fs.target" doesn't. From systemd.unit(5):
OnFailureIsolate= Takes a boolean argument. If true, the unit listed in OnFailure= will be enqueued in isolation mode, i.e. all units that are not its dependency will be stopped. If this is set, only a single unit may be listed in OnFailure=. Defaults to false.
NOTE: Harald who contributed "x-initrd.mount" in systemd, confirmed that this feature will stay.
Signed-off-by: WANG Chao chaowang@redhat.com --- mkdumprd | 6 ++++++ 1 file changed, 6 insertions(+)
diff --git a/mkdumprd b/mkdumprd index ba35800..9ee2f0b 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,12 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel + # "x-initrd.mount" mount failure will trigger isolate emergency service + # W/o this, systemd won't isolate, thus we won't get to emergency + # This only applicable for local fs mount + if ! is_nfs_target; then + _options="$_options,x-initrd.mount" + fi _mntopts="$_target $_fstype $_options" #for non-nfs _dev converting to use udev persistent name if [ -b "$_source" ]; then
On Fri, Jul 25, 2014 at 12:13:45AM +0800, WANG Chao wrote:
Now when mount in /etc/fstab fails, systemd would not consider it as critical and it would continue to boot. In fact, emergency service is triggered, but not in a isolation mode, and it results in the emergency service getting shutdown at some point later of the boot process. We need isolation otherwise we won't see any emergency service.
That is because in kdump initramfs, mount units specified in /etc/fstab are required by "local-fs.target". When any of these mounts fails, local-fs.target fails.
For kdump initramfs, we need to isolate to emergency service on any of the mount failure, that said, every service should be stopped and onlu emergency service would run. But local-fs.target won't trigger that on its failure. That means in case of mount failure, local-fs.target also enters failure state, but all the service will continue without any interruption.
After digging looking into source code of systemd-fstab-generator. I find "x-initrd.mount" using in initramfs mount, will make the mount units required by "initrd-root-fs.target" rather than it's used to be "local-fs.target".
"initrd-root-fs.target" is suitable to us because if it fails, it will isolate to emergency service. That means in case of any mount failure, the emergeny service will start and everything else will stop. We want this effect because we need to take kdump fail-safe action when there's a mount failure.
From systemd unit point of view, "initrd-root-fs.target" has
OnFailureIsolate=yes, but "local-fs.target" doesn't. From systemd.unit(5):
OnFailureIsolate= Takes a boolean argument. If true, the unit listed in OnFailure= will be enqueued in isolation mode, i.e. all units that are not its dependency will be stopped. If this is set, only a single unit may be listed in OnFailure=. Defaults to false.
NOTE: Harald who contributed "x-initrd.mount" in systemd, confirmed that this feature will stay.
Signed-off-by: WANG Chao chaowang@redhat.com
mkdumprd | 6 ++++++ 1 file changed, 6 insertions(+)
diff --git a/mkdumprd b/mkdumprd index ba35800..9ee2f0b 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,12 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel
- # "x-initrd.mount" mount failure will trigger isolate emergency service
- # W/o this, systemd won't isolate, thus we won't get to emergency
- # This only applicable for local fs mount
- if ! is_nfs_target; then
_options="$_options,x-initrd.mount"- fi
So why is it applicable to local fs mounts only. What will happen in case of remote mount. Do we already isolate to emergency service in that case?
Also what happens in case of raw disk? What happens in case of ssh dump when network interface does not show up?
Thanks Vivek
On 07/24/14 at 01:35pm, Vivek Goyal wrote:
On Fri, Jul 25, 2014 at 12:13:45AM +0800, WANG Chao wrote:
Now when mount in /etc/fstab fails, systemd would not consider it as critical and it would continue to boot. In fact, emergency service is triggered, but not in a isolation mode, and it results in the emergency service getting shutdown at some point later of the boot process. We need isolation otherwise we won't see any emergency service.
That is because in kdump initramfs, mount units specified in /etc/fstab are required by "local-fs.target". When any of these mounts fails, local-fs.target fails.
For kdump initramfs, we need to isolate to emergency service on any of the mount failure, that said, every service should be stopped and onlu emergency service would run. But local-fs.target won't trigger that on its failure. That means in case of mount failure, local-fs.target also enters failure state, but all the service will continue without any interruption.
After digging looking into source code of systemd-fstab-generator. I find "x-initrd.mount" using in initramfs mount, will make the mount units required by "initrd-root-fs.target" rather than it's used to be "local-fs.target".
"initrd-root-fs.target" is suitable to us because if it fails, it will isolate to emergency service. That means in case of any mount failure, the emergeny service will start and everything else will stop. We want this effect because we need to take kdump fail-safe action when there's a mount failure.
From systemd unit point of view, "initrd-root-fs.target" has
OnFailureIsolate=yes, but "local-fs.target" doesn't. From systemd.unit(5):
OnFailureIsolate= Takes a boolean argument. If true, the unit listed in OnFailure= will be enqueued in isolation mode, i.e. all units that are not its dependency will be stopped. If this is set, only a single unit may be listed in OnFailure=. Defaults to false.
NOTE: Harald who contributed "x-initrd.mount" in systemd, confirmed that this feature will stay.
Signed-off-by: WANG Chao chaowang@redhat.com
mkdumprd | 6 ++++++ 1 file changed, 6 insertions(+)
diff --git a/mkdumprd b/mkdumprd index ba35800..9ee2f0b 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,12 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel
- # "x-initrd.mount" mount failure will trigger isolate emergency service
- # W/o this, systemd won't isolate, thus we won't get to emergency
- # This only applicable for local fs mount
- if ! is_nfs_target; then
_options="$_options,x-initrd.mount"- fi
So why is it applicable to local fs mounts only. What will happen in case of remote mount. Do we already isolate to emergency service in that case?
If we use "x-initrd.mount", remote mount will become required by "initrd-root-fs.target", instead of "remote-fs.target". That's how it is handled within systemd internal.
We need remote mount to be required "remote-fs.target", because we need to bring up network before any remote mount and "remote-fs.target" can be a checkpoint of that.
Also what happens in case of raw disk? What happens in case of ssh dump when network interface does not show up?
In dracut-initqueue hook, both network and disk will be brought up, if needed. And kdump.sh runs after dracut-initqueue, so network and disk will be there before running kdump.sh.
Thanks WANG Chao
On Fri, Jul 25, 2014 at 01:44:40PM +0800, WANG Chao wrote:
On 07/24/14 at 01:35pm, Vivek Goyal wrote:
On Fri, Jul 25, 2014 at 12:13:45AM +0800, WANG Chao wrote:
Now when mount in /etc/fstab fails, systemd would not consider it as critical and it would continue to boot. In fact, emergency service is triggered, but not in a isolation mode, and it results in the emergency service getting shutdown at some point later of the boot process. We need isolation otherwise we won't see any emergency service.
That is because in kdump initramfs, mount units specified in /etc/fstab are required by "local-fs.target". When any of these mounts fails, local-fs.target fails.
For kdump initramfs, we need to isolate to emergency service on any of the mount failure, that said, every service should be stopped and onlu emergency service would run. But local-fs.target won't trigger that on its failure. That means in case of mount failure, local-fs.target also enters failure state, but all the service will continue without any interruption.
After digging looking into source code of systemd-fstab-generator. I find "x-initrd.mount" using in initramfs mount, will make the mount units required by "initrd-root-fs.target" rather than it's used to be "local-fs.target".
"initrd-root-fs.target" is suitable to us because if it fails, it will isolate to emergency service. That means in case of any mount failure, the emergeny service will start and everything else will stop. We want this effect because we need to take kdump fail-safe action when there's a mount failure.
From systemd unit point of view, "initrd-root-fs.target" has
OnFailureIsolate=yes, but "local-fs.target" doesn't. From systemd.unit(5):
OnFailureIsolate= Takes a boolean argument. If true, the unit listed in OnFailure= will be enqueued in isolation mode, i.e. all units that are not its dependency will be stopped. If this is set, only a single unit may be listed in OnFailure=. Defaults to false.
NOTE: Harald who contributed "x-initrd.mount" in systemd, confirmed that this feature will stay.
Signed-off-by: WANG Chao chaowang@redhat.com
mkdumprd | 6 ++++++ 1 file changed, 6 insertions(+)
diff --git a/mkdumprd b/mkdumprd index ba35800..9ee2f0b 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,12 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel
- # "x-initrd.mount" mount failure will trigger isolate emergency service
- # W/o this, systemd won't isolate, thus we won't get to emergency
- # This only applicable for local fs mount
- if ! is_nfs_target; then
_options="$_options,x-initrd.mount"- fi
So why is it applicable to local fs mounts only. What will happen in case of remote mount. Do we already isolate to emergency service in that case?
If we use "x-initrd.mount", remote mount will become required by "initrd-root-fs.target", instead of "remote-fs.target". That's how it is handled within systemd internal.
We need remote mount to be required "remote-fs.target", because we need to bring up network before any remote mount and "remote-fs.target" can be a checkpoint of that.
So first of all this explanation should be part of the comment near the code.
Secondly, so what will happen if nfs mount fails and remote-fs.target does not reach. Will we isolate to kdump error handler or not? So handling of remote-fs.target is different from local-fs.target when it comes to emergency shell? I think all this needs to be part of changelog/comment so that reader can understand why are we treating remote mounts and local mounts differently.
Also what happens in case of raw disk? What happens in case of ssh dump when network interface does not show up?
In dracut-initqueue hook, both network and disk will be brought up, if needed. And kdump.sh runs after dracut-initqueue, so network and disk will be there before running kdump.sh.
So if dracut-initqueue fails/timesout, will we isolate to kdump emergency shell?
Thanks Vivek
On 07/25/14 at 08:01am, Vivek Goyal wrote:
On Fri, Jul 25, 2014 at 01:44:40PM +0800, WANG Chao wrote:
On 07/24/14 at 01:35pm, Vivek Goyal wrote:
On Fri, Jul 25, 2014 at 12:13:45AM +0800, WANG Chao wrote:
Now when mount in /etc/fstab fails, systemd would not consider it as critical and it would continue to boot. In fact, emergency service is triggered, but not in a isolation mode, and it results in the emergency service getting shutdown at some point later of the boot process. We need isolation otherwise we won't see any emergency service.
That is because in kdump initramfs, mount units specified in /etc/fstab are required by "local-fs.target". When any of these mounts fails, local-fs.target fails.
For kdump initramfs, we need to isolate to emergency service on any of the mount failure, that said, every service should be stopped and onlu emergency service would run. But local-fs.target won't trigger that on its failure. That means in case of mount failure, local-fs.target also enters failure state, but all the service will continue without any interruption.
After digging looking into source code of systemd-fstab-generator. I find "x-initrd.mount" using in initramfs mount, will make the mount units required by "initrd-root-fs.target" rather than it's used to be "local-fs.target".
"initrd-root-fs.target" is suitable to us because if it fails, it will isolate to emergency service. That means in case of any mount failure, the emergeny service will start and everything else will stop. We want this effect because we need to take kdump fail-safe action when there's a mount failure.
From systemd unit point of view, "initrd-root-fs.target" has
OnFailureIsolate=yes, but "local-fs.target" doesn't. From systemd.unit(5):
OnFailureIsolate= Takes a boolean argument. If true, the unit listed in OnFailure= will be enqueued in isolation mode, i.e. all units that are not its dependency will be stopped. If this is set, only a single unit may be listed in OnFailure=. Defaults to false.
NOTE: Harald who contributed "x-initrd.mount" in systemd, confirmed that this feature will stay.
Signed-off-by: WANG Chao chaowang@redhat.com
mkdumprd | 6 ++++++ 1 file changed, 6 insertions(+)
diff --git a/mkdumprd b/mkdumprd index ba35800..9ee2f0b 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,12 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel
- # "x-initrd.mount" mount failure will trigger isolate emergency service
- # W/o this, systemd won't isolate, thus we won't get to emergency
- # This only applicable for local fs mount
- if ! is_nfs_target; then
_options="$_options,x-initrd.mount"- fi
So why is it applicable to local fs mounts only. What will happen in case of remote mount. Do we already isolate to emergency service in that case?
If we use "x-initrd.mount", remote mount will become required by "initrd-root-fs.target", instead of "remote-fs.target". That's how it is handled within systemd internal.
We need remote mount to be required "remote-fs.target", because we need to bring up network before any remote mount and "remote-fs.target" can be a checkpoint of that.
So first of all this explanation should be part of the comment near the code.
Sure. Will do.
Secondly, so what will happen if nfs mount fails and remote-fs.target does not reach. Will we isolate to kdump error handler or not? So handling of remote-fs.target is different from local-fs.target when it comes to emergency shell? I think all this needs to be part of changelog/comment so that reader can understand why are we treating remote mounts and local mounts differently.
In case of nfs mount failure, we won't isolate kdump error handler right away. But will do in the later point in kdump.sh fails because the dump target isn't mounted.
Also what happens in case of raw disk? What happens in case of ssh dump when network interface does not show up?
In dracut-initqueue hook, both network and disk will be brought up, if needed. And kdump.sh runs after dracut-initqueue, so network and disk will be there before running kdump.sh.
So if dracut-initqueue fails/timesout, will we isolate to kdump emergency shell?
No. dracut-initqueue is configured by dracut and it's emergency handler is dracut-emergency.service.
In case of dracut-initqueue failure, I think we can still proceed kdump.sh and gets failed there.
Thanks WANG Chao
On Mon, Jul 28, 2014 at 05:13:52PM +0800, WANG Chao wrote:
On 07/25/14 at 08:01am, Vivek Goyal wrote:
On Fri, Jul 25, 2014 at 01:44:40PM +0800, WANG Chao wrote:
On 07/24/14 at 01:35pm, Vivek Goyal wrote:
On Fri, Jul 25, 2014 at 12:13:45AM +0800, WANG Chao wrote:
Now when mount in /etc/fstab fails, systemd would not consider it as critical and it would continue to boot. In fact, emergency service is triggered, but not in a isolation mode, and it results in the emergency service getting shutdown at some point later of the boot process. We need isolation otherwise we won't see any emergency service.
That is because in kdump initramfs, mount units specified in /etc/fstab are required by "local-fs.target". When any of these mounts fails, local-fs.target fails.
For kdump initramfs, we need to isolate to emergency service on any of the mount failure, that said, every service should be stopped and onlu emergency service would run. But local-fs.target won't trigger that on its failure. That means in case of mount failure, local-fs.target also enters failure state, but all the service will continue without any interruption.
After digging looking into source code of systemd-fstab-generator. I find "x-initrd.mount" using in initramfs mount, will make the mount units required by "initrd-root-fs.target" rather than it's used to be "local-fs.target".
"initrd-root-fs.target" is suitable to us because if it fails, it will isolate to emergency service. That means in case of any mount failure, the emergeny service will start and everything else will stop. We want this effect because we need to take kdump fail-safe action when there's a mount failure.
From systemd unit point of view, "initrd-root-fs.target" has
OnFailureIsolate=yes, but "local-fs.target" doesn't. From systemd.unit(5):
OnFailureIsolate= Takes a boolean argument. If true, the unit listed in OnFailure= will be enqueued in isolation mode, i.e. all units that are not its dependency will be stopped. If this is set, only a single unit may be listed in OnFailure=. Defaults to false.
NOTE: Harald who contributed "x-initrd.mount" in systemd, confirmed that this feature will stay.
Signed-off-by: WANG Chao chaowang@redhat.com
mkdumprd | 6 ++++++ 1 file changed, 6 insertions(+)
diff --git a/mkdumprd b/mkdumprd index ba35800..9ee2f0b 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,12 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel
- # "x-initrd.mount" mount failure will trigger isolate emergency service
- # W/o this, systemd won't isolate, thus we won't get to emergency
- # This only applicable for local fs mount
- if ! is_nfs_target; then
_options="$_options,x-initrd.mount"- fi
So why is it applicable to local fs mounts only. What will happen in case of remote mount. Do we already isolate to emergency service in that case?
If we use "x-initrd.mount", remote mount will become required by "initrd-root-fs.target", instead of "remote-fs.target". That's how it is handled within systemd internal.
We need remote mount to be required "remote-fs.target", because we need to bring up network before any remote mount and "remote-fs.target" can be a checkpoint of that.
So first of all this explanation should be part of the comment near the code.
Sure. Will do.
Secondly, so what will happen if nfs mount fails and remote-fs.target does not reach. Will we isolate to kdump error handler or not? So handling of remote-fs.target is different from local-fs.target when it comes to emergency shell? I think all this needs to be part of changelog/comment so that reader can understand why are we treating remote mounts and local mounts differently.
In case of nfs mount failure, we won't isolate kdump error handler right away. But will do in the later point in kdump.sh fails because the dump target isn't mounted.
And why do we treat those two cases differently? One problem with local mounts was that we will never hit the kdump.sh because some intermediate targets have not reached. That's why we had to isolate to emergency shell immediately.
So in case of remote mounts we don't have the same issue? We will still hit kdump.sh, no matter what.
Also what happens in case of raw disk? What happens in case of ssh dump when network interface does not show up?
In dracut-initqueue hook, both network and disk will be brought up, if needed. And kdump.sh runs after dracut-initqueue, so network and disk will be there before running kdump.sh.
So if dracut-initqueue fails/timesout, will we isolate to kdump emergency shell?
No. dracut-initqueue is configured by dracut and it's emergency handler is dracut-emergency.service.
But I thought we disabled dracut-emergency.service. So are you saying that if failure happens early, then dracut emergency handler will run. But that will put us on shell instead of rebooting?
In case of dracut-initqueue failure, I think we can still proceed kdump.sh and gets failed there.
Not sure what does this mean.
Thanks Vivek
On 07/28/14 at 08:56am, Vivek Goyal wrote:
On Mon, Jul 28, 2014 at 05:13:52PM +0800, WANG Chao wrote:
On 07/25/14 at 08:01am, Vivek Goyal wrote:
On Fri, Jul 25, 2014 at 01:44:40PM +0800, WANG Chao wrote:
On 07/24/14 at 01:35pm, Vivek Goyal wrote:
On Fri, Jul 25, 2014 at 12:13:45AM +0800, WANG Chao wrote:
Now when mount in /etc/fstab fails, systemd would not consider it as critical and it would continue to boot. In fact, emergency service is triggered, but not in a isolation mode, and it results in the emergency service getting shutdown at some point later of the boot process. We need isolation otherwise we won't see any emergency service.
That is because in kdump initramfs, mount units specified in /etc/fstab are required by "local-fs.target". When any of these mounts fails, local-fs.target fails.
For kdump initramfs, we need to isolate to emergency service on any of the mount failure, that said, every service should be stopped and onlu emergency service would run. But local-fs.target won't trigger that on its failure. That means in case of mount failure, local-fs.target also enters failure state, but all the service will continue without any interruption.
After digging looking into source code of systemd-fstab-generator. I find "x-initrd.mount" using in initramfs mount, will make the mount units required by "initrd-root-fs.target" rather than it's used to be "local-fs.target".
"initrd-root-fs.target" is suitable to us because if it fails, it will isolate to emergency service. That means in case of any mount failure, the emergeny service will start and everything else will stop. We want this effect because we need to take kdump fail-safe action when there's a mount failure.
>From systemd unit point of view, "initrd-root-fs.target" has OnFailureIsolate=yes, but "local-fs.target" doesn't. From systemd.unit(5):
OnFailureIsolate= Takes a boolean argument. If true, the unit listed in OnFailure= will be enqueued in isolation mode, i.e. all units that are not its dependency will be stopped. If this is set, only a single unit may be listed in OnFailure=. Defaults to false.
NOTE: Harald who contributed "x-initrd.mount" in systemd, confirmed that this feature will stay.
Signed-off-by: WANG Chao chaowang@redhat.com
mkdumprd | 6 ++++++ 1 file changed, 6 insertions(+)
diff --git a/mkdumprd b/mkdumprd index ba35800..9ee2f0b 100644 --- a/mkdumprd +++ b/mkdumprd @@ -106,6 +106,12 @@ to_mount() { _fstype=$(findmnt -k -f -n -r -o FSTYPE $_dev) _options=$(findmnt -k -f -n -r -o OPTIONS $_dev) _options=${_options/#ro/rw} #mount fs target as rw in 2nd kernel
- # "x-initrd.mount" mount failure will trigger isolate emergency service
- # W/o this, systemd won't isolate, thus we won't get to emergency
- # This only applicable for local fs mount
- if ! is_nfs_target; then
_options="$_options,x-initrd.mount"- fi
So why is it applicable to local fs mounts only. What will happen in case of remote mount. Do we already isolate to emergency service in that case?
If we use "x-initrd.mount", remote mount will become required by "initrd-root-fs.target", instead of "remote-fs.target". That's how it is handled within systemd internal.
We need remote mount to be required "remote-fs.target", because we need to bring up network before any remote mount and "remote-fs.target" can be a checkpoint of that.
So first of all this explanation should be part of the comment near the code.
Sure. Will do.
Secondly, so what will happen if nfs mount fails and remote-fs.target does not reach. Will we isolate to kdump error handler or not? So handling of remote-fs.target is different from local-fs.target when it comes to emergency shell? I think all this needs to be part of changelog/comment so that reader can understand why are we treating remote mounts and local mounts differently.
In case of nfs mount failure, we won't isolate kdump error handler right away. But will do in the later point in kdump.sh fails because the dump target isn't mounted.
And why do we treat those two cases differently? One problem with local mounts was that we will never hit the kdump.sh because some intermediate targets have not reached. That's why we had to isolate to emergency shell immediately.
So in case of remote mounts we don't have the same issue? We will still hit kdump.sh, no matter what.
In fact, "remote-fs-pre.target" plays a role as a checkpoint for network is brought up.
Say we use "initrd-root-fs.target" for nfs mount, systemd could start to mount nfs before network is OK. nfs mount would fail.
Now we use "remote-fs.target", systemd will start mount nfs after "remote-fs-pre.target". The call graph looks like:
remote-fs-pre.target -> bring up network -> nfs.mount -> remote-fs.target
The reason we treat nfs and local mount diffferent because nfs mount must be started after network is ok but local mount doesn't need network.
Also what happens in case of raw disk? What happens in case of ssh dump when network interface does not show up?
In dracut-initqueue hook, both network and disk will be brought up, if needed. And kdump.sh runs after dracut-initqueue, so network and disk will be there before running kdump.sh.
So if dracut-initqueue fails/timesout, will we isolate to kdump emergency shell?
No. dracut-initqueue is configured by dracut and it's emergency handler is dracut-emergency.service.
But I thought we disabled dracut-emergency.service. So are you saying that if failure happens early, then dracut emergency handler will run. But that will put us on shell instead of rebooting?
We've disabled dracut-emergency.service, so failure of dracut-initqueue is ignored and systemd would just continue to run kdump.sh
Rethinking another round, I think it's better to enable dracut-emergency service but replace it with our own error handler. So that we can fail early and run kdump error handler, instead of ignoring the failure.
Thanks WANG Chao
In case of dracut-initqueue failure, I think we can still proceed kdump.sh and gets failed there.
Not sure what does this mean.
Thanks Vivek
On Mon, Jul 28, 2014 at 09:48:41PM +0800, WANG Chao wrote:
[..]
No. dracut-initqueue is configured by dracut and it's emergency handler is dracut-emergency.service.
But I thought we disabled dracut-emergency.service. So are you saying that if failure happens early, then dracut emergency handler will run. But that will put us on shell instead of rebooting?
We've disabled dracut-emergency.service, so failure of dracut-initqueue is ignored and systemd would just continue to run kdump.sh
Rethinking another round, I think it's better to enable dracut-emergency service but replace it with our own error handler. So that we can fail early and run kdump error handler, instead of ignoring the failure.
I agree. If error happens early, we should launch our own error handler early instead of continuing to boot.
This will limit our choices of what we can do in error handling. I am not sure what will happen if you try to start initqueue and mount root again inside error handler (dump_to_rootfs), when error has happened this early.
Thanks Vivek
On 07/29/14 at 09:00am, Vivek Goyal wrote:
On Mon, Jul 28, 2014 at 09:48:41PM +0800, WANG Chao wrote:
[..]
No. dracut-initqueue is configured by dracut and it's emergency handler is dracut-emergency.service.
But I thought we disabled dracut-emergency.service. So are you saying that if failure happens early, then dracut emergency handler will run. But that will put us on shell instead of rebooting?
We've disabled dracut-emergency.service, so failure of dracut-initqueue is ignored and systemd would just continue to run kdump.sh
Rethinking another round, I think it's better to enable dracut-emergency service but replace it with our own error handler. So that we can fail early and run kdump error handler, instead of ignoring the failure.
I agree. If error happens early, we should launch our own error handler early instead of continuing to boot.
This will limit our choices of what we can do in error handling. I am not sure what will happen if you try to start initqueue and mount root again inside error handler (dump_to_rootfs), when error has happened this early.
I just sent an update for both cleanup and error handling. Please take a look when you have time.
Thanks WANG Chao
On Tue, Jul 22, 2014 at 03:17:58PM +0800, WANG Chao wrote:
Extract functions from kdump.sh, and construct kdump-lib-initramfs.sh as kdump common functions/varaibles library.
kdump-lib-initramfs.sh will include kdump-lib.sh, because it will use the functions from there. IOW, kdump-lib-initramfs.sh will be a superset of kdump-lib.sh
So after this cleanup:
- scripts running in 1st kernel only have to include kdump-lib.sh
- scripts running in 2nd kernel only have to include kdump-lib-initramfs.sh
Signed-off-by: WANG Chao chaowang@redhat.com
Acked-by: Vivek Goyal vgoyal@redhat.com
Vivek
dracut-kdump.sh | 145 ++---------------------------------------------- dracut-module-setup.sh | 1 + kdump-lib-initramfs.sh | 146 +++++++++++++++++++++++++++++++++++++++++++++++++ kexec-tools.spec | 2 + 4 files changed, 154 insertions(+), 140 deletions(-) create mode 100755 kdump-lib-initramfs.sh
diff --git a/dracut-kdump.sh b/dracut-kdump.sh index cb13d92..08f4322 100755 --- a/dracut-kdump.sh +++ b/dracut-kdump.sh @@ -2,30 +2,14 @@
exec &> /dev/console . /lib/dracut-lib.sh -. /lib/kdump-lib.sh +. /lib/kdump-lib-initramfs.sh
if [ -f "$initdir/lib/dracut/no-emergency-shell" ]; then rm -f -- $initdir/lib/dracut/no-emergency-shell fi
set -o pipefail -KDUMP_PATH="/var/crash" -CORE_COLLECTOR="" -DEFAULT_CORE_COLLECTOR="makedumpfile -l --message-level 1 -d 31" -DMESG_COLLECTOR="/sbin/vmcore-dmesg" -DEFAULT_ACTION="reboot -f" -DATEDIR=`date +%Y.%m.%d-%T` -HOST_IP='127.0.0.1' -DUMP_INSTRUCTION="" -SSH_KEY_LOCATION="/root/.ssh/kdump_id_rsa" -KDUMP_SCRIPT_DIR="/kdumpscripts" -DD_BLKSIZE=512 -FINAL_ACTION="reboot -f" DUMP_RETVAL=0 -conf_file="/etc/kdump.conf" -KDUMP_PRE="" -KDUMP_POST="" -MOUNTS=""
export PATH=$PATH:$KDUMP_SCRIPT_DIR
@@ -43,27 +27,6 @@ do_dump() return $_ret }
-do_umount() -{
- if [ -n "$MOUNTS" ]; then
for mount in $MOUNTS; doismounted $mount && umount -R $mountdone- fi
-}
-do_final_action() -{
- do_umount
- eval $FINAL_ACTION
-}
-do_default_action() -{
- wait_for_loginit
- eval $DEFAULT_ACTION
-}
do_kdump_pre() { if [ -n "$KDUMP_PRE" ]; then @@ -83,39 +46,6 @@ add_dump_code() DUMP_INSTRUCTION=$1 }
-# dump_fs <mount point| device> -dump_fs() -{
- local _dev=$(findmnt -k -f -n -r -o SOURCE $1)
- local _mp=$(findmnt -k -f -n -r -o TARGET $1)
- echo "kdump: dump target is $_dev"
- if [ -z "$_mp" ]; then
echo "kdump: error: Dump target $_dev is not mounted."return 1- fi
- MOUNTS="$MOUNTS $_mp"
- # Remove -F in makedumpfile case. We don't want a flat format dump here.
- [[ $CORE_COLLECTOR = *makedumpfile* ]] && CORE_COLLECTOR=`echo $CORE_COLLECTOR | sed -e "s/-F//g"`
- echo "kdump: saving to $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/"
- mount -o remount,rw $_mp || return 1
- mkdir -p $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR || return 1
- save_vmcore_dmesg_fs ${DMESG_COLLECTOR} "$_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/"
- echo "kdump: saving vmcore"
- $CORE_COLLECTOR /proc/vmcore $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete || return 1
- mv $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore
- sync
- echo "kdump: saving vmcore complete"
- return 0
-}
dump_raw() { local _raw=$1 @@ -165,21 +95,6 @@ dump_ssh() return 0 }
-save_vmcore_dmesg_fs() {
- local _dmesg_collector=$1
- local _path=$2
- echo "kdump: saving vmcore-dmesg.txt"
- $_dmesg_collector /proc/vmcore > ${_path}/vmcore-dmesg-incomplete.txt
- _exitcode=$?
- if [ $_exitcode -eq 0 ]; then
mv ${_path}/vmcore-dmesg-incomplete.txt ${_path}/vmcore-dmesg.txtecho "kdump: saving vmcore-dmesg.txt complete"- else
echo "kdump: saving vmcore-dmesg.txt failed"- fi
-}
save_vmcore_dmesg_ssh() { local _dmesg_collector=$1 local _path=$2 @@ -198,7 +113,6 @@ save_vmcore_dmesg_ssh() { fi }
get_host_ip() { local _host @@ -218,61 +132,12 @@ get_host_ip()
read_kdump_conf() {
- if [ ! -f "$conf_file" ]; then
echo "kdump: $conf_file not found"
- if [ ! -f "$KDUMP_CONF" ]; then
fiecho "kdump: $KDUMP_CONF not found" return
- # first get the necessary variables
- while read config_opt config_val;
- do
# remove inline comments after the end of a directive.config_val=$(strip_comments $config_val)case "$config_opt" inpath)KDUMP_PATH="$config_val";;core_collector)[ -n "$config_val" ] && CORE_COLLECTOR="$config_val";;sshkey)if [ -f "$config_val" ]; thenSSH_KEY_LOCATION=$config_valfi;;kdump_pre)KDUMP_PRE="$config_val";;kdump_post)KDUMP_POST="$config_val";;fence_kdump_args)FENCE_KDUMP_ARGS="$config_val";;fence_kdump_nodes)FENCE_KDUMP_NODES="$config_val";;default)case $config_val inshell)DEFAULT_ACTION="_emergency_shell kdump";;reboot)DEFAULT_ACTION="do_umount; reboot -f";;halt)DEFAULT_ACTION="do_umount; halt -f";;poweroff)DEFAULT_ACTION="do_umount; poweroff -f";;dump_to_rootfs)DEFAULT_ACTION="dump_fs $NEWROOT";;esac;;esac- done < $conf_file
get_kdump_confs
# rescan for add code for dump target while read config_opt config_val;
@@ -290,7 +155,7 @@ read_kdump_conf() add_dump_code "dump_ssh $SSH_KEY_LOCATION $config_val" ;; esac
- done < $conf_file
- done < $KDUMP_CONF
}
fence_kdump_notify() diff --git a/dracut-module-setup.sh b/dracut-module-setup.sh index 87ad072..84bf975 100755 --- a/dracut-module-setup.sh +++ b/dracut-module-setup.sh @@ -555,6 +555,7 @@ install() { inst "/sbin/vmcore-dmesg" "/sbin/vmcore-dmesg" inst_hook pre-pivot 9999 "$moddir/kdump.sh" inst "/lib/kdump/kdump-lib.sh" "/lib/kdump-lib.sh"
inst "/lib/kdump/kdump-lib-initramfs.sh" "/lib/kdump-lib-initramfs.sh"
# Check for all the devices and if any device is iscsi, bring up iscsi # target. Ideally all this should be pushed into dracut iscsi module
diff --git a/kdump-lib-initramfs.sh b/kdump-lib-initramfs.sh new file mode 100755 index 0000000..e2807c7 --- /dev/null +++ b/kdump-lib-initramfs.sh @@ -0,0 +1,146 @@ +# These variables and functions are useful in 2nd kernel
+. /lib/dracut-lib.sh +. /lib/kdump-lib.sh
+KDUMP_PATH="/var/crash" +CORE_COLLECTOR="" +DEFAULT_CORE_COLLECTOR="makedumpfile -l --message-level 1 -d 31" +DMESG_COLLECTOR="/sbin/vmcore-dmesg" +DEFAULT_ACTION="reboot -f" +DATEDIR=`date +%Y.%m.%d-%T` +HOST_IP='127.0.0.1' +DUMP_INSTRUCTION="" +SSH_KEY_LOCATION="/root/.ssh/kdump_id_rsa" +KDUMP_SCRIPT_DIR="/kdumpscripts" +DD_BLKSIZE=512 +FINAL_ACTION="reboot -f" +KDUMP_CONF="/etc/kdump.conf" +KDUMP_PRE="" +KDUMP_POST="" +NEWROOT="/sysroot" +MOUNTS=""
+get_kdump_confs() +{
- local config_opt config_val
- while read config_opt config_val;
- do
# remove inline comments after the end of a directive.config_val=$(strip_comments $config_val)case "$config_opt" inpath)KDUMP_PATH="$config_val";;core_collector)[ -n "$config_val" ] && CORE_COLLECTOR="$config_val";;sshkey)if [ -f "$config_val" ]; thenSSH_KEY_LOCATION=$config_valfi;;kdump_pre)KDUMP_PRE="$config_val";;kdump_post)KDUMP_POST="$config_val";;fence_kdump_args)FENCE_KDUMP_ARGS="$config_val";;fence_kdump_nodes)FENCE_KDUMP_NODES="$config_val";;default)case $config_val inshell)DEFAULT_ACTION="_emergency_shell kdump";;reboot)DEFAULT_ACTION="do_umount; reboot -f";;halt)DEFAULT_ACTION="do_umount; halt -f";;poweroff)DEFAULT_ACTION="do_umount; poweroff -f";;dump_to_rootfs)DEFAULT_ACTION="dump_fs $NEWROOT";;esac;;esac- done < $KDUMP_CONF
+}
+# dump_fs <mount point| device> +dump_fs() +{
- local _dev=$(findmnt -k -f -n -r -o SOURCE $1)
- local _mp=$(findmnt -k -f -n -r -o TARGET $1)
- echo "kdump: dump target is $_dev"
- if [ -z "$_mp" ]; then
echo "kdump: error: Dump target $_dev is not mounted."return 1- fi
- MOUNTS="$MOUNTS $_mp"
- # Remove -F in makedumpfile case. We don't want a flat format dump here.
- [[ $CORE_COLLECTOR = *makedumpfile* ]] && CORE_COLLECTOR=`echo $CORE_COLLECTOR | sed -e "s/-F//g"`
- echo "kdump: saving to $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/"
- mount -o remount,rw $_mp || return 1
- mkdir -p $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR || return 1
- save_vmcore_dmesg_fs ${DMESG_COLLECTOR} "$_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/"
- echo "kdump: saving vmcore"
- $CORE_COLLECTOR /proc/vmcore $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete || return 1
- mv $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore
- sync
- echo "kdump: saving vmcore complete"
+}
+save_vmcore_dmesg_fs() {
- local _dmesg_collector=$1
- local _path=$2
- echo "kdump: saving vmcore-dmesg.txt"
- $_dmesg_collector /proc/vmcore > ${_path}/vmcore-dmesg-incomplete.txt
- _exitcode=$?
- if [ $_exitcode -eq 0 ]; then
mv ${_path}/vmcore-dmesg-incomplete.txt ${_path}/vmcore-dmesg.txtecho "kdump: saving vmcore-dmesg.txt complete"- else
echo "kdump: saving vmcore-dmesg.txt failed"- fi
+}
+do_umount() +{
- if [ -n "$MOUNTS" ]; then
for mount in $MOUNTS; doismounted $mount && umount -R $mountdone- fi
+}
+do_default_action() +{
- wait_for_loginit
- eval $DEFAULT_ACTION
+}
+do_final_action() +{
- do_umount
- eval $FINAL_ACTION
+} diff --git a/kexec-tools.spec b/kexec-tools.spec index 25dae2b..a613b43 100644 --- a/kexec-tools.spec +++ b/kexec-tools.spec @@ -27,6 +27,7 @@ Source20: kdump-lib.sh Source21: kdump-in-cluster-environment.txt Source22: kdump-dep-generator.sh Source23: kdump-anaconda-addon-20140721.tar.gz +Source24: kdump-lib-initramfs.sh
####################################### # These are sources for mkdumpramfs @@ -178,6 +179,7 @@ install -m 644 kexec/kexec.8 $RPM_BUILD_ROOT%{_mandir}/man8/kexec.8 install -m 755 %{SOURCE11} $RPM_BUILD_ROOT%{_datadir}/kdump/firstboot_kdump.py install -m 644 %{SOURCE12} $RPM_BUILD_ROOT%{_mandir}/man8/mkdumprd.8 install -m 755 %{SOURCE20} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-lib.sh +install -m 755 %{SOURCE24} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-lib-initramfs.sh %ifnarch s390x # For s390x the ELF header is created in the kdump kernel and therefore kexec
# udev rules are not required
1.9.3