[PATCH v3 2/4] selftest: Add basic test framework

[Kairui Song]

Now, by execute `make test-run` in tests/, kexec-tools sanity tests
will be performed using VMs. There are currently 3 test cases, for local
kdump, nfs kdump and ssh kdump.

For each test VM, the selftest framework will create a snapshot layer,
do setup as required by the test case, this ensure each test runs in a
clean VM.

This framework will install a custom systemd service that starts when
system have finished booting, and the service will do basic routine
(fetch and set boot counter, etc..), then call the test case which is
installed in /kexec-kdump-test/test.sh in VM.

Each VM will have two serial consoles, one for ordinary console usage,
one for the test communication and log. The test script will watch the
second test console to know the test status.

The test cases are located in tests/scripts/testcases, documents about
the test cases structure will be provided in following commits.

Signed-off-by: Kairui Song <kasong(a)redhat.com&gt;
---
 tests/Makefile                                |   8 +
 tests/scripts/build-scripts/base-image.sh     |   1 -
 .../scripts/build-scripts/test-base-image.sh  |   6 +
 tests/scripts/build-scripts/test-image.sh     |  21 +++
 tests/scripts/kexec-kdump-test/init.sh        | 112 +++++++++++
 .../kexec-kdump-test/kexec-kdump-test.service |   9 +
 tests/scripts/kexec-kdump-test/test.sh        |  15 ++
 tests/scripts/run-test.sh                     | 131 +++++++++++++
 tests/scripts/spawn-image-shell.sh            |  16 ++
 tests/scripts/test-lib.sh                     | 177 ++++++++++++++++++
 .../scripts/testcases/local-kdump/0-local.sh  |  32 ++++
 tests/scripts/testcases/nfs-kdump/0-server.sh |  38 ++++
 tests/scripts/testcases/nfs-kdump/1-client.sh |  30 +++
 tests/scripts/testcases/ssh-kdump/0-server.sh |  34 ++++
 tests/scripts/testcases/ssh-kdump/1-client.sh |  40 ++++
 15 files changed, 669 insertions(+), 1 deletion(-)
 create mode 100755 tests/scripts/build-scripts/test-image.sh
 create mode 100755 tests/scripts/kexec-kdump-test/init.sh
 create mode 100644 tests/scripts/kexec-kdump-test/kexec-kdump-test.service
 create mode 100755 tests/scripts/kexec-kdump-test/test.sh
 create mode 100755 tests/scripts/run-test.sh
 create mode 100755 tests/scripts/spawn-image-shell.sh
 create mode 100644 tests/scripts/test-lib.sh
 create mode 100755 tests/scripts/testcases/local-kdump/0-local.sh
 create mode 100755 tests/scripts/testcases/nfs-kdump/0-server.sh
 create mode 100755 tests/scripts/testcases/nfs-kdump/1-client.sh
 create mode 100755 tests/scripts/testcases/ssh-kdump/0-server.sh
 create mode 100755 tests/scripts/testcases/ssh-kdump/1-client.sh

diff --git a/tests/Makefile b/tests/Makefile
index a352ab3..6144809 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,3 +1,4 @@
+TEST_CASE ?=
 BASE_IMAGE ?=
 
 TEST_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
@@ -75,6 +76,13 @@ $(TEST_ROOT)/output/test-base-image: $(BUILD_ROOT)/inst-base-image
$(KEXEC_TOOLS
 		$(TEST_ROOT)/scripts/build-scripts/test-base-image.sh \
 		$(KEXEC_TOOLS_RPM)
 
+test-run: $(TEST_ROOT)/output/test-base-image
+ifeq ($(strip $(TEST_CASE)),)
+	$(TEST_ROOT)/scripts/run-test.sh
+else
+	$(TEST_ROOT)/scripts/run-test.sh --console $(TEST_CASE)
+endif
+
 clean:
 	rm -rf $(TEST_ROOT)/build
 	rm -rf $(TEST_ROOT)/output
diff --git a/tests/scripts/build-scripts/base-image.sh
b/tests/scripts/build-scripts/base-image.sh
index 59f4574..62624e4 100755
--- a/tests/scripts/build-scripts/base-image.sh
+++ b/tests/scripts/build-scripts/base-image.sh
@@ -8,4 +8,3 @@ img_inst_pkg grubby\
 img_run_cmd "grubby --args systemd.journald.forward_to_console=1
systemd.log_target=console --update-kernel ALL"
 img_run_cmd "grubby --args selinux=0 --update-kernel ALL"
 img_run_cmd "grubby --args crashkernel=224M --update-kernel ALL"
-img_run_cmd "mkdir -p /kexec-kdump-test"
diff --git a/tests/scripts/build-scripts/test-base-image.sh
b/tests/scripts/build-scripts/test-base-image.sh
index a649306..5e2d86a 100755
--- a/tests/scripts/build-scripts/test-base-image.sh
+++ b/tests/scripts/build-scripts/test-base-image.sh
@@ -10,6 +10,12 @@ for _rpm in $@; do
 	fi
 done
 
+img_run_cmd "mkdir -p /kexec-kdump-test"
+img_inst $TESTDIR/scripts/kexec-kdump-test/init.sh /kexec-kdump-test/init.sh
+img_inst $TESTDIR/scripts/kexec-kdump-test/test.sh /kexec-kdump-test/test.sh
+img_inst $TESTDIR/scripts/kexec-kdump-test/kexec-kdump-test.service
/etc/systemd/system/kexec-kdump-test.service
+img_run_cmd "systemctl enable kexec-kdump-test.service"
+
 img_inst_pkg $TEST_RPMS
 # Test script should start kdump manually to save time
 img_run_cmd "systemctl disable kdump.service"
diff --git a/tests/scripts/build-scripts/test-image.sh
b/tests/scripts/build-scripts/test-image.sh
new file mode 100755
index 0000000..d8e907e
--- /dev/null
+++ b/tests/scripts/build-scripts/test-image.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+. $TESTDIR/scripts/test-lib.sh
+TEST_SCRIPT=$1
+
+QEMU_CMD="$DEFAULT_QEMU_CMD \
+-serial stdio \
+-serial file:$(get_test_output_file $TEST_SCRIPT) \
+-monitor none \
+-hda $OUTPUT_IMAGE"
+
+img_add_qemu_cmd() {
+	QEMU_CMD+=" $@"
+}
+
+source $TEST_SCRIPT
+
+on_build
+
+img_inst $TEST_SCRIPT /kexec-kdump-test/test.sh
+
+echo $QEMU_CMD > $(get_test_qemu_cmd_file $TEST_SCRIPT)
diff --git a/tests/scripts/kexec-kdump-test/init.sh
b/tests/scripts/kexec-kdump-test/init.sh
new file mode 100755
index 0000000..bc2a642
--- /dev/null
+++ b/tests/scripts/kexec-kdump-test/init.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env sh
+BOOT_ARG="test_boot_count"
+_YELLOW='\033[1;33m'
+_GREEN='\033[0;32m'
+_RED='\033[0;31m'
+_NC='\033[0m' # No Color
+
+if [ -n "$(cat /proc/cmdline | grep "\bno_test\b")" ]; then
+	exit 0
+fi
+
+get_test_boot_count() {
+	local boot_count=$(cat /proc/cmdline | sed -n
"s/.*$BOOT_ARG=\([0-9]*\).*/\1/p")
+
+	if [ -z "$boot_count" ]; then
+		boot_count=1
+	fi
+
+	echo $boot_count
+}
+
+test_output() {
+	echo $@ > /dev/ttyS1
+	echo $@ > /dev/ttyS0
+
+	sync
+}
+
+test_passed() {
+	echo -e "${_GREEN}TEST PASSED${_NC}" > /dev/ttyS1
+	echo -e "${_GREEN}kexec-kdump-test: TEST PASSED${_NC}" > /dev/ttyS0
+
+	echo $@ > /dev/ttyS1
+	echo $@ > /dev/ttyS0
+
+	sync
+
+	shutdown -h 0
+
+	exit 0
+}
+
+test_failed() {
+	echo -e "${_RED}TEST FAILED${_NC}" > /dev/ttyS1
+	echo -e "${_RED}kexec-kdump-test: TEST FAILED${_NC}" > /dev/ttyS0
+
+	echo $@ > /dev/ttyS1
+	echo $@ > /dev/ttyS0
+
+	sync
+
+	shutdown -h 0
+
+	exit 1
+}
+
+test_abort() {
+	echo -e "${_YELLOW}TEST ABORTED${_NC}" > /dev/ttyS1
+	echo -e "${_YELLOW}kexec-kdump-test: TEST ABORTED${_NC}" > /dev/ttyS0
+
+	echo $@ > /dev/ttyS1
+	echo $@ > /dev/ttyS0
+
+	sync
+
+	shutdown -h 0
+
+	exit 2
+}
+
+has_valid_vmcore_dir() {
+	local path=$1
+	local vmcore_dir=$path/$(ls -1 $path | tail -n 1)
+	local vmcore="<invalid>"
+
+	# Checking with `crash` is slow and consume a lot of memory/disk,
+	# just do a sanity check by check if log are available.
+	if [ -e $vmcore_dir/vmcore ]; then
+		vmcore=$vmcore_dir/vmcore
+		makedumpfile --dump-dmesg $vmcore $vmcore_dir/vmcore-dmesg.txt.2 || return 1
+	elif [ -e $vmcore_dir/vmcore.flat ]; then
+		vmcore=$vmcore_dir/vmcore.flat
+		makedumpfile -R $vmcore_dir/vmcore < $vmcore || return 1
+		makedumpfile --dump-dmesg $vmcore_dir/vmcore $vmcore_dir/vmcore-dmesg.txt.2 || return
1
+		rm $vmcore_dir/vmcore
+	else
+		return 1
+	fi
+
+	if diff $vmcore_dir/vmcore-dmesg.txt.2 $vmcore_dir/vmcore-dmesg.txt; then
+		return 1
+	fi
+
+	test_output "Found a valid vmcore in \"$vmcore_dir\""
+
+	return 0
+}
+
+BOOT_COUNT=$(get_test_boot_count)
+test_output "Kexec-Kdump-Test Boot #$BOOT_COUNT"
+
+echo 'fedora' | passwd --stdin root
+
+test_output "Updating kernel cmdline"
+grubby --update-kernel ALL --args $BOOT_ARG=$(expr $BOOT_COUNT + 1) && sync
+
+test_output "Executing test hook"
+source /kexec-kdump-test/test.sh
+
+on_test;
+
+test_output "Test exited, system hang for inspect"
diff --git a/tests/scripts/kexec-kdump-test/kexec-kdump-test.service
b/tests/scripts/kexec-kdump-test/kexec-kdump-test.service
new file mode 100644
index 0000000..ba7b11e
--- /dev/null
+++ b/tests/scripts/kexec-kdump-test/kexec-kdump-test.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Kexec Kdump Test Service
+
+[Service]
+ExecStart=/kexec-kdump-test/init.sh
+Type=idle
+
+[Install]
+WantedBy=multi-user.target
diff --git a/tests/scripts/kexec-kdump-test/test.sh
b/tests/scripts/kexec-kdump-test/test.sh
new file mode 100755
index 0000000..66ac15d
--- /dev/null
+++ b/tests/scripts/kexec-kdump-test/test.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+# A test example that do nothing
+
+# Executed before VM starts
+on_build() {
+	:
+}
+
+# Executed when VM boots
+on_test() {
+	:
+	# call get_test_boot_count to get boot cound
+	# call test_passed if test passed
+	# call test_failed if test passed
+}
diff --git a/tests/scripts/run-test.sh b/tests/scripts/run-test.sh
new file mode 100755
index 0000000..cdba7f4
--- /dev/null
+++ b/tests/scripts/run-test.sh
@@ -0,0 +1,131 @@
+#!/bin/bash
+
+_kill_all_jobs() {
+	local _jobs=$(jobs -r -p)
+
+	[ -n "$_jobs" ] && kill $_jobs
+}
+
+trap '
+ret=$?;
+_kill_all_jobs
+exit $ret;
+' EXIT
+
+trap 'exit 1;' SIGINT
+
+BASEDIR=$(realpath $(dirname "$0"))
+. $BASEDIR/test-lib.sh
+TESTCASEDIR="$BASEDIR/testcases"
+
+console=0
+testcases=""
+
+while [ $# -gt 0 ]; do
+	case $1 in
+		'')
+			break
+			;;
+		--console )
+			console=1
+			;;
+		-*)
+			echo "Invalid option $1"
+			;;
+		*)
+			testcases+=" $1"
+			;;
+	esac
+	shift;
+done
+
+if [ -z "$testcases" ]; then
+	echo "==== Starting all tests: ===="
+	testcases=$(ls -1 $TESTCASEDIR)
+else
+	echo "==== Starting specified tests: ===="
+fi
+echo ${testcases##*/}
+echo
+
+declare -A results
+ret=0
+
+for test_case in $testcases; do
+	echo "======== Running Test Case $test_case ========"
+	results[$test_case]="<Test Skipped>"
+
+	testdir=$TESTCASEDIR/$test_case
+	script_num=$(ls -1 $testdir | wc -l)
+	scripts=$(ls -r -1 $testdir | tr '\n' ' ')
+	test_outputs=""
+	read main_script aux_script <<< "$scripts"
+
+	if [ -z "$main_script" ]; then
+		echo "ERROR: Empty testcase dir $testdir"
+		continue
+	fi
+
+	for script in $scripts; do
+		echo "---- Building image for: $script ----"
+		echo "-------- Output image is: $(get_test_image $testdir/$script)"
+		echo "-------- Building log is: $(get_test_image $testdir/$script).log"
+
+		mkdir -p $(dirname $(get_test_image $testdir/$script))
+		build_test_image $testdir/$script &> $(get_test_image $testdir/$script).log
+
+		if [ $? -ne 0 ]; then
+			echo "Failing building image!"
+			continue 2
+		fi
+	done
+
+	for script in $aux_script; do
+		echo "---- Starting VM: $script ----"
+
+		script="$testdir/$script"
+		echo "-------- Qemu cmdline: $(get_test_qemu_cmd_file $script)"
+		echo "-------- Console log: $(get_test_console_file $script)"
+		echo "-------- Test log: $(get_test_output_file $script)"
+		test_outputs+="$(get_test_output_file $script) "
+		rm -f $(get_test_console_file $script)
+		rm -f $(get_test_output_file $script)
+
+		$(run_test_sync $script > $(get_test_console_file $script)) &
+
+		sleep 3
+	done
+
+	script="$main_script"
+	echo "---- Starting test VM: $(basename $script) ----"
+	script="$testdir/$script"
+
+	echo "-------- Qemu cmdline: $(get_test_qemu_cmd_file $script)"
+	echo "-------- Console log: $(get_test_console_file $script)"
+	echo "-------- Test log: $(get_test_output_file $script)"
+	test_outputs+="$(get_test_output_file $script) "
+	rm -f $(get_test_console_file $script)
+	rm -f $(get_test_output_file $script)
+
+	if [ $console -eq 1 ]; then
+		run_test_sync $script | tee $(get_test_console_file $script)
+		[ -n "$(jobs -p)" ] && kill $(jobs -p)
+	else
+		$(run_test_sync $script > $(get_test_console_file $script)) &
+		watch_test_outputs $test_outputs
+	fi
+
+	res="$(gather_test_result $test_outputs)"
+	[ $? -ne 0 ] && ret=$(expr $ret + 1)
+	results[$test_case]="$res"
+
+	echo -e "-------- Test finished: $test_case $res --------"
+done
+
+echo "======== Test results ========"
+for i in ${!results[@]}; do
+	echo "----------------"
+	echo -e "$i:\t\t${results[$i]}"
+done
+
+exit $ret
diff --git a/tests/scripts/spawn-image-shell.sh b/tests/scripts/spawn-image-shell.sh
new file mode 100755
index 0000000..ccfb655
--- /dev/null
+++ b/tests/scripts/spawn-image-shell.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+BASEDIR=$(realpath $(dirname "$0"))
+. $BASEDIR/image-init-lib.sh
+
+# Base image to build from
+BOOT_IMAGE=$1
+if [[ ! -e $BOOT_IMAGE ]]; then
+	perror_exit "Image '$BOOT_IMAGE' not found"
+else
+	BOOT_IMAGE=$(realpath "$BOOT_IMAGE")
+fi
+
+mount_image $BOOT_IMAGE
+
+shell_in_image $BOOT_IMAGE
diff --git a/tests/scripts/test-lib.sh b/tests/scripts/test-lib.sh
new file mode 100644
index 0000000..aff5cd3
--- /dev/null
+++ b/tests/scripts/test-lib.sh
@@ -0,0 +1,177 @@
+#!/usr/bin/env sh
+[ -z "$BASEDIR" ] && BASEDIR=$(realpath $(dirname "$0"))
+[ -z "$TESTDIR" ] && TESTDIR=$(realpath $BASEDIR/../)
+[ -z "$TEST_BASE_IMAGE" ] &&
TEST_BASE_IMAGE=$TESTDIR/output/test-base-image
+
+[[ ! -e $TEST_BASE_IMAGE ]] && echo "Test base image not found."
&& exit 1
+
+DEFAULT_QEMU_CMD="-nodefaults \
+-nographic \
+-smp 2 \
+-m 768M \
+-monitor none"
+
+_YELLOW='\033[1;33m'
+_GREEN='\033[0;32m'
+_RED='\033[0;31m'
+_NC='\033[0m' # No Color
+
+get_test_path() {
+	local script=$1
+	local testname=$(basename $(dirname $script))
+	local output=$TESTDIR/output/$testname
+
+	echo $output
+}
+
+get_test_entry_name() {
+	echo $(basename ${1%.*})
+}
+
+get_test_image() {
+	local script=$1
+	local testout=$(get_test_path $script)
+	local entry=$(get_test_entry_name $script)
+
+	echo $testout/$entry.img
+}
+
+get_test_qemu_cmd_file() {
+	local script=$1
+	local testout=$(get_test_path $script)
+	local entry=$(get_test_entry_name $script)
+
+	echo $testout/$entry.qemu_cmd
+}
+
+get_test_qemu_cmd() {
+	cat $(get_test_qemu_cmd_file $1)
+}
+
+get_test_output_file() {
+	local script=$1
+	local testout=$(get_test_path $script)
+	local entry=$(get_test_entry_name $script)
+
+	echo $testout/$entry.output
+}
+
+get_test_console_file() {
+	local script=$1
+	local testout=$(get_test_path $script)
+	local entry=$(get_test_entry_name $script)
+
+	echo $testout/$entry.console
+}
+
+get_test_output() {
+	local output=$(get_test_output_file $1)
+	if [ -e "$output" ]; then
+		cat $(get_test_output_file $1)
+	else
+		echo "<No Output>"
+	fi
+}
+
+build_test_image() {
+	local script=$1
+	local test_image=$(get_test_image $script)
+	mkdir -p $(dirname $test_image)
+
+	$BASEDIR/build-image.sh \
+		$TEST_BASE_IMAGE \
+		$test_image \
+		$BASEDIR/build-scripts/test-image.sh \
+		$script
+}
+
+run_test_sync() {
+	local qemu_cmd=$(get_test_qemu_cmd $1)
+
+	if [ -n "$qemu_cmd" ]; then
+		timeout --foreground 10m qemu-kvm $(get_test_qemu_cmd $1)
+	else
+		echo "error: test qemu command line is not configured" > /dev/stderr
+		return 1
+	fi
+}
+
+_check_test_result() {
+	grep "TEST PASSED" $1 2>/dev/null
+	[ $? -eq 0 ] && return 0
+
+	grep "TEST FAILED" $1 2>/dev/null
+	[ $? -eq 0 ] && return 1
+
+	grep "TEST ABORTED" $1 2>/dev/null
+	[ $? -eq 0 ] && return 2
+
+	return 255
+}
+
+# Print test result and return below value:
+# 0: Test passed
+# 1: Test failed
+# 2: Test aborted, test scripts errored out
+# 3: Test exited unexpectely, VM got killed early, or time out
+gather_test_result() {
+	local ret=255
+	local res=""
+
+	for i in $@; do
+		res=$(_check_test_result $i)
+		ret=$?
+
+		if [ $ret -ne 255 ]; then
+			echo $res
+			return $ret
+		fi
+	done
+
+	echo "${_RED}TEST RESULT NOT FOUND!${_NC}"
+	return 3
+}
+
+# Wait and watch for test result
+watch_test_outputs() {
+	local ret=255
+	local res=""
+	# If VMs are still running, check for test result, if
+	# test finished, kill remaining VMs
+	while true; do
+		if [ -n "$(jobs -r)" ]; then
+			# VMs still running
+			for i in $@; do
+				res=$(_check_test_result $i)
+				ret=$?
+
+				if [ $ret -ne 255 ]; then
+					# Test finished, kill VMs
+					kill $(jobs -p)
+					break 2
+				fi
+			done
+		else
+			# VMs exited
+			ret=255
+
+			for i in $@; do
+				res=$(_check_test_result $i)
+				ret=$?
+
+				if [ $ret -ne 255 ]; then
+					break 2
+				fi
+			done
+
+			if [ $ret -eq 255 ]; then
+				ret=3
+				break
+			fi
+		fi
+
+		sleep 1
+	done
+
+	return $ret
+}
diff --git a/tests/scripts/testcases/local-kdump/0-local.sh
b/tests/scripts/testcases/local-kdump/0-local.sh
new file mode 100755
index 0000000..d09c6f0
--- /dev/null
+++ b/tests/scripts/testcases/local-kdump/0-local.sh
@@ -0,0 +1,32 @@
+on_build() {
+	:
+}
+
+on_test() {
+	local boot_count=$(get_test_boot_count)
+
+	if [ $boot_count -eq 1 ]; then
+		cat << EOF > /etc/kdump.conf
+path /var/crash
+core_collector makedumpfile -l --message-level 1 -d 31
+EOF
+		kdumpctl start || test_failed "Failed to start kdump"
+
+		sync
+
+		echo 1 > /proc/sys/kernel/sysrq
+		echo c > /proc/sysrq-trigger
+
+	elif [ $boot_count -eq 2 ]; then
+
+		if has_valid_vmcore_dir /var/crash; then
+			test_passed
+		else
+			test_failed
+		fi
+
+		shutdown -h 0
+	else
+		test_failed "Unexpected reboot"
+	fi
+}
diff --git a/tests/scripts/testcases/nfs-kdump/0-server.sh
b/tests/scripts/testcases/nfs-kdump/0-server.sh
new file mode 100755
index 0000000..04f102e
--- /dev/null
+++ b/tests/scripts/testcases/nfs-kdump/0-server.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+# Executed before VM starts
+on_build() {
+	img_inst_pkg "nfs-utils dnsmasq"
+
+	img_run_cmd "mkdir -p /srv/nfs/var/crash"
+	img_run_cmd "echo /srv/nfs 192.168.77.1/24\(rw,async,insecure,no_root_squash\) >
/etc/exports"
+	img_run_cmd "systemctl enable nfs-server"
+
+	img_run_cmd "echo interface=eth0 > /etc/dnsmasq.conf"
+	img_run_cmd "echo dhcp-authoritative >> /etc/dnsmasq.conf"
+	img_run_cmd "echo dhcp-range=192.168.77.50,192.168.77.100,255.255.255.0,12h
>> /etc/dnsmasq.conf"
+	img_run_cmd "systemctl enable dnsmasq"
+
+	img_run_cmd 'echo DEVICE="eth0" >
/etc/sysconfig/network-scripts/ifcfg-eth0'
+	img_run_cmd 'echo BOOTPROTO="none >>
/etc/sysconfig/network-scripts/ifcfg-eth0"'
+	img_run_cmd 'echo ONBOOT="yes" >>
/etc/sysconfig/network-scripts/ifcfg-eth0'
+	img_run_cmd 'echo PREFIX="24" >>
/etc/sysconfig/network-scripts/ifcfg-eth0'
+	img_run_cmd 'echo IPADDR="192.168.77.1" >>
/etc/sysconfig/network-scripts/ifcfg-eth0'
+	img_run_cmd 'echo TYPE="Ethernet" >>
/etc/sysconfig/network-scripts/ifcfg-eth0'
+
+	img_add_qemu_cmd "-nic socket,listen=:8010,mac=52:54:00:12:34:56"
+}
+
+# Executed when VM boots
+on_test() {
+	while true; do
+		if has_valid_vmcore_dir /srv/nfs/var/crash; then
+			# Wait a few seconds so client finish it's work to generate a full log
+			sleep 5
+
+			test_passed
+		fi
+
+		sleep 1
+	done
+}
diff --git a/tests/scripts/testcases/nfs-kdump/1-client.sh
b/tests/scripts/testcases/nfs-kdump/1-client.sh
new file mode 100755
index 0000000..df62463
--- /dev/null
+++ b/tests/scripts/testcases/nfs-kdump/1-client.sh
@@ -0,0 +1,30 @@
+# Executed before VM starts
+on_build() {
+	img_inst_pkg "nfs-utils"
+	img_add_qemu_cmd "-nic socket,connect=127.0.0.1:8010,mac=52:54:00:12:34:57"
+}
+
+on_test() {
+	local boot_count=$(get_test_boot_count)
+	local nfs_server=192.168.77.1
+
+	if [ "$boot_count" -eq 1 ]; then
+		cat << EOF > /etc/kdump.conf
+nfs $nfs_server:/srv/nfs
+core_collector makedumpfile -l --message-level 1 -d 31
+EOF
+
+		while ! ping -c 1 $nfs_server -W 1; do
+			:
+		done
+
+		kdumpctl start || test_failed "Failed to start kdump"
+
+		sync
+
+		echo 1 > /proc/sys/kernel/sysrq
+		echo c > /proc/sysrq-trigger
+	else
+		shutdown -h 0
+	fi
+}
diff --git a/tests/scripts/testcases/ssh-kdump/0-server.sh
b/tests/scripts/testcases/ssh-kdump/0-server.sh
new file mode 100755
index 0000000..f1e5073
--- /dev/null
+++ b/tests/scripts/testcases/ssh-kdump/0-server.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env sh
+
+# Executed before VM starts
+on_build() {
+	img_add_qemu_cmd "-nic socket,listen=:8010,mac=52:54:00:12:34:56"
+
+	img_run_cmd "echo root:fedora | chpasswd"
+	img_run_cmd 'sed -i "s/^.*PasswordAuthentication .*\$/PasswordAuthentication
yes/"  /etc/ssh/sshd_config'
+	img_run_cmd 'sed -i "s/^.*PermitRootLogin .*\$/PermitRootLogin yes/" 
/etc/ssh/sshd_config'
+	img_run_cmd "systemctl enable sshd"
+
+	img_run_cmd "echo interface=eth0 > /etc/dnsmasq.conf"
+	img_run_cmd "echo dhcp-authoritative >> /etc/dnsmasq.conf"
+	img_run_cmd "echo dhcp-range=192.168.77.50,192.168.77.100,255.255.255.0,12h
>> /etc/dnsmasq.conf"
+	img_run_cmd "systemctl enable dnsmasq"
+
+	img_run_cmd 'echo DEVICE="eth0" >
/etc/sysconfig/network-scripts/ifcfg-eth0'
+	img_run_cmd 'echo BOOTPROTO="none >>
/etc/sysconfig/network-scripts/ifcfg-eth0"'
+	img_run_cmd 'echo ONBOOT="yes" >>
/etc/sysconfig/network-scripts/ifcfg-eth0'
+	img_run_cmd 'echo PREFIX="24" >>
/etc/sysconfig/network-scripts/ifcfg-eth0'
+	img_run_cmd 'echo IPADDR="192.168.77.1" >>
/etc/sysconfig/network-scripts/ifcfg-eth0'
+	img_run_cmd 'echo TYPE="Ethernet" >>
/etc/sysconfig/network-scripts/ifcfg-eth0'
+}
+
+# Executed when VM boots
+on_test() {
+	while true; do
+		if has_valid_vmcore_dir /var/crash; then
+			test_passed
+		fi
+
+		sleep 1
+	done
+}
diff --git a/tests/scripts/testcases/ssh-kdump/1-client.sh
b/tests/scripts/testcases/ssh-kdump/1-client.sh
new file mode 100755
index 0000000..d79f00f
--- /dev/null
+++ b/tests/scripts/testcases/ssh-kdump/1-client.sh
@@ -0,0 +1,40 @@
+# Executed before VM starts
+on_build() {
+	img_inst_pkg "sshpass"
+	img_add_qemu_cmd "-nic socket,connect=127.0.0.1:8010,mac=52:54:00:12:34:57"
+}
+
+on_test() {
+	local boot_count=$(get_test_boot_count)
+	local ssh_server=192.168.77.1
+
+	if [ "$boot_count" -eq 1 ]; then
+cat << EOF > /etc/kdump.conf
+ssh root(a)192.168.77.1
+core_collector makedumpfile -l --message-level 1 -d 31 -F
+EOF
+
+		ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa <<< y
&>/dev/ttyS1
+
+		while ! ping -c 1 $ssh_server -W 1; do
+			sleep 1
+		done
+
+		while [ -z "$(cat /root/.ssh/known_hosts)" ]; do
+			ssh-keyscan -H 192.168.77.1 > /root/.ssh/known_hosts
+		done
+
+		sshpass -p fedora ssh-copy-id root@$ssh_server -f &>/dev/ttyS1
+
+		sshpass -p fedora kdumpctl propagate &>/dev/ttyS1
+
+		kdumpctl start || test_failed "Failed to start kdump"
+
+		sync
+
+		echo 1 > /proc/sys/kernel/sysrq
+		echo c > /proc/sysrq-trigger
+	else
+		shutdown -h 0
+	fi
+}
-- 
2.26.2