[PATCH] testsuite: improved stability and speed trac#633 trac#639 trac#644

[Jiri Moskovcak]

- use preinstalled images instead of installing the system every time
  - avoid problems with broken mirrors and Anaconda
  - speeds the testing
- don't use cron job to run the tests
  - causes SELinux problems
- use bridged networking instead of NAT
  - makes it possible to ssh to the VM running the test
---
 tests/helpers/bin/run                 |  106 ++++++++++++---------------------
 tests/helpers/common/fun.sh           |   19 ++++--
 tests/helpers/targets/el6/defs.sh     |    2 +
 tests/helpers/targets/el6/runtests.sh |   35 +++++++++++
 tests/helpers/targets/f16/defs.sh     |    2 +
 tests/helpers/targets/f16/runtests.sh |   35 +++++++++++
 tests/helpers/targets/f17/defs.sh     |    2 +
 tests/helpers/targets/f17/runtests.sh |   35 +++++++++++
 8 files changed, 163 insertions(+), 73 deletions(-)
 create mode 100644 tests/helpers/targets/el6/runtests.sh
 create mode 100644 tests/helpers/targets/f16/runtests.sh
 create mode 100644 tests/helpers/targets/f17/runtests.sh

diff --git a/tests/helpers/bin/run b/tests/helpers/bin/run
index fe0815b..790fee3 100755
--- a/tests/helpers/bin/run
+++ b/tests/helpers/bin/run
@@ -3,6 +3,7 @@
 source ~/common/globaldefs.sh
 source ~/common/fun.sh
 source ~/targets/${1}/defs.sh
+CURR_TARGET=$1
 
 shift
 PREPARE=
@@ -31,74 +32,46 @@ if [ ! -d data ]; then
 fi
 
 reinit_vm
-echo 'Getting kickstart'
-get_ks
-
-cp orig-ks.cfg custom-ks.cfg
-rm -f orig-ks.cfg
-
-echo 'Patching kickstart'
-
-cat ~/common/start_post_hunk >> custom-ks.cfg
-cat ~/common/git_hunk >> custom-ks.cfg
-
-# don't run tests automatically if --prepare is set
-test -z $PREPARE && cat ~/common/cron_hunk >> custom-ks.cfg
-
-# ssh key
-cat ~/common/ssh_pre_hunk >> custom-ks.cfg
-embed_file "/root/.ssh/id_rsa" ~/.ssh/id_rsa
-cat ~/common/ssh_post_hunk >> custom-ks.cfg
-
-# procmail cfg
-cat ~/common/procmail_hunk >> custom-ks.cfg
-
-# custom cfg
-embed_file "/root/abrt/tests/runtests/aux/config.sh.local" ./config.sh.local
-
-# fedorahosted alias
-embed_file "/etc/hosts" ~/common/hosts_hunk
-embed_file "/etc/motd" ~/common/motd_hunk
-
-add_rhel_repos
-add_abrt_repos
-
-# exclude build tests
-cat ~/common/exclude_build_tests >> custom-ks.cfg
-
-# disable gpg check
-cat ~/common/no_gpg_check >> custom-ks.cfg
-
-# %post
-cat ~/common/end_post_hunk >> custom-ks.cfg
-
-echo 'Running virt-install'
-virt-install --name $VM_NAME --ram "1300" \
-  --connect qemu:///system \
-  --location "$LOC" \
-  --disk path=$DISK,cache=none \
-  --initrd-inject=./custom-ks.cfg --extra-args "ks=file:/custom-ks.cfg
${INIT_EXTRA_ARGS}" \
-  --noautoconsole \
-  --os-type=linux \
-  --os-variant=$OS_VARIANT \
-  --graphics type=vnc \
-  --quiet
-
-set +x
-echo 'virt-install done'
-echo -n "Zzzz (until VM is being installed): "
-while virsh --connect qemu:///system list | grep -q $VM_NAME; do
-    echo -n '.'
-    sleep 1m
-done
-echo 'x'
-
-sleep 30
 
 virsh --connect qemu:///system start $VM_NAME
-cat /var/lib/libvirt/dnsmasq/default.leases
+if [ $? != 0 ]; then
+   # how about we create it here instead of dying??
+   echo "VM with name $VM_NAME does not exist!"
+   exit 1;
+fi
 
-test -z $PREPARE || exit 0
+echo "Wait a while for machine to start, before we start ping attact..."
+sleep 20;
+
+try=0;
+while true; do
+  ping -w 1 $IP &> /dev/null;
+  res=$?;
+  if [ $res == 0 ]; then
+    break;
+  fi;
+  
+  if [ $try == 10 ]; then # max tries
+     echo "Seems like machine '$VM_NAME' doesn't want to talk with us on
'$IP'"
+     exit 1;
+  fi
+  let try=try+1;
+  sleep 5; # wait between the attempts, otherwise it floods the log
+done;
+
+echo "Giving the sshd a while to start"
+sleep 10
+scp ~/targets/$CURR_TARGET/runtests.sh root@$IP:~
+if [ $? != 0 ]; then
+  echo "Can't upload runtests.sh to VM";
+  exit 1;
+fi;
+
+ssh root@$IP "sh ./runtests.sh"
+if [ $? != 0 ]; then
+  echo "Running the tests failed, bailing out..";
+  exit 1;
+fi;
 
 echo -n "Zzzz (until VM is executing the tests): "
 while virsh --connect qemu:///system list | grep -q $VM_NAME; do
@@ -107,9 +80,6 @@ while virsh --connect qemu:///system list | grep -q $VM_NAME; do
 done
 echo 'x'
 
-sleep 30
-set -x
-
 handle_results "$DATA_DIR/data/" "/var/nightly_results/$( basename
$DATA_DIR/)"
 echo $result_path
 test -d $result_path || exit 1
diff --git a/tests/helpers/common/fun.sh b/tests/helpers/common/fun.sh
index e948160..5517bf3 100644
--- a/tests/helpers/common/fun.sh
+++ b/tests/helpers/common/fun.sh
@@ -9,14 +9,23 @@ function reinit_vm() {
     if virsh --connect qemu:///system list | grep -q $VM_NAME;  then
         echo 'Destroying virtual machine'
         virsh --connect qemu:///system destroy $VM_NAME
-        sleep 3
     fi
 
-    if virsh --connect qemu:///system list --all | grep -q $VM_NAME;  then
-        echo 'Undef virtual machine'
-        virsh --connect qemu:///system undefine $VM_NAME
-        sleep 1
+#    if virsh --connect qemu:///system list --all | grep -q $VM_NAME;  then
+#        echo 'Undef virtual machine'
+#        virsh --connect qemu:///system undefine $VM_NAME
+
+    echo "Restoring vm ${VM_NAME}"
+    # beware sudo!!
+    sudo dd if=/pub/VM/${VM_NAME}.img of=/dev/mapper/vg-${CURR_TARGET}_vm bs=4M
+    if [ $? -ne 0 ]; then
+        echo "Restoring VM failed";
+        exit 1;
+    else
+        echo "VM restored";
     fi
+
+#    fi
 }
 
 function get_ks() {
diff --git a/tests/helpers/targets/el6/defs.sh b/tests/helpers/targets/el6/defs.sh
index 57db622..5f001de 100644
--- a/tests/helpers/targets/el6/defs.sh
+++ b/tests/helpers/targets/el6/defs.sh
@@ -8,3 +8,5 @@ KS_NAME_PREFIX='brq_el6'

LOC='http://download.englab.brq.redhat.com/pub/rhel/rel-eng/latest-RH...
 DISK=$( echo /dev/mapper/*el6_vm )
 REPOS_REQUIRED='el6_latest'
+MAC="52540010b4df"
+IP="10.34.25.207"
\ No newline at end of file
diff --git a/tests/helpers/targets/el6/runtests.sh
b/tests/helpers/targets/el6/runtests.sh
new file mode 100644
index 0000000..3f365e6
--- /dev/null
+++ b/tests/helpers/targets/el6/runtests.sh
@@ -0,0 +1,35 @@
+#! /bin/bash
+
+pushd $PWD && {
+    cd /root && {
+        git clone git://abrt.brq.redhat.com/abrt.git ./abrt && {
+            cd ./abrt && {
+                case $(cut -d: -f4,5 /etc/system-release-cpe) in
+                    enterprise_linux:6*)
+                        git checkout rhel6
+                        ;;
+                    *)
+                        ;;
+                esac
+            }
+        }
+    }
+    popd
+}
+
+cat > /root/abrt/tests/runtests/aux/config.sh.local << "_END_"
+export FORMAT_SCRIPT='aux/scp_format.sh'
+export REPORT_SCRIPT='aux/scp_report.sh'
+
+export SCPTO='[e-mail]:~/targets/el6/data/'
+
+export SHUTDOWN=1
+_END_
+
+
+EXCLUDE_TESTS="abrt-nightly-build|abrt-make-check|btparser-make-check|libreport-make-check"
+TEST_ORDER="/root/abrt/tests/runtests/aux/test_order"
+
+egrep -v "$EXCLUDE_TESTS" "$TEST_ORDER" > /tmp/test_order
&& mv /tmp/test_order "$TEST_ORDER"
+sed -i 's/OpenGPGCheck.*=.*yes/OpenGPGCheck = no/'
/etc/abrt/abrt-action-save-package-data.conf
+screen -d -m /root/abrt/tests/runtests/run | tee -a /tmp/abrt-nightly-test.log
\ No newline at end of file
diff --git a/tests/helpers/targets/f16/defs.sh b/tests/helpers/targets/f16/defs.sh
index 42f44fd..95d4ffd 100644
--- a/tests/helpers/targets/f16/defs.sh
+++ b/tests/helpers/targets/f16/defs.sh
@@ -7,6 +7,8 @@ OS_VARIANT='fedora16'
 KS_NAME_PREFIX='fedora_16'

LOC='http://download.englab.brq.redhat.com/pub/fedora/linux/releases/...
 DISK=$( echo /dev/mapper/*f16_vm )
+MAC="525400846b5b"
+IP="10.34.25.210"
 
 # http://fedoraproject.org/wiki/Anaconda/Updates

INIT_EXTRA_ARGS="updates=http://abrt.brq.redhat.com/media/updates-an...
diff --git a/tests/helpers/targets/f16/runtests.sh
b/tests/helpers/targets/f16/runtests.sh
new file mode 100644
index 0000000..6929f4f
--- /dev/null
+++ b/tests/helpers/targets/f16/runtests.sh
@@ -0,0 +1,35 @@
+#! /bin/bash
+
+pushd $PWD && {
+    cd /root && {
+        git clone git://abrt.brq.redhat.com/abrt.git ./abrt && {
+            cd ./abrt && {
+                case $(cut -d: -f4,5 /etc/system-release-cpe) in
+                    enterprise_linux:6*)
+                        git checkout rhel6
+                        ;;
+                    *)
+                        ;;
+                esac
+            }
+        }
+    }
+    popd
+}
+
+cat > /root/abrt/tests/runtests/aux/config.sh.local << "_END_"
+export FORMAT_SCRIPT='aux/scp_format.sh'
+export REPORT_SCRIPT='aux/scp_report.sh'
+
+export SCPTO='[e-mail]:~/targets/f16/data/'
+
+export SHUTDOWN=1
+_END_
+
+
+EXCLUDE_TESTS="abrt-nightly-build|abrt-make-check|btparser-make-check|libreport-make-check"
+TEST_ORDER="/root/abrt/tests/runtests/aux/test_order"
+
+egrep -v "$EXCLUDE_TESTS" "$TEST_ORDER" > /tmp/test_order
&& mv /tmp/test_order "$TEST_ORDER"
+
+screen -d -m /root/abrt/tests/runtests/run | tee -a /tmp/abrt-nightly-test.log
\ No newline at end of file
diff --git a/tests/helpers/targets/f17/defs.sh b/tests/helpers/targets/f17/defs.sh
index 7dcdd7c..57306c2 100644
--- a/tests/helpers/targets/f17/defs.sh
+++ b/tests/helpers/targets/f17/defs.sh
@@ -6,6 +6,8 @@ REPO='git://abrt.brq.redhat.com/abrt.git'
 OS_VARIANT='fedora16'
 KS_NAME_PREFIX='fedora_17'

LOC='http://download.englab.brq.redhat.com/pub/fedora/linux/releases/...
+MAC="525400405da9"
+IP="10.34.25.200"
 
 DISK=$( echo /dev/mapper/*f17_vm )
 
diff --git a/tests/helpers/targets/f17/runtests.sh
b/tests/helpers/targets/f17/runtests.sh
new file mode 100644
index 0000000..4d43fde
--- /dev/null
+++ b/tests/helpers/targets/f17/runtests.sh
@@ -0,0 +1,35 @@
+#! /bin/bash
+
+pushd $PWD && {
+    cd /root && {
+        git clone git://abrt.brq.redhat.com/abrt.git ./abrt && {
+            cd ./abrt && {
+                case $(cut -d: -f4,5 /etc/system-release-cpe) in
+                    enterprise_linux:6*)
+                        git checkout rhel6
+                        ;;
+                    *)
+                        ;;
+                esac
+            }
+        }
+    }
+    popd
+}
+
+cat > /root/abrt/tests/runtests/aux/config.sh.local << "_END_"
+export FORMAT_SCRIPT='aux/scp_format.sh'
+export REPORT_SCRIPT='aux/scp_report.sh'
+
+export SCPTO='[e-mail]:~/targets/f17/data/'
+
+export SHUTDOWN=1
+_END_
+
+
+EXCLUDE_TESTS="abrt-nightly-build|abrt-make-check|btparser-make-check|libreport-make-check"
+TEST_ORDER="/root/abrt/tests/runtests/aux/test_order"
+
+egrep -v "$EXCLUDE_TESTS" "$TEST_ORDER" > /tmp/test_order
&& mv /tmp/test_order "$TEST_ORDER"
+
+screen -d -m /root/abrt/tests/runtests/run | tee -a /tmp/abrt-nightly-test.log
\ No newline at end of file
-- 
1.7.10.4

[Richard Marko]

On 07/24/2012 01:47 PM, Jiri Moskovcak wrote:
>  - use preinstalled images instead of installing the system every
> time
>    - avoid problems with broken mirrors and Anaconda
>    - speeds the testing
>  - don't use cron job to run the tests
>    - causes SELinux problems
>  - use bridged networking instead of NAT
>    - makes it possible to ssh to the VM running the test
>  ---
>   tests/helpers/bin/run                 |  106 ++++++++++++---------------------
>   tests/helpers/common/fun.sh           |   19 ++++--
>   tests/helpers/targets/el6/defs.sh     |    2 +
>   tests/helpers/targets/el6/runtests.sh |   35 +++++++++++
>   tests/helpers/targets/f16/defs.sh     |    2 +
>   tests/helpers/targets/f16/runtests.sh |   35 +++++++++++
>   tests/helpers/targets/f17/defs.sh     |    2 +
>   tests/helpers/targets/f17/runtests.sh |   35 +++++++++++
>   8 files changed, 163 insertions(+), 73 deletions(-)
>   create mode 100644 tests/helpers/targets/el6/runtests.sh
>   create mode 100644 tests/helpers/targets/f16/runtests.sh
>   create mode 100644 tests/helpers/targets/f17/runtests.sh
>
>  diff --git a/tests/helpers/bin/run b/tests/helpers/bin/run
>  index fe0815b..790fee3 100755
>  --- a/tests/helpers/bin/run
>  +++ b/tests/helpers/bin/run
>  @@ -3,6 +3,7 @@
>   source ~/common/globaldefs.sh
>   source ~/common/fun.sh
>   source ~/targets/${1}/defs.sh
>  +CURR_TARGET=$1
>   
>   shift
>   PREPARE=
>  @@ -31,74 +32,46 @@ if [ ! -d data ]; then
>   fi
>   
>   reinit_vm
>  -echo 'Getting kickstart'
>  -get_ks
>  -
>  -cp orig-ks.cfg custom-ks.cfg
>  -rm -f orig-ks.cfg
>  -
>  -echo 'Patching kickstart'
>  -
>  -cat ~/common/start_post_hunk >> custom-ks.cfg
>  -cat ~/common/git_hunk >> custom-ks.cfg
>  -
>  -# don't run tests automatically if --prepare is set
>  -test -z $PREPARE && cat ~/common/cron_hunk >> custom-ks.cfg
>  -
>  -# ssh key
>  -cat ~/common/ssh_pre_hunk >> custom-ks.cfg
>  -embed_file "/root/.ssh/id_rsa" ~/.ssh/id_rsa
>  -cat ~/common/ssh_post_hunk >> custom-ks.cfg
>  -
>  -# procmail cfg
>  -cat ~/common/procmail_hunk >> custom-ks.cfg
>  -
>  -# custom cfg
>  -embed_file "/root/abrt/tests/runtests/aux/config.sh.local" ./config.sh.local
>  -
>  -# fedorahosted alias
>  -embed_file "/etc/hosts" ~/common/hosts_hunk
>  -embed_file "/etc/motd" ~/common/motd_hunk
>  -
>  -add_rhel_repos
>  -add_abrt_repos
>  -
>  -# exclude build tests
>  -cat ~/common/exclude_build_tests >> custom-ks.cfg
>  -
>  -# disable gpg check
>  -cat ~/common/no_gpg_check >> custom-ks.cfg
>  -
>  -# %post
>  -cat ~/common/end_post_hunk >> custom-ks.cfg
>  -
>  -echo 'Running virt-install'
>  -virt-install --name $VM_NAME --ram "1300" \
>  -  --connect qemu:///system \
>  -  --location "$LOC" \
>  -  --disk path=$DISK,cache=none \
>  -  --initrd-inject=./custom-ks.cfg --extra-args "ks=file:/custom-ks.cfg
> ${INIT_EXTRA_ARGS}" \
>  -  --noautoconsole \
>  -  --os-type=linux \
>  -  --os-variant=$OS_VARIANT \
>  -  --graphics type=vnc \
>  -  --quiet
>  -
>  -set +x
>  -echo 'virt-install done'
>  -echo -n "Zzzz (until VM is being installed): "
>  -while virsh --connect qemu:///system list | grep -q $VM_NAME; do
>  -    echo -n '.'
>  -    sleep 1m
>  -done
>  -echo 'x'
>  -
>  -sleep 30
>   
>   virsh --connect qemu:///system start $VM_NAME
>  -cat /var/lib/libvirt/dnsmasq/default.leases
>  +if [ $? != 0 ]; then
>  +   # how about we create it here instead of dying?? 
The code you've removed
re-creates the VM from scratch so why remove it
completely?
Would be better to have it preserved so when you need clean machine you
can run this script with some parameter or just remove the VM from libvirt.

We can do this during night (or weekend) and if all the tests pass we
can store this new machine as
a snapshot and use it during the day when we need fast execution.
>  +   echo "VM with name $VM_NAME does not exist!"
>  +   exit 1;
>  +fi
>   
>  -test -z $PREPARE || exit 0
>  +echo "Wait a while for machine to start, before we start ping attact..."
>  +sleep 20; 
I would skip this wait completely. On my notebook f17 vm starts
completely in 5 seconds or so.
>  +
>  +try=0;
>  +while true; do
>  +  ping -w 1 $IP &> /dev/null;
>  +  res=$?;
>  +  if [ $res == 0 ]; then
>  +    break;
>  +  fi;
>  +  
>  +  if [ $try == 10 ]; then # max tries
>  +     echo "Seems like machine '$VM_NAME' doesn't want to talk with us
> on '$IP'"
>  +     exit 1;
>  +  fi
>  +  let try=try+1;
>  +  sleep 5; # wait between the attempts, otherwise it floods the log 
Enclose this
section with 'set +x' and 'set -x' so it doesn't echo to
log, then you can increase
max tries to 100 and try to ping each second.
>  +done;
>  +
>  +echo "Giving the sshd a while to start"
>  +sleep 10
>  +scp ~/targets/$CURR_TARGET/runtests.sh root@$IP:~
>  +if [ $? != 0 ]; then
>  +  echo "Can't upload runtests.sh to VM";
>  +  exit 1;
>  +fi;
>  + 
How about running

ssh root@$IP "yum -y update"

here?

This would update the machine and we can easily see which packages were
updated in the log file.
>  +ssh root@$IP "sh ./runtests.sh"
>  +if [ $? != 0 ]; then
>  +  echo "Running the tests failed, bailing out..";
>  +  exit 1;
>  +fi;
>   
>   echo -n "Zzzz (until VM is executing the tests): "
>   while virsh --connect qemu:///system list | grep -q $VM_NAME; do
>  @@ -107,9 +80,6 @@ while virsh --connect qemu:///system list | grep -q $VM_NAME; do
>   done
>   echo 'x'
>   
>  -sleep 30
>  -set -x
>  -
>   handle_results "$DATA_DIR/data/" "/var/nightly_results/$( basename
> $DATA_DIR/)"
>   echo $result_path
>   test -d $result_path || exit 1
>  diff --git a/tests/helpers/common/fun.sh b/tests/helpers/common/fun.sh
>  index e948160..5517bf3 100644
>  --- a/tests/helpers/common/fun.sh
>  +++ b/tests/helpers/common/fun.sh
>  @@ -9,14 +9,23 @@ function reinit_vm() {
>       if virsh --connect qemu:///system list | grep -q $VM_NAME;  then
>           echo 'Destroying virtual machine'
>           virsh --connect qemu:///system destroy $VM_NAME
>  -        sleep 3
>       fi
>   
>  -    if virsh --connect qemu:///system list --all | grep -q $VM_NAME;  then
>  -        echo 'Undef virtual machine'
>  -        virsh --connect qemu:///system undefine $VM_NAME
>  -        sleep 1
>  +#    if virsh --connect qemu:///system list --all | grep -q $VM_NAME;  then
>  +#        echo 'Undef virtual machine'
>  +#        virsh --connect qemu:///system undefine $VM_NAME
>  +
>  +    echo "Restoring vm ${VM_NAME}"
>  +    # beware sudo!!
>  +    sudo dd if=/pub/VM/${VM_NAME}.img of=/dev/mapper/vg-${CURR_TARGET}_vm bs=4M
>  +    if [ $? -ne 0 ]; then
>  +        echo "Restoring VM failed";
>  +        exit 1;
>  +    else
>  +        echo "VM restored";
>       fi
>  +
>  +#    fi
>   }
>   
>   function get_ks() {
>  diff --git a/tests/helpers/targets/el6/defs.sh b/tests/helpers/targets/el6/defs.sh
>  index 57db622..5f001de 100644
>  --- a/tests/helpers/targets/el6/defs.sh
>  +++ b/tests/helpers/targets/el6/defs.sh
>  @@ -8,3 +8,5 @@ KS_NAME_PREFIX='brq_el6'
>  
> LOC='http://download.englab.brq.redhat.com/pub/rhel/rel-eng/latest-RH...
>   DISK=$( echo /dev/mapper/*el6_vm )
>   REPOS_REQUIRED='el6_latest'
>  +MAC="52540010b4df"
>  +IP="10.34.25.207"
>  \ No newline at end of file
>  diff --git a/tests/helpers/targets/el6/runtests.sh
> b/tests/helpers/targets/el6/runtests.sh
>  new file mode 100644
>  index 0000000..3f365e6
>  --- /dev/null
>  +++ b/tests/helpers/targets/el6/runtests.sh
>  @@ -0,0 +1,35 @@
>  +#! /bin/bash
>  +
>  +pushd $PWD && {
>  +    cd /root && {
>  +        git clone git://abrt.brq.redhat.com/abrt.git ./abrt && {
>  +            cd ./abrt && {
>  +                case $(cut -d: -f4,5 /etc/system-release-cpe) in
>  +                    enterprise_linux:6*)
>  +                        git checkout rhel6
>  +                        ;;
>  +                    *)
>  +                        ;;
>  +                esac
>  +            }
>  +        }
>  +    }
>  +    popd
>  +}
>  +
>  +cat > /root/abrt/tests/runtests/aux/config.sh.local << "_END_"
>  +export FORMAT_SCRIPT='aux/scp_format.sh'
>  +export REPORT_SCRIPT='aux/scp_report.sh'
>  +
>  +export SCPTO='[e-mail]:~/targets/el6/data/'
>  +
>  +export SHUTDOWN=1
>  +_END_
>  +
>  +
>
> +EXCLUDE_TESTS="abrt-nightly-build|abrt-make-check|btparser-make-check|libreport-make-check"
>  +TEST_ORDER="/root/abrt/tests/runtests/aux/test_order"
>  +
>  +egrep -v "$EXCLUDE_TESTS" "$TEST_ORDER" > /tmp/test_order
> && mv /tmp/test_order "$TEST_ORDER"
>  +sed -i 's/OpenGPGCheck.*=.*yes/OpenGPGCheck = no/'
> /etc/abrt/abrt-action-save-package-data.conf
>  +screen -d -m /root/abrt/tests/runtests/run | tee -a /tmp/abrt-nightly-test.log
>  \ No newline at end of file
>  diff --git a/tests/helpers/targets/f16/defs.sh b/tests/helpers/targets/f16/defs.sh
>  index 42f44fd..95d4ffd 100644
>  --- a/tests/helpers/targets/f16/defs.sh
>  +++ b/tests/helpers/targets/f16/defs.sh
>  @@ -7,6 +7,8 @@ OS_VARIANT='fedora16'
>   KS_NAME_PREFIX='fedora_16'
>  
> LOC='http://download.englab.brq.redhat.com/pub/fedora/linux/releases/...
>   DISK=$( echo /dev/mapper/*f16_vm )
>  +MAC="525400846b5b"
>  +IP="10.34.25.210"
>   
>   # http://fedoraproject.org/wiki/Anaconda/Updates
>  
> INIT_EXTRA_ARGS="updates=http://abrt.brq.redhat.com/media/updates-an...
>  diff --git a/tests/helpers/targets/f16/runtests.sh
> b/tests/helpers/targets/f16/runtests.sh
>  new file mode 100644
>  index 0000000..6929f4f
>  --- /dev/null
>  +++ b/tests/helpers/targets/f16/runtests.sh
>  @@ -0,0 +1,35 @@
>  +#! /bin/bash
>  +
>  +pushd $PWD && {
>  +    cd /root && {
>  +        git clone git://abrt.brq.redhat.com/abrt.git ./abrt && {
>  +            cd ./abrt && {
>  +                case $(cut -d: -f4,5 /etc/system-release-cpe) in
>  +                    enterprise_linux:6*)
>  +                        git checkout rhel6
>  +                        ;;
>  +                    *)
>  +                        ;;
>  +                esac
>  +            }
>  +        }
>  +    }
>  +    popd
>  +}
>  +
>  +cat > /root/abrt/tests/runtests/aux/config.sh.local << "_END_"
>  +export FORMAT_SCRIPT='aux/scp_format.sh'
>  +export REPORT_SCRIPT='aux/scp_report.sh'
>  +
>  +export SCPTO='[e-mail]:~/targets/f16/data/'
>  +
>  +export SHUTDOWN=1
>  +_END_
>  +
>  +
>
> +EXCLUDE_TESTS="abrt-nightly-build|abrt-make-check|btparser-make-check|libreport-make-check"
>  +TEST_ORDER="/root/abrt/tests/runtests/aux/test_order"
>  +
>  +egrep -v "$EXCLUDE_TESTS" "$TEST_ORDER" > /tmp/test_order
> && mv /tmp/test_order "$TEST_ORDER"
>  +
>  +screen -d -m /root/abrt/tests/runtests/run | tee -a /tmp/abrt-nightly-test.log
>  \ No newline at end of file
>  diff --git a/tests/helpers/targets/f17/defs.sh b/tests/helpers/targets/f17/defs.sh
>  index 7dcdd7c..57306c2 100644
>  --- a/tests/helpers/targets/f17/defs.sh
>  +++ b/tests/helpers/targets/f17/defs.sh
>  @@ -6,6 +6,8 @@ REPO='git://abrt.brq.redhat.com/abrt.git'
>   OS_VARIANT='fedora16'
>   KS_NAME_PREFIX='fedora_17'
>  
> LOC='http://download.englab.brq.redhat.com/pub/fedora/linux/releases/...
>  +MAC="525400405da9"
>  +IP="10.34.25.200"
>   
>   DISK=$( echo /dev/mapper/*f17_vm )
>   
>  diff --git a/tests/helpers/targets/f17/runtests.sh
> b/tests/helpers/targets/f17/runtests.sh
>  new file mode 100644
>  index 0000000..4d43fde
>  --- /dev/null
>  +++ b/tests/helpers/targets/f17/runtests.sh
>  @@ -0,0 +1,35 @@
>  +#! /bin/bash
>  +
>  +pushd $PWD && {
>  +    cd /root && {
>  +        git clone git://abrt.brq.redhat.com/abrt.git ./abrt && {
>  +            cd ./abrt && {
>  +                case $(cut -d: -f4,5 /etc/system-release-cpe) in
>  +                    enterprise_linux:6*)
>  +                        git checkout rhel6
>  +                        ;;
>  +                    *)
>  +                        ;;
>  +                esac
>  +            }
>  +        }
>  +    }
>  +    popd
>  +}
>  +
>  +cat > /root/abrt/tests/runtests/aux/config.sh.local << "_END_"
>  +export FORMAT_SCRIPT='aux/scp_format.sh'
>  +export REPORT_SCRIPT='aux/scp_report.sh'
>  +
>  +export SCPTO='[e-mail]:~/targets/f17/data/'
>  +
>  +export SHUTDOWN=1
>  +_END_
>  +
>  +
>
> +EXCLUDE_TESTS="abrt-nightly-build|abrt-make-check|btparser-make-check|libreport-make-check"
>  +TEST_ORDER="/root/abrt/tests/runtests/aux/test_order"
>  +
>  +egrep -v "$EXCLUDE_TESTS" "$TEST_ORDER" > /tmp/test_order
> && mv /tmp/test_order "$TEST_ORDER"
>  +
>  +screen -d -m /root/abrt/tests/runtests/run | tee -a /tmp/abrt-nightly-test.log
>  \ No newline at end of file 
targets/$TARGET/runtests.sh files might use some
de-duplication as they
are similar for each target. Would be better to put this file to
~/common/runtests.sh.

-- 
Richard Marko