[PATCH] Test for checking initscripts using beakerlib based tests from RedHat BaseOS team

[Josef Skladanka]

Hello,

attached patch contains differences between the current master and 
branch 'beakerlib', which runs initscript tests developed by RedHat 
BaseOS team.

There is only the test-runner (reviewed by kparal), in the body of this 
email, the whole patch containing some tests, already approved to be 
pushed upstream, is attached.

Joza



diff --git a/hooks/post-koji-build/testlist b/hooks/post-koji-build/testlist
index d0a9b73..0b5a9f3 100644
--- a/hooks/post-koji-build/testlist
+++ b/hooks/post-koji-build/testlist
@@ -1,2 +1,3 @@
  rpmlint
  rpmguard
+initscripts
diff --git a/tests/initscripts/control b/tests/initscripts/control
new file mode 100644
index 0000000..fec4fc0
--- /dev/null
+++ b/tests/initscripts/control
@@ -0,0 +1,17 @@
+# vim: set syntax=python
+TIME="SHORT"
+AUTHOR = "Josef Skladanka <jskladan(a)redhat.com&gt;&quot;
+DOC = """
+Initscript checker, which uses Beakerlib based tests from RHEL BaseOS
+"""
+NAME = 'initscripts'
+TEST_TYPE = 'CLIENT' # SERVER can be used for tests that need multiple 
machines
+TEST_CLASS = 'General'
+TEST_CATEGORY = 'Functional'
+
+# post-koji-build tests can expect the following variables from autoqa:
+# envr: package ENVR (required, epoch can be skipped)
+# name: package name
+# kojitag: koji tag applied to this package
+
+job.run_test('initscripts', name=name, envr=envr, kojitag=kojitag, 
config=autoqa_conf)
diff --git a/tests/initscripts/initscripts.py 
b/tests/initscripts/initscripts.py
new file mode 100644
index 0000000..634a04d
--- /dev/null
+++ b/tests/initscripts/initscripts.py
@@ -0,0 +1,168 @@
+#
+# Copyright 2010, Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Author: Josef Skladanka <jskladan(a)redhat.com&gt;
+
+import os
+import shutil
+import subprocess
+import re
+import autoqa.koji_utils
+import autoqa.util
+import koji
+from autotest_lib.client.bin import test, utils
+from autotest_lib.client.bin.test_config import config_loader
+from autotest_lib.client.common_lib import error
+
+class initscripts(test.test):
+    version = 1 # increment if setup() changes
+
+    def initialize(self, envr, name, kojitag, config):
+        self.config = config_loader(config, self.tmpdir)
+
+    def setup(self):
+        """
+        adds beakerlib repository and installs beakerlib
+        """
+        retval = subprocess.call(["grep", "-R", 
"http://afri.fedorapeople.org/beakerlib/", "/etc/yum.repos.d/"])
+        if retval != 0:
+            f = open("/etc/yum.repos.d/beakerlib.repo", "w")
+            f.write("[beakerlib-testing]\nname=Testing repo for 
BeakerLib\nbaseurl=http://afri.fedorapeople.org/beakerlib/\nenabled=1\ngp...)
+            f.close()
+        utils.system('yum -y install beakerlib')
+
+    def run_beakerlib_test(self, name, path_to_runtest):
+        #install required packages
+        """
+        there are lines like
+          @echo "Requires:        openssh" >> $(METADATA)
+        in the Makefile, specifying which packages are needed to run 
the respective test.
+        this code extracts package names from these lines and installs 
the packages
+
+        We also want to skip the "Requires" line with the package name, 
since it was installed
+         from koji. Makefile has "RunFor" line, which specifies this 
package name.
+        """
+        makefile = os.path.join(path_to_runtest, "Makefile")
+        if (os.path.isfile(makefile)):
+            #grab the Requires lines
+            req_lines = filter(lambda x: x.find("Requires:") > -1, 
open(makefile, 'r').readlines())
+            requires = map(lambda x: 
re.findall(".*Requires:\s*([^\"']+).*", x)[0], req_lines)
+
+            #grab the RunFor lines
+            run_for_lines = filter(lambda x: x.find("RunFor:") > -1, 
open(makefile, 'r').readlines())
+            run_for = map(lambda x: 
re.findall(".*RunFor:\s*([^\"']+).*", x)[0], run_for_lines)
+
+            #filter out RunFor from Requires, as RunFor was installed 
from koji
+            requires = filter(lambda x: x not in run_for, requires)
+            if len(requires) > 0:
+                utils.system("yum -y install %s" % "
".join(requires))
+
+        #run the test
+        cmd = os.path.join(path_to_runtest, "runtest.sh")
+        result = utils.run(cmd, ignore_status = True, stdout_tee = 
utils.TEE_TO_LOGS)
+        self.results += result.stdout
+
+        #store the log journal created by beakerlib
+        """
+        filters lines containing information about journal files (logs 
from the test script)
+        :: [15:22:06] ::  JOURNAL XML: /tmp/beakerlib-CoEg7JM/journal.xml
+        :: [15:22:06] ::  JOURNAL TXT: /tmp/beakerlib-CoEg7JM/journal.txt
+        extracts paths to these files, ands stores the files in 
self.resultsdir
+        """
+        journal_lines = filter(lambda x: x.find("JOURNAL") > -1, 
self.results.splitlines())
+        journal_files = map(lambda x: x.split(':')[-1].strip(), 
journal_lines)
+        for fname in journal_files:
+            shutil.copy(fname, os.path.join(self.resultsdir, 
"journal_%s.%s" % (name, fname.split('.')[-1])))
+
+        return result.exit_status
+
+
+
+
+    def run_once(self, envr, name, kojitag):
+        """
+        name - name of the package for initscript test
+        """
+
+        #find all tests for package $name
+        testdir = os.path.join(self.bindir, "./tests/%s" % name)
+        tests = []
+        if (os.path.isdir(testdir)):
+            """
+            walks through ./tests/$NAME and all it's subdirs. Every 
time it finds
+            file runtest.sh, it stores a tuple(test_name, path) to tests[],
+            where test_name is 'openssh' for ./tests/openssh, 
openssh_subdir for ./tests/openssh/subdir, etc.
+            """
+            for (path, dirs, files) in os.walk(testdir):
+                if "runtest.sh" in files:
+                    test_name = path.replace(os.path.join(self.bindir, 
"./tests/"), "").replace(os.path.sep, "_")
+                    tests.append((test_name, path))
+        else:
+            raise error.TestWarn("No initscript checker found for 
package %s" % name)
+
+
+        #install packages from koji (stolen from rpmlint test)
+        koji = autoqa.koji_utils.SimpleKojiClientSession()
+        pkgurls = koji.nvr_to_urls(envr, arches = os.uname()[-1])
+        rpmdir = os.path.join(self.tmpdir, 'rpms')
+        os.makedirs(rpmdir)
+        rpms = []
+        print "Saving RPMs to %s" % rpmdir
+        #download packages
+        for p in pkgurls:
+            # fetch package to rpmdir
+            print "Grabbing %s" % p
+            localfile = os.path.join(rpmdir, os.path.basename(p))
+            autoqa.util.grabber.urlgrab(p, localfile)
+            rpms.append(localfile)
+        #and install them
+        cmd = "yum -y --nogpgcheck localinstall %s" % " ".join(rpms)
+        utils.system_output(cmd)
+
+        #sort tests alphabetically by test_name
+        tests.sort(key = lambda x: x[0])
+        self.results = ""
+        tests_exit_status = []
+        for test in tests:
+            self.results += "RUNNING: %s\n%s\n" % (test[0], "=" * 
len("RUNNING: %s" % test[0]))
+            tests_exit_status.append((test[0], 
self.run_beakerlib_test(test[0], test[1])))
+            self.results += "\n"
+
+
+        tests_exit_status_nonzero = filter(lambda x: x[1] != 0, 
tests_exit_status)
+        tests_exit_status_zero = filter(lambda x: x[1] == 0, 
tests_exit_status)
+
+        #send email with results
+        mail_to = self.config.get('test', 'result_email')
+        mail_from = self.config.get('test', 'mail_from')
+        mail_server = self.config.get('test', 'smtpserver')
+        if (mail_to and mail_from and mail_server):
+            msg =  "Following tests returned zero exit code: %s\n" % 
repr(tests_exit_status_zero)
+            msg += "Following tests returned NON-zero exit code: %s\n" 
% repr(tests_exit_status_nonzero)
+            msg += "%s\n" % ("#" * 79,)
+
+            print "Sending email with results to '%s' ..." % mail_to
+            utils.send_email(mail_to = mail_to,
+                             mail_from = mail_from,
+                             mail_server = mail_server,
+                             subject = 'initscripts: %d PASSED, %d 
FAILED for package %s' % (len(tests_exit_status_zero), 
len(tests_exit_status_nonzero), envr),
+                             body = msg + self.results)
+
+        if len(tests_exit_status_nonzero) > 0:
+             msg = "Following tests returned non-zero exit code: %s" % 
repr(tests_exit_status_nonzero)
+             raise error.TestError(msg)
+

[James Laska]

On Mon, 2010-03-15 at 10:45 +0100, Josef Skladanka wrote:
>  Hello,
>  
>  attached patch contains differences between the current master and 
>  branch 'beakerlib', which runs initscript tests developed by RedHat 
>  BaseOS team.
>  
>  There is only the test-runner (reviewed by kparal), in the body of this 
>  email, the whole patch containing some tests, already approved to be 
>  pushed upstream, is attached. 
Great stuff Josef!  I'm excited to see initscript jobs processing in
AutoQA.  I'll leave the technical review to wwoods and kparal, but one
thing I noticed is that the body of the mail, and the attached patch are
different.  How did you generate the patch to send to the list, did you
use git-format-patch and git-send-email (from the patch submission
process [1]) or some other technique?

Thanks,
James

[1] https://fedoraproject.org/wiki/AutoQA_PatchProcess#Patch_submission

[Josef Skladanka]

Thank you for the link, James, must have slipped my attention.

I used odrinary git-diff against master to generate the patch, since i 
did not want it to contain all the commit information in my branch, but 
just the code.

The body of the e-mail and the patch itself are different, because i 
wanted to put only the important information to the body (i.e. the code 
i wrote, without the tests we took from base-os team).

Hope it did not cause you (or Will) any trouble, and I can of course 
provide the patch using standard git-format-patch way.

joza

On 03/22/2010 08:26 PM, James Laska wrote:
>  Great stuff Josef!  I'm excited to see initscript jobs processing
> in
>  AutoQA.  I'll leave the technical review to wwoods and kparal, but one
>  thing I noticed is that the body of the mail, and the attached patch are
>  different.  How did you generate the patch to send to the list, did you
>  use git-format-patch and git-send-email (from the patch submission
>  process [1]) or some other technique?
>
>  Thanks,
>  James
>
>  [1] https://fedoraproject.org/wiki/AutoQA_PatchProcess#Patch_submission
>
>
>
>  _______________________________________________
>  autoqa-devel mailing list
>  autoqa-devel(a)lists.fedorahosted.org
>  https://fedorahosted.org/mailman/listinfo/autoqa-devel

[James Laska]

On Tue, 2010-03-23 at 12:48 +0100, Josef Skladanka wrote:
>  Thank you for the link, James, must have slipped my attention.
>  
>  I used odrinary git-diff against master to generate the patch, since i 
>  did not want it to contain all the commit information in my branch, but 
>  just the code.
>  
>  The body of the e-mail and the patch itself are different, because i 
>  wanted to put only the important information to the body (i.e. the code 
>  i wrote, without the tests we took from base-os team).
>  
>  Hope it did not cause you (or Will) any trouble, and I can of course 
>  provide the patch using standard git-format-patch way. 
No worries, all good!  We're still figuring out a good standard practice
for this stuff.

wwoods pointed me to 'git-rebase -i' as a way to combine many patches
into more meaningful units.  I'll have to give that a try if I have a
merge request out to the list (unlikely ... you guys are the coders!)

Thanks,
James

[Josef Skladanka]

Hello Will,

How is the patch rewiev going on - James told me, that you were quite 
happy about it, so i belive, it just slipped you mind.

Let me know, if i can do anything to speed up the process.

Joza

On 03/15/2010 10:45 AM, Josef Skladanka wrote:
>  Hello,
>
>  attached patch contains differences between the current master and
>  branch 'beakerlib', which runs initscript tests developed by RedHat
>  BaseOS team.
>
>  There is only the test-runner (reviewed by kparal), in the body of this
>  email, the whole patch containing some tests, already approved to be
>  pushed upstream, is attached.
>
>  Joza
>
>
>
>  diff --git a/hooks/post-koji-build/testlist
>  b/hooks/post-koji-build/testlist
>  index d0a9b73..0b5a9f3 100644
>  --- a/hooks/post-koji-build/testlist
>  +++ b/hooks/post-koji-build/testlist
>  @@ -1,2 +1,3 @@
>  rpmlint
>  rpmguard
>  +initscripts
>  diff --git a/tests/initscripts/control b/tests/initscripts/control
>  new file mode 100644
>  index 0000000..fec4fc0
>  --- /dev/null
>  +++ b/tests/initscripts/control
>  @@ -0,0 +1,17 @@
>  +# vim: set syntax=python
>  +TIME="SHORT"
>  +AUTHOR = "Josef Skladanka <jskladan(a)redhat.com&gt;&quot;
>  +DOC = """
>  +Initscript checker, which uses Beakerlib based tests from RHEL BaseOS
>  +"""
>  +NAME = 'initscripts'
>  +TEST_TYPE = 'CLIENT' # SERVER can be used for tests that need multiple
>  machines
>  +TEST_CLASS = 'General'
>  +TEST_CATEGORY = 'Functional'
>  +
>  +# post-koji-build tests can expect the following variables from autoqa:
>  +# envr: package ENVR (required, epoch can be skipped)
>  +# name: package name
>  +# kojitag: koji tag applied to this package
>  +
>  +job.run_test('initscripts', name=name, envr=envr, kojitag=kojitag,
>  config=autoqa_conf)
>  diff --git a/tests/initscripts/initscripts.py
>  b/tests/initscripts/initscripts.py
>  new file mode 100644
>  index 0000000..634a04d
>  --- /dev/null
>  +++ b/tests/initscripts/initscripts.py
>  @@ -0,0 +1,168 @@
>  +#
>  +# Copyright 2010, Red Hat, Inc.
>  +#
>  +# This program is free software; you can redistribute it and/or modify
>  +# it under the terms of the GNU General Public License as published by
>  +# the Free Software Foundation; either version 2 of the License, or
>  +# (at your option) any later version.
>  +#
>  +# This program is distributed in the hope that it will be useful,
>  +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>  +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>  +# GNU General Public License for more details.
>  +#
>  +# You should have received a copy of the GNU General Public License along
>  +# with this program; if not, write to the Free Software Foundation, Inc.,
>  +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
>  +#
>  +# Author: Josef Skladanka <jskladan(a)redhat.com&gt;
>  +
>  +import os
>  +import shutil
>  +import subprocess
>  +import re
>  +import autoqa.koji_utils
>  +import autoqa.util
>  +import koji
>  +from autotest_lib.client.bin import test, utils
>  +from autotest_lib.client.bin.test_config import config_loader
>  +from autotest_lib.client.common_lib import error
>  +
>  +class initscripts(test.test):
>  + version = 1 # increment if setup() changes
>  +
>  + def initialize(self, envr, name, kojitag, config):
>  + self.config = config_loader(config, self.tmpdir)
>  +
>  + def setup(self):
>  + """
>  + adds beakerlib repository and installs beakerlib
>  + """
>  + retval = subprocess.call(["grep", "-R",
>  "http://afri.fedorapeople.org/beakerlib/", "/etc/yum.repos.d/"])
>  + if retval != 0:
>  + f = open("/etc/yum.repos.d/beakerlib.repo", "w")
>  + f.write("[beakerlib-testing]\nname=Testing repo for
>
> BeakerLib\nbaseurl=http://afri.fedorapeople.org/beakerlib/\nenabled=1\ngp...)
>
>  + f.close()
>  + utils.system('yum -y install beakerlib')
>  +
>  + def run_beakerlib_test(self, name, path_to_runtest):
>  + #install required packages
>  + """
>  + there are lines like
>  + @echo "Requires: openssh" >> $(METADATA)
>  + in the Makefile, specifying which packages are needed to run the
>  respective test.
>  + this code extracts package names from these lines and installs the
>  packages
>  +
>  + We also want to skip the "Requires" line with the package name, since
>  it was installed
>  + from koji. Makefile has "RunFor" line, which specifies this package name.
>  + """
>  + makefile = os.path.join(path_to_runtest, "Makefile")
>  + if (os.path.isfile(makefile)):
>  + #grab the Requires lines
>  + req_lines = filter(lambda x: x.find("Requires:") > -1, open(makefile,
>  'r').readlines())
>  + requires = map(lambda x: re.findall(".*Requires:\s*([^\"']+).*",
>  x)[0], req_lines)
>  +
>  + #grab the RunFor lines
>  + run_for_lines = filter(lambda x: x.find("RunFor:") > -1,
>  open(makefile, 'r').readlines())
>  + run_for = map(lambda x: re.findall(".*RunFor:\s*([^\"']+).*",
> x)[0],
>  run_for_lines)
>  +
>  + #filter out RunFor from Requires, as RunFor was installed from koji
>  + requires = filter(lambda x: x not in run_for, requires)
>  + if len(requires) > 0:
>  + utils.system("yum -y install %s" % " ".join(requires))
>  +
>  + #run the test
>  + cmd = os.path.join(path_to_runtest, "runtest.sh")
>  + result = utils.run(cmd, ignore_status = True, stdout_tee =
>  utils.TEE_TO_LOGS)
>  + self.results += result.stdout
>  +
>  + #store the log journal created by beakerlib
>  + """
>  + filters lines containing information about journal files (logs from
>  the test script)
>  + :: [15:22:06] :: JOURNAL XML: /tmp/beakerlib-CoEg7JM/journal.xml
>  + :: [15:22:06] :: JOURNAL TXT: /tmp/beakerlib-CoEg7JM/journal.txt
>  + extracts paths to these files, ands stores the files in self.resultsdir
>  + """
>  + journal_lines = filter(lambda x: x.find("JOURNAL") > -1,
>  self.results.splitlines())
>  + journal_files = map(lambda x: x.split(':')[-1].strip(), journal_lines)
>  + for fname in journal_files:
>  + shutil.copy(fname, os.path.join(self.resultsdir, "journal_%s.%s" %
>  (name, fname.split('.')[-1])))
>  +
>  + return result.exit_status
>  +
>  +
>  +
>  +
>  + def run_once(self, envr, name, kojitag):
>  + """
>  + name - name of the package for initscript test
>  + """
>  +
>  + #find all tests for package $name
>  + testdir = os.path.join(self.bindir, "./tests/%s" % name)
>  + tests = []
>  + if (os.path.isdir(testdir)):
>  + """
>  + walks through ./tests/$NAME and all it's subdirs. Every time it finds
>  + file runtest.sh, it stores a tuple(test_name, path) to tests[],
>  + where test_name is 'openssh' for ./tests/openssh, openssh_subdir for
>  ./tests/openssh/subdir, etc.
>  + """
>  + for (path, dirs, files) in os.walk(testdir):
>  + if "runtest.sh" in files:
>  + test_name = path.replace(os.path.join(self.bindir, "./tests/"),
>  "").replace(os.path.sep, "_")
>  + tests.append((test_name, path))
>  + else:
>  + raise error.TestWarn("No initscript checker found for package %s" % name)
>  +
>  +
>  + #install packages from koji (stolen from rpmlint test)
>  + koji = autoqa.koji_utils.SimpleKojiClientSession()
>  + pkgurls = koji.nvr_to_urls(envr, arches = os.uname()[-1])
>  + rpmdir = os.path.join(self.tmpdir, 'rpms')
>  + os.makedirs(rpmdir)
>  + rpms = []
>  + print "Saving RPMs to %s" % rpmdir
>  + #download packages
>  + for p in pkgurls:
>  + # fetch package to rpmdir
>  + print "Grabbing %s" % p
>  + localfile = os.path.join(rpmdir, os.path.basename(p))
>  + autoqa.util.grabber.urlgrab(p, localfile)
>  + rpms.append(localfile)
>  + #and install them
>  + cmd = "yum -y --nogpgcheck localinstall %s" % " ".join(rpms)
>  + utils.system_output(cmd)
>  +
>  + #sort tests alphabetically by test_name
>  + tests.sort(key = lambda x: x[0])
>  + self.results = ""
>  + tests_exit_status = []
>  + for test in tests:
>  + self.results += "RUNNING: %s\n%s\n" % (test[0], "=" *
> len("RUNNING:
>  %s" % test[0]))
>  + tests_exit_status.append((test[0], self.run_beakerlib_test(test[0],
>  test[1])))
>  + self.results += "\n"
>  +
>  +
>  + tests_exit_status_nonzero = filter(lambda x: x[1] != 0,
>  tests_exit_status)
>  + tests_exit_status_zero = filter(lambda x: x[1] == 0, tests_exit_status)
>  +
>  + #send email with results
>  + mail_to = self.config.get('test', 'result_email')
>  + mail_from = self.config.get('test', 'mail_from')
>  + mail_server = self.config.get('test', 'smtpserver')
>  + if (mail_to and mail_from and mail_server):
>  + msg = "Following tests returned zero exit code: %s\n" %
>  repr(tests_exit_status_zero)
>  + msg += "Following tests returned NON-zero exit code: %s\n" %
>  repr(tests_exit_status_nonzero)
>  + msg += "%s\n" % ("#" * 79,)
>  +
>  + print "Sending email with results to '%s' ..." % mail_to
>  + utils.send_email(mail_to = mail_to,
>  + mail_from = mail_from,
>  + mail_server = mail_server,
>  + subject = 'initscripts: %d PASSED, %d FAILED for package %s' %
>  (len(tests_exit_status_zero), len(tests_exit_status_nonzero), envr),
>  + body = msg + self.results)
>  +
>  + if len(tests_exit_status_nonzero) > 0:
>  + msg = "Following tests returned non-zero exit code: %s" %
>  repr(tests_exit_status_nonzero)
>  + raise error.TestError(msg)
>  +
>
>
>
>  _______________________________________________
>  autoqa-devel mailing list
>  autoqa-devel(a)lists.fedorahosted.org
>  https://fedorahosted.org/mailman/listinfo/autoqa-devel

[Will Woods]

On Mon, 2010-03-15 at 10:45 +0100, Josef Skladanka wrote:
>  Hello,
>  
>  attached patch contains differences between the current master and 
>  branch 'beakerlib', which runs initscript tests developed by RedHat 
>  BaseOS team.
>  
>  There is only the test-runner (reviewed by kparal), in the body of this 
>  email, the whole patch containing some tests, already approved to be 
>  pushed upstream, is attached. 
The attached (full) patch seems to just add a new test and not change
any existing files but testlist. Is that right?

If so, then this patch is fine with me - please apply to master.

-w