[PATCH] upgradepath: run on post-bodhi-update-batch, test all updates, send bodhi comments

[Kamil Paral]

    upgradepath: run on post-bodhi-update-batch, test all updates, send bodhi comments

    This patch solves ticket #246 and part of ticket #240.

    Upgradepath is now triggered on batch bodhi events and it tests all
    updates that are present in the -pending tag. That means that some
    updates will be tested many times over, but our intelligent Bodhi
    comment duplication prevention algorithm should prevent us from
    hammering the maintainers with the same results. This behavior is
    probably temporary until we have means to re-schedule a test (ticket #245).

    As stated, this patch also enables upgradepath to send Bodhi comments
    for every update result.

diff --git a/tests/upgradepath/control.autoqa b/tests/upgradepath/control.autoqa
index e63091a..9e17ca2 100644
--- a/tests/upgradepath/control.autoqa
+++ b/tests/upgradepath/control.autoqa
@@ -4,8 +4,8 @@
 # upgradepath can be run just once and on any architecture
 archs = ['noarch']

-# run only with post-bodhi-update hook
-if hook not in ['post-bodhi-update']:
+# do batch run for new updates
+if hook not in ['post-bodhi-update-batch']:
     execute = False

 # don't check requests going into *-updates-testing, upgradepath currently
diff --git a/tests/upgradepath/upgradepath.py b/tests/upgradepath/upgradepath.py
index 15e5c38..567cad5 100755
--- a/tests/upgradepath/upgradepath.py
+++ b/tests/upgradepath/upgradepath.py
@@ -20,6 +20,7 @@

 import operator
 import autoqa.koji_utils
+import autoqa.bodhi_utils
 import rpmUtils.miscutils
 from autoqa.repoinfo import repoinfo
 from autoqa.test import AutoQATest
@@ -34,7 +35,7 @@ class upgradepath(AutoQATest):
         self.result = 'PASSED'
         # order for evaluation of final result; higher index means preference
         self.result_order =
('PASSED','INFO','FAILED','ABORTED')
-        self.envr_results = {} # results for invidual packages
+        self.nvr_results = {} # results for invidual packages
         self.outputs = []
         self.highlights = []

@@ -85,9 +86,19 @@ class upgradepath(AutoQATest):
         return result

     @ExceptionCatcher()
-    def run_once(self, envrs, kojitag, **kwargs):
+    def run_once(self, kojitag, **kwargs):
         super(self.__class__, self).run_once()
-        update_id = kwargs['name'] or kwargs['id']
+
+        # Ideally upgradepath should check just the new updates. But since we don't
+        # yet support test re-scheduling [1], we have to work around somehow [2].
+        # So let's just run upgradepath for *all* updates requesting their move to
+        # <kojitag>. It is a little inefficient, but if for some package the
result
+        # changes, we will report it correctly (and that's more important).
+        # When test re-scheduling is supported, we can revert this test from whole
+        # tag testing to just new updates testing.
+        # [1] https://fedorahosted.org/autoqa/ticket/245
+        # [2] https://fedorahosted.org/autoqa/ticket/246
+

         # Get a list of all repos we monitor (currently not -testing)
         # FIXME - perhaps we should only query for 'active' repos
@@ -103,20 +114,22 @@ class upgradepath(AutoQATest):

         koji = autoqa.koji_utils.SimpleKojiClientSession()

-        for envr in envrs:
-            msg = '%s\n%s into %s\n%s' % (40*'=', envr, kojitag,
40*'=')
+        # get the list of all builds requesting move to the kojitag
+        sourcetag = kojitag + '-pending'
+        print 'Koji tag to be tested: %s' % sourcetag
+        updates = koji.listTagged(sourcetag)
+        nvrs = sorted([u['nvr'] for u in updates])
+        print 'Builds to be tested: %s' % ' '.join(nvrs)
+
+        # for every build let's test it
+        for nvr in nvrs:
+            msg = '%s\n%s into %s\n%s' % (40*'=', nvr, kojitag,
40*'=')
             print msg
             self.outputs.append(msg)

-            # our testing package
-            (name, version, release, epoch, arch) =
rpmUtils.miscutils.splitFilename(envr)
-            matching_build = {
-                    'name': name,
-                    'version' : version,
-                    'release' : release + '.' + arch,
-                    'epoch' : epoch,
-                    'envr' : envr,
-                    }
+            # get all info about this current build
+            matching_build = koji.getBuild(nvr)
+            assert matching_build is not None, 'This build does not exist in Koji:
%s' % nvr

             if kojitag.find('updates') < 0 and repotags[current_tag] !=
repotags[-1]:
                 # not *-updates* and not rawhide
@@ -144,7 +157,7 @@ class upgradepath(AutoQATest):
             # compute pkg result
             if self.result_order.index(result2) > self.result_order.index(result):
                 result = result2
-            self.envr_results[envr] = result
+            self.nvr_results[nvr] = result

             msg = 'RESULT: %s' % result
             print msg
@@ -154,11 +167,13 @@ class upgradepath(AutoQATest):
             if self.result_order.index(result) >
self.result_order.index(self.result):
                 self.result = result

+        print # empty line
+
         # create summary like "1 PASSED, 2 FAILED, 3 INFO"
         summary = []
         for res in self.result_order:
-            if res in self.envr_results.values():
-                count = len([k for k in self.envr_results.keys() if self.envr_results[k]
== res])
+            if res in self.nvr_results.values():
+                count = len([k for k in self.nvr_results.keys() if self.nvr_results[k] ==
res])
                 summary.append('%d %s' % (count, res))
         summary = ', '.join(summary)

@@ -168,5 +183,25 @@ class upgradepath(AutoQATest):
         self.outputs.append(msg)
         self.highlights.append(msg)

-        self.summary = '%s for %s' % (summary, update_id)
-
+        self.summary = '%s for %s' % (summary, sourcetag)
+
+        # send results to Bodhi
+        print 'Sending results to Bodhi...'
+        exc = False
+        for nvr in nvrs:
+            result = self.nvr_results[nvr]
+            try:
+                update = autoqa.bodhi_utils.query_update(nvr)
+                assert update is not None, 'No such update object in Bodhi: %s' %
nvr
+                update_title = update['title']
+                print 'Sending results to Bodhi: %s %s' % (update_title, result)
+                autoqa.bodhi_utils.bodhi_post_testresult(update_title,
'upgradepath',
+                    result, self.autotest_url, 'noarch')
+            except AssertionError, e:
+                msg = 'Failed to send Bodhi results to %s:\n  %s' % (nvr, e)
+                print msg
+                self.outputs.append(msg)
+                exc = True
+        # if assert failed (some Bodhi update doesn't exist), end the test as
CRASHED
+        if exc:
+            raise e

[Kamil Paral]

----- Original Message -----
>  upgradepath: run on post-bodhi-update-batch, test all updates, send
>  bodhi comments
>  
>  This patch solves ticket #246 and part of ticket #240.
>  
>  Upgradepath is now triggered on batch bodhi events and it tests all
>  updates that are present in the -pending tag. That means that some
>  updates will be tested many times over, but our intelligent Bodhi
>  comment duplication prevention algorithm should prevent us from
>  hammering the maintainers with the same results. This behavior is
>  probably temporary until we have means to re-schedule a test (ticket
>  #245).
>  
>  As stated, this patch also enables upgradepath to send Bodhi comments
>  for every update result. 
I have changed the patch a little, I have retained the original option to run on
post-bodhi-update hook, if specified manually. That means we can still use upgradepath
test to manually check some specific build. By default it will run on
post-bodhi-update-batch, checking all builds currently in *-updates-pending repo.

The new patch (against master) is here:


diff --git a/tests/upgradepath/control.autoqa b/tests/upgradepath/control.autoqa
index e63091a..90e7b97 100644
--- a/tests/upgradepath/control.autoqa
+++ b/tests/upgradepath/control.autoqa
@@ -4,8 +4,9 @@
 # upgradepath can be run just once and on any architecture
 archs = ['noarch']
 
-# run only with post-bodhi-update hook
-if hook not in ['post-bodhi-update']:
+# do batch run for new updates
+# (post-bodhi-update is still supported for manual execution)
+if hook not in ['post-bodhi-update-batch']:
     execute = False
 
 # don't check requests going into *-updates-testing, upgradepath currently
diff --git a/tests/upgradepath/upgradepath.py b/tests/upgradepath/upgradepath.py
index 1cf1eaf..d090486 100755
--- a/tests/upgradepath/upgradepath.py
+++ b/tests/upgradepath/upgradepath.py
@@ -20,6 +20,7 @@
 
 import operator
 import autoqa.koji_utils
+import autoqa.bodhi_utils
 import rpmUtils.miscutils
 from autoqa.repoinfo import repoinfo
 from autoqa.test import AutoQATest
@@ -85,9 +86,19 @@ class upgradepath(AutoQATest):
         return result
 
     @ExceptionCatcher()
-    def run_once(self, nvrs, kojitag, **kwargs):
+    def run_once(self, kojitag, hook, **kwargs):
         super(self.__class__, self).run_once()
-        update_id = kwargs['name'] or kwargs['id']
+
+        # Ideally upgradepath should check just the new updates. But since we don't
+        # yet support test re-scheduling [1], we have to work around somehow [2].
+        # So let's just run upgradepath for *all* updates requesting their move to
+        # <kojitag>. It is a little inefficient, but if for some package the
result
+        # changes, we will report it correctly (and that's more important).
+        # When test re-scheduling is supported, we can revert this test from whole
+        # tag testing to just new updates testing.
+        # [1] https://fedorahosted.org/autoqa/ticket/245
+        # [2] https://fedorahosted.org/autoqa/ticket/246
+
 
         # Get a list of all repos we monitor (currently not -testing)
         # FIXME - perhaps we should only query for 'active' repos
@@ -103,20 +114,38 @@ class upgradepath(AutoQATest):
 
         koji = autoqa.koji_utils.SimpleKojiClientSession()
 
+        if hook == "post-bodhi-update-batch":
+            # get the list of all builds requesting move to the kojitag
+            sourcetag = kojitag + '-pending'
+            print 'Koji tag to be tested: %s' % sourcetag
+            updates = koji.listTagged(sourcetag)
+            nvrs = sorted([autoqa.koji_utils.getENVR(u) for u in updates])
+            print 'Builds to be tested: %s' % ' '.join(nvrs)
+            update_id = sourcetag
+        else: # hook is post-bodhi-update
+            update_id = kwargs['name'] or kwargs['id']
+            nvrs = kwargs['nvrs']
+
+        # for every build let's test it
         for nvr in nvrs:
             msg = '%s\n%s into %s\n%s' % (40*'=', nvr, kojitag,
40*'=')
             print msg
             self.outputs.append(msg)
 
-            # our testing package
-            (name, version, release, epoch, arch) = rpmUtils.miscutils.splitFilename(nvr
+ '.noarch')
-            matching_build = {
+            # get all info about this current build
+            if hook == "post-bodhi-update-batch":
+                matching_build = koji.getBuild(nvr)
+                assert matching_build is not None, 'This build does not exist in
Koji: %s' % nvr
+            else: # hook is post-bodhi-update
+                # don't query Koji, allow non-existing builds to be specified
manually
+                (name, version, release, epoch, arch) =
rpmUtils.miscutils.splitFilename(nvr + '.noarch')
+                matching_build = {
                     'name': name,
                     'version' : version,
                     'release' : release,
                     'epoch' : epoch,
                     'nvr' : nvr,
-                    }
+                }
 
             if kojitag.find('updates') < 0 and repotags[current_tag] !=
repotags[-1]:
                 # not *-updates* and not rawhide
@@ -155,6 +184,8 @@ class upgradepath(AutoQATest):
             if self.result_order.index(result) >
self.result_order.index(self.result):
                 self.result = result
 
+        print # empty line
+
         # create summary like "1 PASSED, 2 FAILED, 3 INFO"
         summary = []
         for res in self.result_order:
@@ -171,3 +202,23 @@ class upgradepath(AutoQATest):
 
         self.summary = '%s for %s' % (summary, update_id)
 
+        # send results to Bodhi
+        print 'Sending results to Bodhi...'
+        exc = False
+        for nvr in nvrs:
+            result = self.nvr_results[nvr]
+            try:
+                update = autoqa.bodhi_utils.query_update(nvr)
+                assert update is not None, 'No such update object in Bodhi: %s' %
nvr
+                update_title = update['title']
+                print 'Sending results to Bodhi: %s %s' % (update_title, result)
+                autoqa.bodhi_utils.bodhi_post_testresult(update_title,
'upgradepath',
+                    result, self.autotest_url, 'noarch')
+            except AssertionError, e:
+                msg = 'Failed to send Bodhi results to %s:\n  %s' % (nvr, e)
+                print msg
+                self.outputs.append(msg)
+                exc = True
+        # if assert failed (some Bodhi update doesn't exist), end the test as
CRASHED
+        if exc:
+            raise e