[PATCH] tidy up upgradepath test code
by Kamil Paral
This patch introduces no new features and (hopefully) no new bugs. It
just reformats the code to be more readable and maintainable - now the
semantics and variables are very similar to those used in rpmlint and
rpmguard tests, so it's easy to read through the code.
---
tests/upgradepath/upgradepath.py | 99 ++++++++++++++++++++++++++------------
1 files changed, 68 insertions(+), 31 deletions(-)
diff --git a/tests/upgradepath/upgradepath.py b/tests/upgradepath/upgradepath.py
index 028b8aa..bc382d3 100755
--- a/tests/upgradepath/upgradepath.py
+++ b/tests/upgradepath/upgradepath.py
@@ -36,52 +36,58 @@ from autotest_lib.client.bin.test_config import config_loader
class upgradepath(AutoQATest):
version = 1 # increment if setup() changes
- test_result = 'PASS'
- log = []
- envr_list = set()
def initialize(self, config):
self.config = config_loader(config, self.tmpdir)
#URL of logs/results stored on autotest-server
self.autotest_url = autoqa.util.make_autotest_url(self.config)
+ self.result = 'PASSED'
+ # order for evaluation of final result; higher index means preference
+ self.result_order = ('PASSED','INFO','FAILED','ABORTED')
+ self.envr_results = {} # results for invidual packages
+ self.outputs = []
+ self.highlights = []
+ self.log = open(os.path.join(self.resultsdir,'upgradepath.log'),'wb') # where to log output
+
def setup(self):
pass
def compare(self, matching_build, tags, op, opstr):
koji = autoqa.koji_utils.SimpleKojiClientSession()
+ result = 'PASSED'
for tag in tags:
latest_build = koji.latest_by_tag(tag, matching_build['name'])
if latest_build is None:
msg ='{0:<7}{1}'.format('[ OK ]', tag)
- self.log.append(msg)
print msg
+ self.outputs.append(msg)
msg = "\tNo such package: %s" % matching_build['name']
- self.log.append(msg)
print msg
+ self.outputs.append(msg)
else:
- result = autoqa.koji_utils.compareEVR(matching_build, latest_build)
+ res = autoqa.koji_utils.compareEVR(matching_build, latest_build)
- if op(result, 0):
+ if op(res, 0):
ok = True
msg ='{0:<7}{1}'.format('[ OK ]', tag)
print msg
- self.log.append(msg)
+ self.outputs.append(msg)
else:
ok = False
- self.test_result = 'FAIL'
msg ='{0:<7}{1}'.format('[FAIL]', tag)
print msg
- self.log.append(msg)
- self.envr_list.add(matching_build['envr'])
+ self.outputs.append(msg)
msg = "\tLatest package: %s" % latest_build['nvr']
- self.log.append(msg)
print msg
+ self.outputs.append(msg)
if not ok:
msg = '\tFailed condition: Requested package %s Latest package' % opstr
- self.log.append(msg)
print msg
+ self.outputs.append(msg)
+ result = 'FAILED'
+ return result
@ExceptionCatcher("self.run_once_failed")
def run_once(self, envrs, kojitag, **kwargs):
@@ -99,10 +105,12 @@ class upgradepath(AutoQATest):
hi_tags = repotags[(current_tag + 1):] # tags higher than current
low_eq_tags = repotags[:(current_tag + 1)] # tags lower or equal
+ koji = autoqa.koji_utils.SimpleKojiClientSession()
+
for envr in envrs:
msg = '%s\n%s into %s\n%s' % (40*'=', envr, kojitag, 40*'=')
print msg
- self.log.append(msg)
+ self.outputs.append(msg)
# our testing package
(name, version, release, epoch, arch) = rpmUtils.miscutils.splitFilename(envr)
@@ -114,34 +122,63 @@ class upgradepath(AutoQATest):
'envr' : envr,
}
- if kojitag.find('updates') == -1 and repotags[current_tag] != repotags[-1]:
- msg = "Warning: Pushing into stable repository"
- self.log.append(msg)
+ if kojitag.find('updates') < 0 and repotags[current_tag] != repotags[-1]:
+ # not *-updates* and not rawhide
+ # FIXME: don't warn about pushing into base repo for both rawhide and branched (but possibly only before change deadline?)
+ msg = "Warning: Pushing into the base 'fedora' repository"
print msg
+ self.outputs.append(msg)
+ if msg not in self.highlights:
+ # this message would be repeated for every package, highlight only once
+ self.highlights.append(msg)
- koji = autoqa.koji_utils.SimpleKojiClientSession()
latest_build = koji.latest_by_tag(kojitag, matching_build['name'])
if latest_build is not None:
- result = autoqa.koji_utils.compareEVR(matching_build, latest_build)
- if result <= 0:
- msg = "Warning: Same or newer build already exists: %s" % latest_build['nvr']
- self.log.append(msg)
+ if autoqa.koji_utils.compareEVR(matching_build, latest_build) <= 0:
+ msg = "Warning: Same or newer build already exists in the same repo: %s" % latest_build['nvr']
print msg
+ self.outputs.append(msg)
+ self.highlights.append(msg)
# compare with lower or equal tags, so version has to be greater or equal
- self.compare(matching_build, low_eq_tags, operator.ge, '>=')
+ result = self.compare(matching_build, low_eq_tags, operator.ge, '>=')
# compare with higher tags, so version has to be lower or equal
- self.compare(matching_build, hi_tags, operator.le, '<=')
+ result2 = self.compare(matching_build, hi_tags, operator.le, '<=')
+
+ # compute pkg result
+ if self.result_order.index(result2) > self.result_order.index(result):
+ result = result2
+ self.envr_results[envr] = result
+
+ msg = 'RESULT: %s' % result
+ print msg
+ self.outputs.append(msg)
+
+ # update self.result
+ if self.result_order.index(result) > self.result_order.index(self.result):
+ self.result = result
+
+ # create summary like "1 PASSED, 2 FAILED, 3 INFO"
+ summary = []
+ for res in self.result_order:
+ if res in self.envr_results.values():
+ count = len([k for k in self.envr_results.keys() if self.envr_results[k] == res])
+ summary.append('%d %s' % (count, res))
+ summary = ', '.join(summary)
- summary = str(len(envrs) - len(self.envr_list)) + ' OK, ' + str(len(self.envr_list)) + ' FAILED'
+ # print summary
msg = 'SUMMARY: ' + summary
- self.log.append(msg)
print msg
+ self.outputs.append(msg)
+ self.highlights.append(msg)
- self.result = self.test_result
- self.outputs = ""
- for i in self.log:
- self.outputs += i + '\n'
- self.outputs += '\n \n'
self.summary = '%s for %s' % (summary, update_id)
+ # reformat result variables
+ self.outputs = '\n'.join(self.outputs)
+ self.highlights = '\n'.join(self.highlights)
+
+ # log outputs
+ self.log.write(self.outputs)
+ self.log.close()
+
--
1.7.2.3
13 years, 6 months
Re: [PATCH] upgradepath doesn't support -testing repos yet, disable scheduling
by Kamil Paral
----- "James Laska" <jlaska(a)redhat.com> wrote:
>
> Hey gang,
>
> I'm failing when looking into the list archives to review our
> previous
> conclusions on this topic. I know we talked about this before, but I
> can't find the threads.
Weird, I think we did discuss it, I even think of some ascii art showing
possible problems. But I found just this, nothing more:
https://fedorahosted.org/pipermail/autoqa-devel/2010-August/000982.html
>
> The more I think about the upgradepath test, the more I come up with
> scenarios where I (as a maintainer) would want to know the
> upgradepath
> result for my updates in 'updates-testing'. I recognize there might
> be
> some weird policy issues here, perhaps we need to get clarification
> on
> this from FESCO?
I have search through whole fedoraproject wiki and a lot of pages talk
about upgrade path, but none defines it exactly. Therefore we could ask
FESCo about exact definition, yes, that's probably a good idea (to have
some solid background for what we do).
The main question is whether we want to have upgrade path constraint
fulfilled when dealing with updates-testing repositories and what exactly
it means in this case. Does it mean upgrading from F12-updates-testing to
F13-updates? Or does it mean upgrading from F12-updates-testing to
F13-updates-testing? We don't know, and that's the problem.
I'll write another email talking about the problems that may arise.
>
> I know there is also the potential that an update in
> 'updates-testing'
> fails during functional validation, and never lands in 'stable'.
> Isn't
> this scenario covered by also scheduling upgradepath for bodhi
> 'updates'
> requests?
>
> Sorry for coming back to this topic, I think I need some re-educating
> on
> what 'upgradepath' means.
First of all, I'd like to have my patch accepted, if possible. That should
put into a consistent state - 'updates' checked correctly, 'updates-testing'
not checked at all. Right now we are in an inconsistent state.
Also I have some more patches coming for upgradepath test, so pushing into
master would save me some trouble.
When we are in a consistent state, we can unleash the debate about the
updates-testing problem :) In a following email I'll sum up the main problems
that arise and what we can do about it.
Since Will is now on PTO, do you think I can push this into master, James?
13 years, 6 months
Re: [PATCH] upgradepath doesn't support -testing repos yet, disable scheduling
by Kamil Paral
----- "James Laska" <jlaska(a)redhat.com> wrote:
> On Thu, 2010-09-23 at 13:04 +0200, Kamil Páral wrote:
> > This patch kind of reverts fbe9ee91b. We don't want upgradepath to
> be run
> > for -testing repos, because that would effectively prevent
> maintainers
> > from pushing into -testing. They would need to wait until that
> package
> > gets into stable updates in more recent Fedora releases, and we
> don't
> > want that.
>
>
> > So the solution is not to schedule upgradepath for these update
> requests
> > at all (at least until we have more complex upgradepath). That will
> fix
> > the assertion error we had received.
>
> Sorry Kamil, I'm in need of clarification. Isn't 'upgradepath'
> listed
> as a mandatory test [1] for any updates entering 'updates-testing'?
> I
> feel like we're missing out if we don't run the test when a package
> is
> proposed for updates-testing.
>
> Re-reading the criteria [2], do I understand correctly that
> upgradepath
> must PASS in order to be pushed into 'stable'? If that's true, I
> think
> I better understand the open question now. If the result of
> upgradepath
> would only prevent a package from being pushed to stable, why bother
> running it for 'updates-testing'? Is that correct?
We have decided (there are some mails in this conference about it some
month ago) that updates-testing repo causes too many problems and we
won't implement checks for it, at least for now. So we check upgradepath
constraint only for main+updates repos.
Example use case showing problems with updates-testing:
1. In all repos there is foo-2.0.
2. I want to push foo-3.0 into f13-updates-testing.
3. Currently upgradepath reports FAILED, because neither f14 nor
f14-updates contain foo >= 3.0.
4. The only way in this case would be:
* push into rawhide
* push into f14-updates-testing, *wait* until it goes to f14-updates
* push into f13-updates-testing
Which is kind of PITA, because maintainers want to push the package
into ALL *-updates-testing immediately, without waiting several weeks
until it propagates top to bottom.
The result is that we completely ignore -updates-testing for now. We
don't check it for upgradepath constraint and we also don't want to check
any proposed updates for this repo. We check only builds proposed
for main or -updates repo.
That means we will ensure upgradepath constraint for common users (using
main+updates repos), but not for power users (using -updates-testing).
Does it make sense? It's a little late in here, I might be rambling a bit ;)
I hope I haven't confused anything.
13 years, 6 months
Re: repoinfo.conf and older releases
by Kamil Paral
----- "James Laska" <jlaska(a)redhat.com> wrote:
> > What is the reason to keep repos of unmaintained released in
> repoinfo.conf?
> > I can't find any usage of it. Currently I would just remove all
> unmaintained
> > releases and remove 'isactiverelease' keyword as well.
>
> Hmm, yeah. I don't know why I like keeping the old entries around.
> But
> it does seem silly to continue with old entries that don't work.
> Let's
> just drop f10 and f11 from repoinfo.conf.
If we don't have other uses for it, I would drop it. We have the history
in git, it can be easily added back again when needed in the future.
>
> > By the way, 'isactiverelease' variable confuses me anyway. Why is
> Branched
> > active, but Rawhide inactive? I don't understand that.
>
> While it's not 100% clear from the mail thread or the commit log on
> this
> subject, I understand that the 'isactiverelease' was added to
> indicate
> what entries has install images available. Wwoods can confirm.
> Iirc,
> post-tree-compose was failing because install images are no longer
> built
> and provided for rawhide (and for EOL'd releases [1]). According to
> the
> comment in repoinfo.py,
>
> getreleases() - '''Return the list of known, non-EOL
> releases.'''
>
> Using that definition, Rawhide isn't a Fedora release as installation
> images are not provided. Rawhide is just a repository of packages.
> I
> wonder if it makes sense to rename 'isactiverelease' to
> 'isinstallable'?
Ah, in that case it makes sense. The variable name could really be
improved to be clearer.
If the variable exists just for this one use case, we can also avoid
it and just disable test scheduling for post-tree-compose tests
for rawhide (hardcoded in control.autoqa). Whatever you like.
13 years, 6 months
[AutoQA] #229: Create per-update file conflicts test
by fedora-badges
#229: Create per-update file conflicts test
-------------------------+--------------------------------------------------
Reporter: jlaska | Owner:
Type: enhancement | Status: new
Priority: major | Milestone: Future test cases
Component: tests | Version: 1.0
Keywords: |
-------------------------+--------------------------------------------------
The conflicts test (appropriately named ''potential_conflicts.py'') uses
yum file lists to make an educated guess as to what file conflicts may be
present. Also, this script operates over the '''entire''' repository, not
just proposed updates. Note, that when installing packages directly on a
system, yum relies on rpm logic to determine whether the transaction would
result in file conflicts.
Similar in behavior to the depcheck test, we need a new test that takes
the following inputs:
1) Proposed package update(s)
2) Desired repo (e.g. dist-f14-updates-testing)
Based on those inputs, the script would determine whether file conflicts
would be introduced by the proposed package update(s). The script would
return a list of package updates that could be successfully added to the
requested repo without introducing file conflicts.
--
Ticket URL: <https://fedorahosted.org/autoqa/ticket/229>
AutoQA <http://autoqa.fedorahosted.org>
Automated QA project
13 years, 6 months
[AutoQA] #202: testcases for depcheck
by fedora-badges
#202: testcases for depcheck
--------------------+-------------------------------------------------------
Reporter: wwoods | Owner:
Type: task | Status: new
Priority: major | Milestone: Package Update Acceptance Test Plan - depcheck
Component: tests | Version: 1.0
Keywords: |
--------------------+-------------------------------------------------------
To be sure that depcheck is operating as expected, we should have a
library of test cases for it.
These test cases could take the form of a set (or sets) of trivial RPMs
that illustrate various possible dependency problems - or a script that
generates RPMs like that.
--
Ticket URL: <https://fedorahosted.org/autoqa/ticket/202>
AutoQA <http://autoqa.fedorahosted.org>
Automated QA project
13 years, 6 months
2010-09-24 - AutoQA depcheck meeting recap
by James Laska
# AutoQA 'Depcheck'-in
# Date: 2010-09-24
# Time: 14:00 UTC (10:00 EDT, 16:00 CEST)
# Attendees: jlaska, jskladan, kparal, mkrizek, wwoods
Depcheck - 'next steps' -
https://fedorahosted.org/pipermail/autoqa-devel/2010-August/001010.html
= Agenda =
1) Review current state of depcheck
* autoqa/tests/depcheck/depcheck available
> inputs:
+ a list of local packages,
+ name of a repo you want to move these packages to
> outputs:
+ list of the packages that could be successfully added to the
requested repo
* autoqa/tests/depcheck/depcheck unittests
> TODO - a few remaining unittests (fileconflicts?)
* mash script to create proper multilib setup available
* post-bodhi-update working now to submit upgradepath tests
2) Next steps
* ticket#201 Add mash support to depcheck
> next - integrate mash script into depcheck test
* ticket#202 testcases for depcheck
> next - close out (remaining conflicts test will be part of a new
ticket)
* ticket#204 add depcheck to post-bodhi-update hook
> includes autoqa integration (control and control.autoqa files
etc...)
> next - not too difficult to enable, this is likely the last ticket
* ticket#205 make depcheck submit karma/comments to bodhi
> will monitor depcheck results to determine correctness
> next - ideal solution, submit 1 karma for N tests
> next - if unable to determine karma submission for multiple tests,
will consider karma for depcheck
* Revise post-bodhi-update watcher to use new -pending koji tags (more
robust than current watcher implementation)
> not a must have for current milestone, but will be an issue in the
future
* new rpmguard test - updates don't drop architecture
> FIXME - unclear how to proceed
* documentation - Purpose, Basic depcheck use cases
> For example, see
https://fedoraproject.org/wiki/Updates_Lessons#2010-05-27_-_nss-softokn_u...
> document the purpose for each of the unittests
= Questions =
Q. Can depcheck test be run stand-alone?
A. Yeah, it's possible to run ./depcheck stand-alone
Q. When should we enable depcheck enforcing?
A. Ideally, after resultsdb is completed and all package update tests
are reporting into resultsdb. However, as a fallback, if timing becomes
an issue we will consider enabling karma submission for just depcheck
Q. What's the future of Conflicts and Reposanity tests in PUATP as
Mandatory tests?
A. Reposanity will be replaced by Depcheck and Conflicts will be
replaced by a new 'fileconflicts' test.
Q. What happens if a new update request comes in after depcheck has been
scheduled?
A. New updates will be tested when the watcher fires next.
= Action items =
* [kparal] - Documentation - Merge/create
https://fedoraproject.org/wiki/QA:Depcheck_Test_Case
* [wwoods] - Documentation - make sure there's some documentation
what depcheck is good for and how to use it generally (bullets
with stuff that depcheck exactly checks) README
* [future] - Add a new fileconflicts test that operates over a
list of packages, and a desired repo
* [wwoods] - ticket#201 - integrate jskladan's mash script into
depcheck
* [jlaska] - Move ticket#205 - move into "Package Update
Acceptance Test Plan"
* [jlaska] - Create ticket - revised post-bodhi-update that uses
-pending koji tags instead
* [jskladan] - summarize current state of resultsdb
* [jskladan] - review resultdb roadmap and submitting karma for
PUATP
13 years, 6 months
Telephone meeting on friday - Depcheck
by Josef Skladanka
Hello gang!
I'd like to propose a meeting for friday. I think, that we should
discuss the current state of the Depcheck test, and the next steps
required to finishing this task.
Is Friday OK? Is there any preferred time slot for you?
Thanks
Joza
13 years, 7 months
[PATCH] upgradepath doesn't support -testing repos yet, disable scheduling
by Kamil Paral
This patch kind of reverts fbe9ee91b. We don't want upgradepath to be run
for -testing repos, because that would effectively prevent maintainers
from pushing into -testing. They would need to wait until that package
gets into stable updates in more recent Fedora releases, and we don't
want that.
So the solution is not to schedule upgradepath for these update requests
at all (at least until we have more complex upgradepath). That will fix
the assertion error we had received.
I have one more concern with upgradepath, but I'll create a ticket for
it tomorrow. This patch should work fine.
Kamil Páral (1):
upgradepath doesn't support -testing repos yet, disable scheduling
tests/upgradepath/control.autoqa | 4 ++++
tests/upgradepath/upgradepath.py | 9 ++++-----
2 files changed, 8 insertions(+), 5 deletions(-)
--
1.7.2.3
13 years, 7 months
Re: repoinfo.conf and older releases
by Kamil Paral
----- "James Laska" <jlaska(a)redhat.com> wrote:
> Greetings,
>
> Currently, we don't remove old releases from repoinfo.conf. However,
> we
> do mark some of them (only the dist-fXX entries) as
> 'isactiverelease=no'. This affects some test results in that we
> don't
> have a way to determine what releases are no longer needed for
> testing.
> Specifically, this affects the post-repo-update watcher, and the
> upgradepath test.
>
> For example with upgradepath, we are testing a proposed
> f13-updates-testing package against everything from dist-f10 to
> dist-f15
> (see sample result [1]). While the results from older unmaintained
> releases may be interesting, they shouldn't affect the test outcome.
> Some options to address this ...
>
> 1. Use isactiverelease more - In a recent commit
> (6b3ae44dee122b030bcede1a2059a7533ee8203f), we introduced the
> 'isactiverelease' repoinfo.conf option. This is currently
> only
> used on the unmaintained dist-fNN entires. I'd propose we
> use
> this for all unmaintained repoinfo.conf entries (includes
> -updates and -updates-testing) *and* updating any tests or
> scripts to honor this option.
> 2. Remove (or comment out) unmaintained releases - Certainly an
> option, but my preference is #1 for some reason. But if
> folks
> overwhelmingly feel that just removing older entries is
> ideal ... I can't think of any objections.
>
> Thoughts/concerns/comments?
What is the reason to keep repos of unmaintained released in repoinfo.conf?
I can't find any usage of it. Currently I would just remove all unmaintained
releases and remove 'isactiverelease' keyword as well.
By the way, 'isactiverelease' variable confuses me anyway. Why is Branched
active, but Rawhide inactive? I don't understand that.
We're mixing repos and releases in our repoinfo.conf file. For me it seems
that two different semantics are intertwined in a single file. Not good.
13 years, 7 months
