autoqa-devel September 2010

autoqa-devel@lists.fedorahosted.org

8 participants
88 discussions

[PATCH] tidy up upgradepath test code
by Kamil Paral 30 Sep '10

30 Sep '10

This patch introduces no new features and (hopefully) no new bugs. It just reformats the code to be more readable and maintainable - now the semantics and variables are very similar to those used in rpmlint and rpmguard tests, so it's easy to read through the code. --- tests/upgradepath/upgradepath.py | 99 ++++++++++++++++++++++++++------------ 1 files changed, 68 insertions(+), 31 deletions(-) diff --git a/tests/upgradepath/upgradepath.py b/tests/upgradepath/upgradepath.py index 028b8aa..bc382d3 100755 --- a/tests/upgradepath/upgradepath.py +++ b/tests/upgradepath/upgradepath.py @@ -36,52 +36,58 @@ from autotest_lib.client.bin.test_config import config_loader class upgradepath(AutoQATest): version = 1 # increment if setup() changes - test_result = 'PASS' - log = [] - envr_list = set() def initialize(self, config): self.config = config_loader(config, self.tmpdir) #URL of logs/results stored on autotest-server self.autotest_url = autoqa.util.make_autotest_url(self.config) + self.result = 'PASSED' + # order for evaluation of final result; higher index means preference + self.result_order = ('PASSED','INFO','FAILED','ABORTED') + self.envr_results = {} # results for invidual packages + self.outputs = [] + self.highlights = [] + self.log = open(os.path.join(self.resultsdir,'upgradepath.log'),'wb') # where to log output + def setup(self): pass def compare(self, matching_build, tags, op, opstr): koji = autoqa.koji_utils.SimpleKojiClientSession() + result = 'PASSED' for tag in tags: latest_build = koji.latest_by_tag(tag, matching_build['name']) if latest_build is None: msg ='{0:<7}{1}'.format('[ OK ]', tag) - self.log.append(msg) print msg + self.outputs.append(msg) msg = "\tNo such package: %s" % matching_build['name'] - self.log.append(msg) print msg + self.outputs.append(msg) else: - result = autoqa.koji_utils.compareEVR(matching_build, latest_build) + res = autoqa.koji_utils.compareEVR(matching_build, latest_build) - if op(result, 0): + if op(res, 0): ok = True msg ='{0:<7}{1}'.format('[ OK ]', tag) print msg - self.log.append(msg) + self.outputs.append(msg) else: ok = False - self.test_result = 'FAIL' msg ='{0:<7}{1}'.format('[FAIL]', tag) print msg - self.log.append(msg) - self.envr_list.add(matching_build['envr']) + self.outputs.append(msg) msg = "\tLatest package: %s" % latest_build['nvr'] - self.log.append(msg) print msg + self.outputs.append(msg) if not ok: msg = '\tFailed condition: Requested package %s Latest package' % opstr - self.log.append(msg) print msg + self.outputs.append(msg) + result = 'FAILED' + return result @ExceptionCatcher("self.run_once_failed") def run_once(self, envrs, kojitag, **kwargs): @@ -99,10 +105,12 @@ class upgradepath(AutoQATest): hi_tags = repotags[(current_tag + 1):] # tags higher than current low_eq_tags = repotags[:(current_tag + 1)] # tags lower or equal + koji = autoqa.koji_utils.SimpleKojiClientSession() + for envr in envrs: msg = '%s\n%s into %s\n%s' % (40*'=', envr, kojitag, 40*'=') print msg - self.log.append(msg) + self.outputs.append(msg) # our testing package (name, version, release, epoch, arch) = rpmUtils.miscutils.splitFilename(envr) @@ -114,34 +122,63 @@ class upgradepath(AutoQATest): 'envr' : envr, } - if kojitag.find('updates') == -1 and repotags[current_tag] != repotags[-1]: - msg = "Warning: Pushing into stable repository" - self.log.append(msg) + if kojitag.find('updates') < 0 and repotags[current_tag] != repotags[-1]: + # not *-updates* and not rawhide + # FIXME: don't warn about pushing into base repo for both rawhide and branched (but possibly only before change deadline?) + msg = "Warning: Pushing into the base 'fedora' repository" print msg + self.outputs.append(msg) + if msg not in self.highlights: + # this message would be repeated for every package, highlight only once + self.highlights.append(msg) - koji = autoqa.koji_utils.SimpleKojiClientSession() latest_build = koji.latest_by_tag(kojitag, matching_build['name']) if latest_build is not None: - result = autoqa.koji_utils.compareEVR(matching_build, latest_build) - if result <= 0: - msg = "Warning: Same or newer build already exists: %s" % latest_build['nvr'] - self.log.append(msg) + if autoqa.koji_utils.compareEVR(matching_build, latest_build) <= 0: + msg = "Warning: Same or newer build already exists in the same repo: %s" % latest_build['nvr'] print msg + self.outputs.append(msg) + self.highlights.append(msg) # compare with lower or equal tags, so version has to be greater or equal - self.compare(matching_build, low_eq_tags, operator.ge, '>=') + result = self.compare(matching_build, low_eq_tags, operator.ge, '>=') # compare with higher tags, so version has to be lower or equal - self.compare(matching_build, hi_tags, operator.le, '<=') + result2 = self.compare(matching_build, hi_tags, operator.le, '<=') + + # compute pkg result + if self.result_order.index(result2) > self.result_order.index(result): + result = result2 + self.envr_results[envr] = result + + msg = 'RESULT: %s' % result + print msg + self.outputs.append(msg) + + # update self.result + if self.result_order.index(result) > self.result_order.index(self.result): + self.result = result + + # create summary like "1 PASSED, 2 FAILED, 3 INFO" + summary = [] + for res in self.result_order: + if res in self.envr_results.values(): + count = len([k for k in self.envr_results.keys() if self.envr_results[k] == res]) + summary.append('%d %s' % (count, res)) + summary = ', '.join(summary) - summary = str(len(envrs) - len(self.envr_list)) + ' OK, ' + str(len(self.envr_list)) + ' FAILED' + # print summary msg = 'SUMMARY: ' + summary - self.log.append(msg) print msg + self.outputs.append(msg) + self.highlights.append(msg) - self.result = self.test_result - self.outputs = "" - for i in self.log: - self.outputs += i + '\n' - self.outputs += '\n \n' self.summary = '%s for %s' % (summary, update_id) + # reformat result variables + self.outputs = '\n'.join(self.outputs) + self.highlights = '\n'.join(self.highlights) + + # log outputs + self.log.write(self.outputs) + self.log.close() + -- 1.7.2.3

1 0

Re: [PATCH] upgradepath doesn't support -testing repos yet, disable scheduling
by Kamil Paral 29 Sep '10

29 Sep '10

----- "James Laska" <jlaska(a)redhat.com> wrote: > > Hey gang, > > I'm failing when looking into the list archives to review our > previous > conclusions on this topic. I know we talked about this before, but I > can't find the threads. Weird, I think we did discuss it, I even think of some ascii art showing possible problems. But I found just this, nothing more: https://fedorahosted.org/pipermail/autoqa-devel/2010-August/000982.html > > The more I think about the upgradepath test, the more I come up with > scenarios where I (as a maintainer) would want to know the > upgradepath > result for my updates in 'updates-testing'. I recognize there might > be > some weird policy issues here, perhaps we need to get clarification > on > this from FESCO? I have search through whole fedoraproject wiki and a lot of pages talk about upgrade path, but none defines it exactly. Therefore we could ask FESCo about exact definition, yes, that's probably a good idea (to have some solid background for what we do). The main question is whether we want to have upgrade path constraint fulfilled when dealing with updates-testing repositories and what exactly it means in this case. Does it mean upgrading from F12-updates-testing to F13-updates? Or does it mean upgrading from F12-updates-testing to F13-updates-testing? We don't know, and that's the problem. I'll write another email talking about the problems that may arise. > > I know there is also the potential that an update in > 'updates-testing' > fails during functional validation, and never lands in 'stable'. > Isn't > this scenario covered by also scheduling upgradepath for bodhi > 'updates' > requests? > > Sorry for coming back to this topic, I think I need some re-educating > on > what 'upgradepath' means. First of all, I'd like to have my patch accepted, if possible. That should put into a consistent state - 'updates' checked correctly, 'updates-testing' not checked at all. Right now we are in an inconsistent state. Also I have some more patches coming for upgradepath test, so pushing into master would save me some trouble. When we are in a consistent state, we can unleash the debate about the updates-testing problem :) In a following email I'll sum up the main problems that arise and what we can do about it. Since Will is now on PTO, do you think I can push this into master, James?

2 2

Re: [PATCH] upgradepath doesn't support -testing repos yet, disable scheduling
by Kamil Paral 28 Sep '10

28 Sep '10

----- "James Laska" <jlaska(a)redhat.com> wrote: > On Thu, 2010-09-23 at 13:04 +0200, Kamil Páral wrote: > > This patch kind of reverts fbe9ee91b. We don't want upgradepath to > be run > > for -testing repos, because that would effectively prevent > maintainers > > from pushing into -testing. They would need to wait until that > package > > gets into stable updates in more recent Fedora releases, and we > don't > > want that. > > > > So the solution is not to schedule upgradepath for these update > requests > > at all (at least until we have more complex upgradepath). That will > fix > > the assertion error we had received. > > Sorry Kamil, I'm in need of clarification. Isn't 'upgradepath' > listed > as a mandatory test [1] for any updates entering 'updates-testing'? > I > feel like we're missing out if we don't run the test when a package > is > proposed for updates-testing. > > Re-reading the criteria [2], do I understand correctly that > upgradepath > must PASS in order to be pushed into 'stable'? If that's true, I > think > I better understand the open question now. If the result of > upgradepath > would only prevent a package from being pushed to stable, why bother > running it for 'updates-testing'? Is that correct? We have decided (there are some mails in this conference about it some month ago) that updates-testing repo causes too many problems and we won't implement checks for it, at least for now. So we check upgradepath constraint only for main+updates repos. Example use case showing problems with updates-testing: 1. In all repos there is foo-2.0. 2. I want to push foo-3.0 into f13-updates-testing. 3. Currently upgradepath reports FAILED, because neither f14 nor f14-updates contain foo >= 3.0. 4. The only way in this case would be: * push into rawhide * push into f14-updates-testing, *wait* until it goes to f14-updates * push into f13-updates-testing Which is kind of PITA, because maintainers want to push the package into ALL *-updates-testing immediately, without waiting several weeks until it propagates top to bottom. The result is that we completely ignore -updates-testing for now. We don't check it for upgradepath constraint and we also don't want to check any proposed updates for this repo. We check only builds proposed for main or -updates repo. That means we will ensure upgradepath constraint for common users (using main+updates repos), but not for power users (using -updates-testing). Does it make sense? It's a little late in here, I might be rambling a bit ;) I hope I haven't confused anything.

2 2

Re: repoinfo.conf and older releases
by Kamil Paral 27 Sep '10

27 Sep '10

----- "James Laska" <jlaska(a)redhat.com> wrote: > > What is the reason to keep repos of unmaintained released in > repoinfo.conf? > > I can't find any usage of it. Currently I would just remove all > unmaintained > > releases and remove 'isactiverelease' keyword as well. > > Hmm, yeah. I don't know why I like keeping the old entries around. > But > it does seem silly to continue with old entries that don't work. > Let's > just drop f10 and f11 from repoinfo.conf. If we don't have other uses for it, I would drop it. We have the history in git, it can be easily added back again when needed in the future. > > > By the way, 'isactiverelease' variable confuses me anyway. Why is > Branched > > active, but Rawhide inactive? I don't understand that. > > While it's not 100% clear from the mail thread or the commit log on > this > subject, I understand that the 'isactiverelease' was added to > indicate > what entries has install images available. Wwoods can confirm. > Iirc, > post-tree-compose was failing because install images are no longer > built > and provided for rawhide (and for EOL'd releases [1]). According to > the > comment in repoinfo.py, > > getreleases() - '''Return the list of known, non-EOL > releases.''' > > Using that definition, Rawhide isn't a Fedora release as installation > images are not provided. Rawhide is just a repository of packages. > I > wonder if it makes sense to rename 'isactiverelease' to > 'isinstallable'? Ah, in that case it makes sense. The variable name could really be improved to be clearer. If the variable exists just for this one use case, we can also avoid it and just disable test scheduling for post-tree-compose tests for rawhide (hardcoded in control.autoqa). Whatever you like.

2 1

[AutoQA] #229: Create per-update file conflicts test
by fedora-badges 27 Sep '10

27 Sep '10

#229: Create per-update file conflicts test -------------------------+-------------------------------------------------- Reporter: jlaska | Owner: Type: enhancement | Status: new Priority: major | Milestone: Future test cases Component: tests | Version: 1.0 Keywords: | -------------------------+-------------------------------------------------- The conflicts test (appropriately named ''potential_conflicts.py'') uses yum file lists to make an educated guess as to what file conflicts may be present. Also, this script operates over the '''entire''' repository, not just proposed updates. Note, that when installing packages directly on a system, yum relies on rpm logic to determine whether the transaction would result in file conflicts. Similar in behavior to the depcheck test, we need a new test that takes the following inputs: 1) Proposed package update(s) 2) Desired repo (e.g. dist-f14-updates-testing) Based on those inputs, the script would determine whether file conflicts would be introduced by the proposed package update(s). The script would return a list of package updates that could be successfully added to the requested repo without introducing file conflicts. -- Ticket URL: <https://fedorahosted.org/autoqa/ticket/229> AutoQA <http://autoqa.fedorahosted.org> Automated QA project

1 0

[AutoQA] #202: testcases for depcheck
by fedora-badges 25 Sep '10

25 Sep '10

#202: testcases for depcheck --------------------+------------------------------------------------------- Reporter: wwoods | Owner: Type: task | Status: new Priority: major | Milestone: Package Update Acceptance Test Plan - depcheck Component: tests | Version: 1.0 Keywords: | --------------------+------------------------------------------------------- To be sure that depcheck is operating as expected, we should have a library of test cases for it. These test cases could take the form of a set (or sets) of trivial RPMs that illustrate various possible dependency problems - or a script that generates RPMs like that. -- Ticket URL: <https://fedorahosted.org/autoqa/ticket/202> AutoQA <http://autoqa.fedorahosted.org> Automated QA project

1 6

2010-09-24 - AutoQA depcheck meeting recap
by James Laska 24 Sep '10

24 Sep '10

# AutoQA 'Depcheck'-in # Date: 2010-09-24 # Time: 14:00 UTC (10:00 EDT, 16:00 CEST) # Attendees: jlaska, jskladan, kparal, mkrizek, wwoods Depcheck - 'next steps' - https://fedorahosted.org/pipermail/autoqa-devel/2010-August/001010.html = Agenda = 1) Review current state of depcheck * autoqa/tests/depcheck/depcheck available > inputs: + a list of local packages, + name of a repo you want to move these packages to > outputs: + list of the packages that could be successfully added to the requested repo * autoqa/tests/depcheck/depcheck unittests > TODO - a few remaining unittests (fileconflicts?) * mash script to create proper multilib setup available * post-bodhi-update working now to submit upgradepath tests 2) Next steps * ticket#201 Add mash support to depcheck > next - integrate mash script into depcheck test * ticket#202 testcases for depcheck > next - close out (remaining conflicts test will be part of a new ticket) * ticket#204 add depcheck to post-bodhi-update hook > includes autoqa integration (control and control.autoqa files etc...) > next - not too difficult to enable, this is likely the last ticket * ticket#205 make depcheck submit karma/comments to bodhi > will monitor depcheck results to determine correctness > next - ideal solution, submit 1 karma for N tests > next - if unable to determine karma submission for multiple tests, will consider karma for depcheck * Revise post-bodhi-update watcher to use new -pending koji tags (more robust than current watcher implementation) > not a must have for current milestone, but will be an issue in the future * new rpmguard test - updates don't drop architecture > FIXME - unclear how to proceed * documentation - Purpose, Basic depcheck use cases > For example, see https://fedoraproject.org/wiki/Updates_Lessons#2010-05-27_-_nss-softokn_upd… > document the purpose for each of the unittests = Questions = Q. Can depcheck test be run stand-alone? A. Yeah, it's possible to run ./depcheck stand-alone Q. When should we enable depcheck enforcing? A. Ideally, after resultsdb is completed and all package update tests are reporting into resultsdb. However, as a fallback, if timing becomes an issue we will consider enabling karma submission for just depcheck Q. What's the future of Conflicts and Reposanity tests in PUATP as Mandatory tests? A. Reposanity will be replaced by Depcheck and Conflicts will be replaced by a new 'fileconflicts' test. Q. What happens if a new update request comes in after depcheck has been scheduled? A. New updates will be tested when the watcher fires next. = Action items = * [kparal] - Documentation - Merge/create https://fedoraproject.org/wiki/QA:Depcheck_Test_Case * [wwoods] - Documentation - make sure there's some documentation what depcheck is good for and how to use it generally (bullets with stuff that depcheck exactly checks) README * [future] - Add a new fileconflicts test that operates over a list of packages, and a desired repo * [wwoods] - ticket#201 - integrate jskladan's mash script into depcheck * [jlaska] - Move ticket#205 - move into "Package Update Acceptance Test Plan" * [jlaska] - Create ticket - revised post-bodhi-update that uses -pending koji tags instead * [jskladan] - summarize current state of resultsdb * [jskladan] - review resultdb roadmap and submitting karma for PUATP

1 0

Telephone meeting on friday - Depcheck
by Josef Skladanka 23 Sep '10

23 Sep '10

Hello gang! I'd like to propose a meeting for friday. I think, that we should discuss the current state of the Depcheck test, and the next steps required to finishing this task. Is Friday OK? Is there any preferred time slot for you? Thanks Joza

3 3

[PATCH] upgradepath doesn't support -testing repos yet, disable scheduling
by Kamil Paral 23 Sep '10

23 Sep '10

This patch kind of reverts fbe9ee91b. We don't want upgradepath to be run for -testing repos, because that would effectively prevent maintainers from pushing into -testing. They would need to wait until that package gets into stable updates in more recent Fedora releases, and we don't want that. So the solution is not to schedule upgradepath for these update requests at all (at least until we have more complex upgradepath). That will fix the assertion error we had received. I have one more concern with upgradepath, but I'll create a ticket for it tomorrow. This patch should work fine. Kamil Páral (1): upgradepath doesn't support -testing repos yet, disable scheduling tests/upgradepath/control.autoqa | 4 ++++ tests/upgradepath/upgradepath.py | 9 ++++----- 2 files changed, 8 insertions(+), 5 deletions(-) -- 1.7.2.3

2 2

Re: repoinfo.conf and older releases
by Kamil Paral 23 Sep '10

23 Sep '10

----- "James Laska" <jlaska(a)redhat.com> wrote: > Greetings, > > Currently, we don't remove old releases from repoinfo.conf. However, > we > do mark some of them (only the dist-fXX entries) as > 'isactiverelease=no'. This affects some test results in that we > don't > have a way to determine what releases are no longer needed for > testing. > Specifically, this affects the post-repo-update watcher, and the > upgradepath test. > > For example with upgradepath, we are testing a proposed > f13-updates-testing package against everything from dist-f10 to > dist-f15 > (see sample result [1]). While the results from older unmaintained > releases may be interesting, they shouldn't affect the test outcome. > Some options to address this ... > > 1. Use isactiverelease more - In a recent commit > (6b3ae44dee122b030bcede1a2059a7533ee8203f), we introduced the > 'isactiverelease' repoinfo.conf option. This is currently > only > used on the unmaintained dist-fNN entires. I'd propose we > use > this for all unmaintained repoinfo.conf entries (includes > -updates and -updates-testing) *and* updating any tests or > scripts to honor this option. > 2. Remove (or comment out) unmaintained releases - Certainly an > option, but my preference is #1 for some reason. But if > folks > overwhelmingly feel that just removing older entries is > ideal ... I can't think of any objections. > > Thoughts/concerns/comments? What is the reason to keep repos of unmaintained released in repoinfo.conf? I can't find any usage of it. Currently I would just remove all unmaintained releases and remove 'isactiverelease' keyword as well. By the way, 'isactiverelease' variable confuses me anyway. Why is Branched active, but Rawhide inactive? I don't understand that. We're mixing repos and releases in our repoinfo.conf file. For me it seems that two different semantics are intertwined in a single file. Not good.

2 1

← Newer
1
2
3
4
5
6
7
8
9
Older →

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

autoqa-devel September 2010