Freeze Break Request: new depcheck on taskotron clients
Tim Flink
tflink at redhat.com
Wed Nov 19 20:57:15 UTC 2014
I thought that I had updated the depcheck task on production taskotron
a while back but it turns out that never happened and depcheck is
hitting failures in production that it shouldn't be hitting. Original
issue is:
https://phab.qadevel.cloud.fedoraproject.org/T366
The fix was reviewed before updating dev/stg and has been running
there more than 2 weeks now without issue.
I'd like to update the depcheck task in production to fix some
recurring issues that some devs have been complaining about. Diff is at
the end of this email.
Tim
diff --git a/depcheck/__init__.py b/depcheck/__init__.py
index ccac8a4..609eb13 100755
--- a/depcheck/__init__.py
+++ b/depcheck/__init__.py
@@ -36,6 +36,7 @@ import sys
import argparse
import tempfile
import platform
+import logging
from . import depcheck
from . import metadata
@@ -91,7 +92,7 @@ def run(rpms, arch, repos, report_format="updates",
workdir=None): arch = 'i686'
# handle data formatting for repo metadata (list of dicts)
- log.logger.debug("Prepairing metadata download")
+ log.logger.debug("Preparing metadata download")
handled_repos = {}
for repo, path in repos.items():
log.logger.debug(" - %r: %r, %r", repo, arch, path)
@@ -121,7 +122,7 @@ def run(rpms, arch, repos, report_format="updates",
workdir=None):
# Sometimes, depcheck gets run with no rpms
if not run_result:
- log.logger.info("No results")
+ log.logger.warn("No results")
output = squash_results.format_output(run_result, arch,
report_format) log.logger.info(output)
@@ -131,6 +132,7 @@ def run(rpms, arch, repos, report_format="updates",
workdir=None): def depcheck_cli():
parser = argparse.ArgumentParser()
parser.add_argument("rpm", nargs='+', help="RPMs to be tested")
+ parser.add_argument("-d", "--debug", action="store_true",
help="enable debug logging for depcheck") parser.add_argument("-a",
"--arch", choices=["i386", "x86_64", "armhfp"], help=""" Architecture
to be tested. If omitted, defaults to the current machine's
architecture. @@ -143,9 +145,17 @@ def depcheck_cli():
parser.add_argument("-f", "--format", choices=["rpms", "updates"],
help=""" Specify output format to override configuration.
""")
-
+ parser.add_argument("-w", "--workdir", help="workdir to use for
depcheck")
args = parser.parse_args()
+
+ # initialize logging when entering depcheck via cli - this won't
be executed
+ # otherwise but in that case, taskotron is responsible for
initialization
+ logging.basicConfig()
+
+ if args.debug:
+ log.logger.setLevel(logging.DEBUG)
+
# autodetect arch if none is passed in
arch = args.arch
if arch is None:
@@ -155,8 +165,8 @@ def depcheck_cli():
repos = dict([(repo, repo) for repo in args.repos])
try:
- print run(args.rpm, arch, repos, args.format)
- except KeyboardInterrupt, e:
+ run(args.rpm, arch, repos, args.format, args.workdir)
+ except KeyboardInterrupt:
print >> sys.stderr, ("\n\nExiting on user cancel.")
sys.exit(1)
diff --git a/depcheck/depcheck.py b/depcheck/depcheck.py
index bb0b37a..e5ba15c 100755
--- a/depcheck/depcheck.py
+++ b/depcheck/depcheck.py
@@ -30,7 +30,6 @@ import solv
import tempfile
from . import Pass, Fail
-from . import exc
from . import log
@@ -77,6 +76,8 @@ class Depcheck(object):
checking dependencies for the supplied repositories
"""
+ log.logger.debug('preparing libsolv repos for depcheck run')
+
for repo in self.repos:
primary = open(self.repos[repo]['primary'], 'rb')
filelist = open(self.repos[repo]['filelists'], 'rb')
@@ -85,37 +86,36 @@ class Depcheck(object):
self.solvrepo.add_rpmmd(filelist, None,
solv.Repo.REPO_EXTEND_SOLVABLES)
- def _check_rpm(self, rpmfile):
+ def _check_rpm(self, rpmfile, solvable):
"""Check a single rpm file against prepared repodata
:param rpmfile: filename of rpm file to check
:type rpmfile: str
+ :param solvable: libsolv solvable from solvpool for the rpm
to check
+ :type solvable: libsolv.Solver.Solvable
"""
- solvable = self.solvrepo.add_rpm(rpmfile)
- self.solvpool.addfileprovides()
- self.solvpool.createwhatprovides()
+ # the job setup here is mostly taken from libsolv's
installcheck tool
+ # create a job using the predetermined solvable in the repo
job = self.solvpool.Job(solv.Job.SOLVER_INSTALL |
solv.Job.SOLVER_SOLVABLE,
- solvable.id)
+ solvable.id)
+
self.solvpool.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, 1)
- # important note - make sure to keep the solver in the same
scope as
- # the problem analysis code or else python-solv segfults will
ensue
- solvpool_solver = self.solvpool.Solver()
- solv_problems = solvpool_solver.solve([job])
+ # solve the deps for the input solvable's rpm and look for
problems
+ solver = self.solvpool.Solver()
+ solv_problems = solver.solve([job])
if len(solv_problems) == 0:
- return Pass, []
-
+ return {'result': Pass,'details':[]}
else:
problem_strs = []
for problem in solv_problems:
probrules = problem.findallproblemrules()
-
for rule in probrules:
ruleinfos = rule.allinfos()
for info in ruleinfos:
problem_strs.append(info.problemstr())
+ return {'result':Fail, 'details':problem_strs}
- return Fail, problem_strs
def check_rpm_deps(self, rpmfiles):
"""Check dependencies of the given rpms against supplied
repodata @@ -125,26 +125,33 @@ class Depcheck(object):
:returns: dictionary of results {rpm: {'results': Pass/Fail,
'details': []}} """
- self._prepare_libsolv()
+ log.logger.debug('adding all rpms to libsolv repos and
preparing solvpool')
- run_result = {}
+ # add all rpmfiles that we're checking to the libsolv repo and
save the
+ # solvable that is assigned to them.
+ # this makes libsolv aware of all the incoming rpms and allows
it to
+ # check for any dep breakage between the rpms
+ self._prepare_libsolv()
+ solvables = {}
for rpmfile in rpmfiles:
- log.logger.debug('running depcheck for %s' % rpmfile)
- result_state, result_details = self._check_rpm(rpmfile)
+ rpmname = rpmfile.split('/')[-1]
+ solvable = self.solvrepo.add_rpm(rpmfile)
+ solvables[rpmname] = solvable
- if rpmfile in run_result:
- if result_state != run_result[rpmfile]['result']:
- #TODO maybe do something saner, ideas welcomed
- raise exc.DepcheckException("RPM %r has multiple
conflicting results" % rpmfile)
- else:
-
run_result[rpmfile]['details'].append(result_details)
+ # now that the rpms have been added, prepare the solvpool
+ self.solvpool.addfileprovides()
+ self.solvpool.createwhatprovides()
- else:
- run_result[rpmfile] = {'result': result_state,
- 'details': result_details}
- return run_result
+ # check each rpm using its solvable that was assigned while
adding to
+ # the repo
+ results = {}
+ for rpmname in solvables:
+ log.logger.debug('running depcheck for %s' % rpmname)
+ run_result = self._check_rpm(rpmname, solvables[rpmname])
+ results[rpmname] = run_result
+ return results
diff --git a/depcheck/squash_results.py b/depcheck/squash_results.py
index 7376f0d..30330ca 100644
--- a/depcheck/squash_results.py
+++ b/depcheck/squash_results.py
@@ -1,4 +1,3 @@
-import os
import json
from . import Pass, Fail
@@ -7,7 +6,6 @@ from . import log
from libtaskotron.koji_utils import KojiClient
from libtaskotron.bodhi_utils import BodhiUtils
from libtaskotron import check
-from libtaskotron.rpm_utils import rpmformat
def _squash_rpms_to_builds(run_result, koji_client = None):
"""
diff --git a/tests/depcheck_scenarios b/tests/depcheck_scenarios
index f1e5d57..3ad160a 160000
--- a/tests/depcheck_scenarios
+++ b/tests/depcheck_scenarios
@@ -1 +1 @@
-Subproject commit f1e5d57b0e68c53bd05c1b4d6eab3078706d269e
+Subproject commit 3ad160ad3145dc4595286291e5e43e2d91737ea9
diff --git a/tests/functest_depcheck_scenarios.py
b/tests/functest_depcheck_scenarios.py index dc69e07..3f5733d 100644
--- a/tests/functest_depcheck_scenarios.py
+++ b/tests/functest_depcheck_scenarios.py
@@ -36,9 +36,12 @@ def get_depcheck_scenarios():
for dircontent in dircontents:
if len(dircontent[2]) == 0:
continue
+ if dircontent[1] == 'known_broken':
+ continue
for infile in dircontent[2]:
filename = '%s/%s' % (dircontent[0], infile)
- yamlfiles.append((scenario_type == 'pass', filename))
+ if infile.split('.')[-1] in ['yml', 'yaml']:
+ yamlfiles.append((scenario_type == 'pass',
filename)) return yamlfiles
diff --git a/tests/test_depcheck.py b/tests/test_depcheck.py
index d27d011..909b2ad 100644
--- a/tests/test_depcheck.py
+++ b/tests/test_depcheck.py
@@ -36,10 +36,10 @@ class TestDepcheck(object):
monkeypatch.setattr(test_depcheck, '_prepare_libsolv',
Dingus())
rpmfiles = ['rpm_1', 'rpm_2']
- def fake_check_rpm(rpmfile):
+ def fake_check_rpm(rpmfile, solvable):
results = {
- rpmfiles[0]: (Pass, []),
- rpmfiles[1]: (Fail, ['message']),
+ rpmfiles[0]: {'result':Pass, 'details':[]},
+ rpmfiles[1]: {'result':Fail,
'details':['message']}, }
return results.get(rpmfile, (Pass, []))
@@ -54,25 +54,3 @@ class TestDepcheck(object):
assert outcome['rpm_2']['result'] == Fail
assert outcome['rpm_2']['details'] == ['message']
- def test_check_rpm_deps_conflicting_results(self, monkeypatch):
- """
- Checks whether rpm_deps raises Exceptions when conflicting
- results are provided for one RPM.
- """
-
- test_depcheck = depcheck.Depcheck(None, None, "fakedir")
- monkeypatch.setattr(test_depcheck, '_prepare_libsolv',
Dingus()) -
- self.tmp_result = Pass
- def fake_check_rpm(rpmfile):
- if self.tmp_result == Pass:
- self.tmp_result = Fail
- else:
- self.tmp_result = Pass
- return (self.tmp_result, [])
-
- monkeypatch.setattr(test_depcheck, '_check_rpm',
fake_check_rpm) -
- with pytest.raises(exc.DepcheckException):
- outcome = test_depcheck.check_rpm_deps(['rpm1', 'rpm1'])
-
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 473 bytes
Desc: OpenPGP digital signature
URL: <http://lists.fedoraproject.org/pipermail/infrastructure/attachments/20141119/014a28d3/attachment.sig>
More information about the infrastructure
mailing list