[PATCH] mergeScratch(): import rpms from a scratch build into an existing build
Mike Bonnet
mikeb at redhat.com
Wed Apr 22 21:56:22 UTC 2015
Hi. The attached patch implements a mergeScratch() method. This
enables importing the rpms built by a scratch build and associating them
with an existing build, if that build was built from the same SCM URL
and has the same NVR.
The use-case for the method is arch enablement. The general approach
would be:
- setup a new build tag that builds only for the new arch
- this would use either imported rpms or an external repo of rpms
that had been built/bootstrapped manually
- run a scratch build in the new build tag, from the same SCM URL as
an existing build
- call mergeScratch(scratch_task_id)
- rpms built for the new arch are added to the existing build
This allows arch coverage to be added incrementally to existing builds
in a non-disruptive way. When all builds in the -build tag have rpms
for the new arch, the arch can be enabled in the main tags and all new
builds will generate rpms for the new arch. This simplifies the
bootstrap process for a new arch, and avoids mass-rebuilds that would
result in no change in the existing arch rpms.
This could be accomplished with the existing rpm import functionality,
but mergeScratch() retains the build logs and the buildroot metadata
(associated with the scratch build task) so merged rpms have better
tracking and auditability.
Questions and comments welcome.
Thanks,
Mike
-------------- next part --------------
From 04681857aeced8100b9e2251bae8a4d3f212aa38 Mon Sep 17 00:00:00 2001
From: Mike Bonnet <mikeb at redhat.com>
Date: Wed, 22 Apr 2015 16:06:48 -0400
Subject: [PATCH] mergeScratch(): import rpms from a scratch build into an
existing build
The mergeScratch() method allows importing rpms built by a scratch build into an existing build, if that build
did not produce rpms matching the arch of the scratch build. This is useful for bootstrapping a new arch into
existing builds, and avoiding a mass-rebuild to add arch support.
---
hub/kojihub.py | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
koji/__init__.py | 3 ++
2 files changed, 144 insertions(+)
diff --git a/hub/kojihub.py b/hub/kojihub.py
index 9decf7c..d11ea7d 100644
--- a/hub/kojihub.py
+++ b/hub/kojihub.py
@@ -4720,6 +4720,117 @@ def _import_wrapper(task_id, build_info, rpm_results):
import_build_log(os.path.join(rpm_task_dir, log),
build_info, subdir='noarch')
+def merge_scratch(task_id):
+ """Import rpms from a scratch build into an existing build, retaining
+ buildroot metadata and build logs."""
+ task = Task(task_id)
+ try:
+ task_info = task.getInfo(request=True)
+ except koji.GenericError:
+ raise koji.ImportError, 'invalid task: %s' % task_id
+ if task_info['state'] != koji.TASK_STATES['CLOSED']:
+ raise koji.ImportError, 'task %s did not complete successfully' % task_id
+ if task_info['method'] != 'build':
+ raise koji.ImportError, 'task %s is not a build task' % task_id
+ if len(task_info['request']) < 3 or not task_info['request'][2].get('scratch'):
+ raise koji.ImportError, 'task %s is not a scratch build' % task_id
+
+ # sanity check the task, and extract data required for import
+ srpm = None
+ tasks = {}
+ for child in task.getChildren():
+ if child['method'] != 'buildArch':
+ continue
+ info = {'rpms': [],
+ 'logs': []}
+ for output in list_task_output(child['id']):
+ if output.endswith('.src.rpm'):
+ srpm_name = os.path.basename(output)
+ if not srpm:
+ srpm = srpm_name
+ else:
+ if srpm != srpm_name:
+ raise koji.ImportError, 'task srpm names do not match: %s, %s' % \
+ (srpm, srpm_name)
+ elif output.endswith('.noarch.rpm'):
+ continue
+ elif output.endswith('.rpm'):
+ rpminfo = koji.parse_NVRA(os.path.basename(output))
+ if 'arch' not in info:
+ info['arch'] = rpminfo['arch']
+ elif info['arch'] != rpminfo['arch']:
+ raise koji.ImportError, 'multiple arches generated by task %s: %s, %s' % \
+ (child['id'], info['arch'], rpminfo['arch'])
+ info['rpms'].append(output)
+ elif output.endswith('.log'):
+ info['logs'].append(output)
+ if not info['rpms']:
+ raise koji.ImportError, 'no arch-specific rpms produced by task %s' % child['id']
+ if not info['logs']:
+ raise koji.ImportError, 'task %s is missing logs' % child['id']
+ buildroots = query_buildroots(taskID=child['id'],
+ queryOpts={'order': '-id', 'limit': 1})
+ if not buildroots:
+ raise koji.ImportError, 'no buildroot associated with task %s' % child['id']
+ info['buildroot_id'] = buildroots[0]['id']
+ tasks[child['id']] = info
+
+ # sanity check the build
+ build_nvr = koji.parse_NVRA(srpm)
+ build = get_build(build_nvr)
+ if not build:
+ raise koji.ImportError, 'no such build: %(name)s-%(version)s-%(release)s' % \
+ build_nvr
+ if build['state'] != koji.BUILD_STATES['COMPLETE']:
+ raise koji.ImportError, '%s did not complete successfully' % build['nvr']
+ if not build['task_id']:
+ raise koji.ImportError, 'no task for %s' % build['nvr']
+ build_task_info = Task(build['task_id']).getInfo(request=True)
+ # Intentionally skip checking the build task state.
+ # There are cases where the build can be valid even though the task has failed,
+ # e.g. tagging failures.
+
+ # compare the task and build and make sure they are compatible with importing
+ if task_info['request'][0] != build_task_info['request'][0]:
+ raise koji.ImportError, 'SCM URLs for the task and build do not match: %s, %s' % \
+ (task_info['request'][0], build_task_info['request'][0])
+ build_arches = set()
+ for rpm in list_rpms(buildID=build['id']):
+ if rpm['arch'] == 'src':
+ build_srpm = '%s.src.rpm' % rpm['nvr']
+ if srpm != build_srpm:
+ raise koji.ImportError, 'task and build srpm names do not match: %s, %s' % \
+ (srpm, build_srpm)
+ elif rpm['arch'] == 'noarch':
+ continue
+ else:
+ build_arches.add(rpm['arch'])
+ if not build_arches:
+ raise koji.ImportError, 'no arch-specific rpms found for %s' % build['nvr']
+ task_arches = set([t['arch'] for t in tasks.values()])
+ overlapping_arches = task_arches.intersection(build_arches)
+ if overlapping_arches:
+ raise koji.ImportError, 'task %s and %s produce rpms with the same arches: %s' % \
+ (task_info['id'], build['nvr'], ', '.join(overlapping_arches))
+
+ # everything looks good, do the import
+ for task_id, info in tasks.items():
+ taskpath = koji.pathinfo.task(task_id)
+ for filename in info['rpms']:
+ filepath = os.path.realpath(os.path.join(taskpath, filename))
+ rpminfo = import_rpm(filepath, build, info['buildroot_id'])
+ import_rpm_file(filepath, build, rpminfo)
+ add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(filepath))
+ for logname in info['logs']:
+ logpath = os.path.realpath(os.path.join(taskpath, logname))
+ import_build_log(logpath, build, subdir=info['arch'])
+
+ # flag tags whose content has changed, so relevant repos can be regen'ed
+ for tag in list_tags(build=build['id']):
+ set_tag_update(tag['id'], 'IMPORT')
+
+ return build['id']
+
def get_archive_types():
"""Return a list of all supported archivetypes"""
select = """SELECT id, name, description, extensions FROM archivetypes
@@ -7647,6 +7758,36 @@ class RootExports(object):
for tag in list_tags(build=rpminfo['build_id']):
set_tag_update(tag['id'], 'IMPORT')
+ def mergeScratch(self, task_id):
+ """Import the rpms generated by a scratch build, and associate
+ them with an existing build.
+
+ To be eligible for import, the build must:
+ - be successfully completed
+ - contain at least one arch-specific rpm
+
+ The task must:
+ - be a 'build' task
+ - be successfully completed
+ - use the exact same SCM URL as the build
+ - contain at least one arch-specific rpm
+ - have no overlap between the arches of the rpms it contains and
+ the rpms contained by the build
+ - contain a .src.rpm whose filename exactly matches the .src.rpm
+ of the build
+
+ Only arch-specific rpms will be imported. noarch rpms and the src
+ rpm will be skipped. Build logs and buildroot metadata from the
+ scratch build will be imported along with the rpms.
+
+ This is useful for bootstrapping a new arch. RPMs can be built
+ for the new arch using a scratch build and then merged into an
+ existing build, incrementally expanding arch coverage and avoiding
+ the need for a mass-rebuild to support the new arch.
+ """
+ context.session.assertPerm('admin')
+ return merge_scratch(task_id)
+
def addExternalRPM(self, rpminfo, external_repo, strict=True):
"""Import an external RPM
diff --git a/koji/__init__.py b/koji/__init__.py
index d133ed1..c162e7e 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -323,6 +323,9 @@ class ParameterError(GenericError):
"""Raised when an rpc call receives incorrect arguments"""
faultCode = 1019
+class ImportError(GenericError):
+ """Raised when an import fails"""
+ faultCode = 1020
class MultiCallInProgress(object):
"""
--
1.8.2.1
More information about the buildsys
mailing list