[Patch] show top blocking failures in koji-shadow
by Karsten Hopp
Attached is a patch that changes the output of koji-shadow. Instead of reporting the most
frequent problem deps the output now shows the build failures that actually cause those
issues, sorted by the number of packages that are blocked by the failed build.
This is very useful for the secondary arch maintainers to determine which build failures
to fix next.
7 years, 1 month
[PATCH] Add filter option for download-build
by Mark Hamzy
Add a filter option to the download-build command to skip any file
not matching the supplied glob filter.
Example:
koji download-build --debuginfo --filter '*-debuginfo-*' python-2.7.5-9.fc20
---
cli/koji | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/cli/koji b/cli/koji
index 19727da..82b1088 100755
--- a/cli/koji
+++ b/cli/koji
@@ -5897,6 +5897,7 @@ def anon_handle_download_build(options, session, args):
help=_("URL under which Koji files are accessible"))
parser.add_option("-q", "--quiet", action="store_true", help=_("Do not display progress meter"),
default=options.quiet)
+ parser.add_option("--filter", dest="filter", help=_("Only download files matching, via glob, the supplied filter"))
(suboptions, args) = parser.parse_args(args)
if len(args) < 1:
parser.error(_("Please specify a package N-V-R or build ID"))
@@ -5989,6 +5990,13 @@ def anon_handle_download_build(options, session, args):
pg = progress.TextMeter()
for url, relpath in urls:
+ if suboptions.filter:
+ match = fnmatch.fnmatch(relpath, suboptions.filter)
+ if not match:
+ if options.debug:
+ print "Skipping %s, filtered out by %s" % (relpath, suboptions.filter)
+ continue
+
file = grabber.urlopen(url, progress_obj=pg, text=relpath)
if '/' in relpath:
--
1.9.3
7 years, 1 month
remove wrongly named package from koji?
by Pat Riehecky
I made a typo when I ran a 'koji add-pkg' and can't seem to find a way
to remove the entry from the database.
I've never built anything against the entry, and never will.
The package removed from the relevant tag, but still shows up in some of
my searches (regex with a .*).
How can I make this go away?
Pat
--
Pat Riehecky
Scientific Linux developer
http://www.scientificlinux.org/
7 years, 1 month
[koji PATCH] koji-shadow: convert certificate paths to absolute
by Dan Horák
Converting the certificate paths to absolute using os.path.expanduser() will
allow using a common shadow config for multiple users.
---
util/koji-shadow | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/util/koji-shadow b/util/koji-shadow
index 9d0ce67..bdf3ce3 100755
--- a/util/koji-shadow
+++ b/util/koji-shadow
@@ -293,6 +293,12 @@ def ensure_connection(session):
def activate_session(session):
"""Test and login the session is applicable"""
global options
+
+ # convert to absolute paths
+ options.auth_cert = os.path.expanduser(options.auth_cert)
+ options.auth_ca = os.path.expanduser(options.auth_ca)
+ options.serverca = os.path.expanduser(options.serverca)
+
if options.noauth:
#skip authentication
pass
--
1.9.3
7 years, 1 month
[PATCH] move workdir from /tmp/koji/ to /var/tmp/koji/
by Dennis Gilmore
moving the workdir is needed as /tmp can be quite small with /tmp on tmpfs
Signed-off-by: Dennis Gilmore <dennis(a)ausil.us>
---
builder/kojid | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/builder/kojid b/builder/kojid
index f101e0f..14309bb 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -3657,7 +3657,7 @@ def get_options():
'log_level': None,
'topdir': '/mnt/koji',
'topurl': None,
- 'workdir': '/tmp/koji',
+ 'workdir': '/var/tmp/koji',
'pluginpath': '/usr/lib/koji-builder-plugins',
'mockdir': '/var/lib/mock',
'mockuser': 'kojibuilder',
--
1.9.3
7 years, 1 month
[PATCH] Add retry for unreliable network
by Mark Hamzy
There are two particularly troublesome network calls that can fail
if the network has issues. Therefore, add a retries option to allow
looping until the call succeeds or too many retries happen.
---
mash/__init__.py | 32 +++++++++++++++++++++++++++-----
mash/config.py | 1 +
2 files changed, 28 insertions(+), 5 deletions(-)
diff --git a/mash/__init__.py b/mash/__init__.py
index 566b8c1..9325794 100644
--- a/mash/__init__.py
+++ b/mash/__init__.py
@@ -31,6 +31,8 @@ import yum
import rpmUtils.arch
+import xml.parsers.expat
+
def nevra(pkg):
return '%s-%s:%s-%s.%s' % (pkg['name'],pkg['epoch'],pkg['version'],pkg['release'],pkg['arch'])
@@ -214,9 +216,17 @@ class Mash:
self.logger.error("ERROR: can't download %s from signed path %s" % (nevra(pkg), srcurl))
return 1
srcurl = os.path.join(koji.pathinfo.build(z), koji.pathinfo.rpm(pkg))
- try:
- result = urlgrabber.grabber.urlgrab(srcurl, cachepath)
- except:
+ tries = 0
+ downloaded = False
+ while tries < self.config.retries and not downloaded:
+ try:
+ tries += 1
+ result = urlgrabber.grabber.urlgrab(srcurl, cachepath)
+ downloaded = True
+ except:
+ self.logger.warning("WARNING: can't download %s from %s, attempt #%d" % (nevra(pkg), srcurl, tries))
+ os.remove(cachepath)
+ if not downloaded:
self.logger.error("ERROR: can't download %s from %s" % (nevra(pkg), srcurl))
return 1
@@ -300,8 +310,20 @@ class Mash:
os.makedirs(self.config.cachedir, 0755)
# Get package list. This is an expensive operation.
self.logger.info("Getting package lists for %s..." % (self.config.tag))
-
- (pkglist, buildlist) = self.session.listTaggedRPMS(self.config.tag, inherit = self.config.inherit, latest = self.config.latest, rpmsigs = True)
+
+ tries = 0
+ downloaded = False
+ while tries < self.config.retries and not downloaded:
+ try:
+ tries += 1
+ (pkglist, buildlist) = self.session.listTaggedRPMS(self.config.tag, inherit = self.config.inherit, latest = self.config.latest, rpmsigs = True)
+ downloaded = True
+ except xml.parsers.expat.ExpatError:
+ self.logger.warning("WARNING: can't listTaggedRPMS from koji, attempt #%d" % (tries))
+ if not downloaded:
+ self.logger.error("ERROR: can't listTaggedRPMS from koji")
+ sys.exit(1)
+
# filter by key
biglist = PackageList(self.config)
for pkg in pkglist:
diff --git a/mash/config.py b/mash/config.py
index e4e9b95..3da98d1 100644
--- a/mash/config.py
+++ b/mash/config.py
@@ -100,6 +100,7 @@ class MashDistroConfig(config.BaseConfig):
hash_packages = config.BoolOption(False)
parent_repos = config.ListOption()
previous = None
+ retries = config.Option(1)
def fixup(self, sect):
if not self.name:
--
1.9.0
7 years, 1 month
[PATCH] parallel-drpm enable in mash
by Kevin Fenzi
Createrepo in rawhide now has parallel drpm creation, this patch should hopefully
enable it in mash (with 8 workers).
Note that it's missing a createrepo version check right now since there's not yet
been a released createrepo with the support.
kevin
---
configs/rawhide.mash | 1 +
mash/__init__.py | 6 +++++-
mash/config.py | 2 ++
mash/metadata.py | 5 +++--
4 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/configs/rawhide.mash b/configs/rawhide.mash
index 1deba24..69992c9 100644
--- a/configs/rawhide.mash
+++ b/configs/rawhide.mash
@@ -17,6 +17,7 @@ arches = i386 x86_64 armhfp
delta = True
max_delta_rpm_size = 800000000
max_delta_rpm_age = 604800
+delta_workers = 8
# Change distro_tags as fedora-release version gets bumped
distro_tags = cpe:/o:fedoraproject:fedora:21 rawhide
hash_packages = True
diff --git a/mash/__init__.py b/mash/__init__.py
index 8ced405..566b8c1 100644
--- a/mash/__init__.py
+++ b/mash/__init__.py
@@ -121,7 +121,11 @@ class Mash:
max_delta_rpm_age = self.config.max_delta_rpm_age
else:
max_delta_rpm_age = None # No age specified. Copy all.
- md.set_delta(paths, max_delta_rpm_size, max_delta_rpm_age)
+ if self.config.delta_workers:
+ delta_workers = self.config.delta_workers
+ else:
+ delta_workers = 1 # default to 1 worker
+ md.set_delta(paths, max_delta_rpm_size, max_delta_rpm_age, delta_workers)
if previous:
md.set_previous(previous)
# Setup the distro tags
diff --git a/mash/config.py b/mash/config.py
index 1e9f140..e4e9b95 100644
--- a/mash/config.py
+++ b/mash/config.py
@@ -47,6 +47,7 @@ class MashConfig(config.BaseConfig):
repoviewtitle = config.Option('"Mash - %(arch)s"')
delta = config.BoolOption(False)
delta_dirs = config.ListOption()
+ delta_workers = config.Option(1)
max_delta_rpm_size = config.Option(300000000)
max_delta_rpm_age = config.Option()
make_ancient = config.BoolOption(False)
@@ -90,6 +91,7 @@ class MashDistroConfig(config.BaseConfig):
delta_dirs = config.Inherit(MashConfig.delta_dirs)
max_delta_rpm_size = config.Inherit(MashConfig.delta_dirs)
max_delta_rpm_age = config.Inherit(MashConfig.max_delta_rpm_age)
+ delta_workers = config.Inherit(MashConfig.delta_workers)
make_ancient = config.Inherit(MashConfig.make_ancient)
latest = config.Inherit(MashConfig.latest)
distro_tags = config.Inherit(MashConfig.distro_tags)
diff --git a/mash/metadata.py b/mash/metadata.py
index e20b1d6..50a67da 100644
--- a/mash/metadata.py
+++ b/mash/metadata.py
@@ -71,7 +71,7 @@ class MetadataOld:
if skip:
self.args.append('--skip-stat')
- def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age):
+ def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age, delta_workers):
# Sorry, can't do that here.
pass
@@ -136,12 +136,13 @@ class MetadataNew:
def set_skipstat(self, skip):
self.conf.skip_stat = skip
- def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age):
+ def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age, delta_workers):
if rpm.labelCompare([createrepo.__version__,'0','0'], ['0.9.7', '0', '0']) >= 0:
self.conf.deltas = True
self.conf.oldpackage_paths = deltapaths
self.conf.max_delta_rpm_size = max_delta_rpm_size
self.conf.max_delta_rpm_age = max_delta_rpm_age
+ self.conf.delta_workers = delta_workers
def set_previous(self, previous):
if rpm.labelCompare([createrepo.__version__,'0','0'], ['0.9.7', '0', '0']) >= 0:
--
2.0.0
7 years, 1 month
[PATCH] image: support xz compressed raw files
by Dennis Gilmore
as we publish the raw files on the mirrors we want to be able to request
xz compressed versions of theraw image.
Signed-off-by: Dennis Gilmore <dennis(a)ausil.us>
---
builder/kojid | 25 ++++++++++++++++++++++++-
cli/koji | 4 ++--
docs/schema-upgrade-1.9-next.sql | 9 +++++++++
docs/schema.sql | 1 +
4 files changed, 36 insertions(+), 3 deletions(-)
create mode 100644 docs/schema-upgrade-1.9-next.sql
diff --git a/builder/kojid b/builder/kojid
index 14309bb..34c62d8 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2737,7 +2737,7 @@ class BaseImageTask(OzImageTask):
we have to do this. rhevm-ova requires rhevm, but if the user did not
request it, we should not pass it back up.
"""
- supported = ('raw', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker')
+ supported = ('raw', 'raw-xz', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker')
for f in formats:
if f not in supported:
raise koji.ApplianceError('Invalid format: %s' % f)
@@ -2863,6 +2863,27 @@ class BaseImageTask(OzImageTask):
base.base_image.parameters['libvirt_xml'])
images[format] = {'image': newimg, 'libvirt': lxml}
+ # xz compress the raw disk image if asked for
+ for format in ('raw-xz',):
+ if format not in self.formats:
+ continue
+ newimg = os.path.join(self.workdir, imgname + 'raw.xz')
+ rawimg = os.path.join(self.workdir, imgname + 'raw')
+ cmd = ['/bin/cp', base.base_image.data, rawimg]
+ conlog = os.path.join(self.workdir,
+ 'xz-cp-%s-%s.log' % (format, arch))
+ log_output(self.session, cmd[0], cmd, conlog,
+ self.getUploadDir(), logerror=1)
+ cmd = ['/usr/bin/xz', '-z', rawimg]
+ conlog = os.path.join(self.workdir,
+ 'xz-%s-%s.log' % (format, arch))
+ log_output(self.session, cmd[0], cmd, conlog,
+ self.getUploadDir(), logerror=1)
+ lxml = self.fixImageXML(format, imgname,
+ 'libvirt-%s-%s.xml' % (format, arch),
+ base.base_image.parameters['libvirt_xml'])
+ images[format] = {'image': newimg, 'libvirt': lxml}
+
return images
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None):
@@ -2954,6 +2975,8 @@ class BaseImageTask(OzImageTask):
newname = imgname + '.' + format.replace('-', '.')
elif format == 'docker':
newname = imgname + '.' + 'tar.gz'
+ elif format == 'raw-xz':
+ newname = imgname + '.' + 'raw.xz'
else:
newname = imgname + '.' + format
if format != 'docker':
diff --git a/cli/koji b/cli/koji
index 504b4ba..1ba273f 100755
--- a/cli/koji
+++ b/cli/koji
@@ -4981,7 +4981,7 @@ def handle_spin_appliance(options, session, args):
help=_("Set the number of virtual cpus in the appliance, " +
"default is 1"))
parser.add_option("--format", metavar="DISK_FORMAT", default='raw',
- help=_("Disk format, default is raw. Other options are qcow, " +
+ help=_("Disk format, default is raw. Other options are raw-xz, qcow, " +
"qcow2, and vmx."))
(task_options, args) = parser.parse_args(args)
@@ -4998,7 +4998,7 @@ def handle_spin_appliance(options, session, args):
def handle_image_build(options, session, args):
"""Create a disk image given an install tree"""
formats = ('vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova',
- 'docker')
+ 'docker', 'raw-xz')
usage = _("usage: %prog image-build [options] <name> <version> " +
"<target> <install-tree-url> <arch> [<arch>...]")
usage += _("\n %prog image-build --config FILE")
diff --git a/docs/schema-upgrade-1.9-next.sql b/docs/schema-upgrade-1.9-next.sql
new file mode 100644
index 0000000..7d45e91
--- /dev/null
+++ b/docs/schema-upgrade-1.9-next.sql
@@ -0,0 +1,9 @@
+-- schema migration from version 1.9 to next
+-- note: this update will require additional steps, please see the migration doc
+
+BEGIN;
+
+-- new archive types
+insert into archivetypes (name, description, extensions) values ('raw-xz', 'xz compressed raw disk image', 'raw.xz');
+
+COMMIT;
diff --git a/docs/schema.sql b/docs/schema.sql
index 56418c9..91bcfd2 100644
--- a/docs/schema.sql
+++ b/docs/schema.sql
@@ -713,6 +713,7 @@ insert into archivetypes (name, description, extensions) values ('pdb', 'Windows
insert into archivetypes (name, description, extensions) values ('oem', 'Windows driver oem file', 'oem');
insert into archivetypes (name, description, extensions) values ('iso', 'CD/DVD Image', 'iso');
insert into archivetypes (name, description, extensions) values ('raw', 'Raw disk image', 'raw');
+insert into archivetypes (name, description, extensions) values ('raw-xz', 'xz compressed raw disk image', 'raw.xz');
insert into archivetypes (name, description, extensions) values ('qcow', 'QCOW image', 'qcow');
insert into archivetypes (name, description, extensions) values ('qcow2', 'QCOW2 image', 'qcow2');
insert into archivetypes (name, description, extensions) values ('vmdk', 'vSphere image', 'vmdk');
--
2.0.0
7 years, 1 month
