[createrepo] latest head
Seth Vidal
skvidal at fedoraproject.org
Fri Jul 29 19:37:54 UTC 2011
commit 7e7446089b3d4772f1d06561fd12827f0be82e0a
Author: Seth Vidal <skvidal at fedoraproject.org>
Date: Fri Jul 29 15:37:42 2011 -0400
latest head
createrepo-head.patch | 419 +++++++++++++++++++++++++++++++++++++++++++++++--
createrepo.spec | 6 +-
2 files changed, 413 insertions(+), 12 deletions(-)
---
diff --git a/createrepo-head.patch b/createrepo-head.patch
index 5203f35..14e40e6 100644
--- a/createrepo-head.patch
+++ b/createrepo-head.patch
@@ -1,5 +1,31 @@
+diff --git a/createrepo.bash b/createrepo.bash
+index 54ac8b2..4222fa0 100644
+--- a/createrepo.bash
++++ b/createrepo.bash
+@@ -30,6 +30,10 @@ _cr_createrepo()
+ COMPREPLY=( $( compgen -f -o plusdirs -X '!*.rpm' -- "$2" ) )
+ return 0
+ ;;
++ --retain-old-md)
++ COMPREPLY=( $( compgen -W '0 1 2 3 4 5 6 7 8 9' -- "$2" ) )
++ return 0
++ ;;
+ --num-deltas)
+ COMPREPLY=( $( compgen -W '1 2 3 4 5 6 7 8 9' -- "$2" ) )
+ return 0
+@@ -42,8 +46,8 @@ _cr_createrepo()
+ --cachedir --checkts --no-database --update --update-md-path
+ --skip-stat --split --pkglist --includepkg --outputdir
+ --skip-symlinks --changelog-limit --unique-md-filenames
+- --simple-md-filenames --distro --content --repo --revision --deltas
+- --oldpackagedirs --num-deltas --read-pkgs-list
++ --simple-md-filenames --retain-old-md --distro --content --repo
++ --revision --deltas --oldpackagedirs --num-deltas --read-pkgs-list
+ --max-delta-rpm-size --workers' -- "$2" ) )
+ else
+ COMPREPLY=( $( compgen -d -- "$2" ) )
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
-index 8f2538e..8549188 100644
+index 8f2538e..30f7422 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -27,11 +27,11 @@ import stat
@@ -17,7 +43,15 @@ index 8f2538e..8549188 100644
import rpmUtils.transaction
from utils import _, errorprint, MDError
-@@ -530,39 +530,19 @@ class MetaDataGenerator:
+@@ -110,6 +110,7 @@ class MetaDataConfig(object):
+ self.worker_cmd = '/usr/share/createrepo/worker.py'
+
+ #self.worker_cmd = './worker.py' # helpful when testing
++ self.retain_old_md = 0
+
+ class SimpleMDCallBack(object):
+ def errorlog(self, thing):
+@@ -530,39 +531,19 @@ class MetaDataGenerator:
old_pkg = pkg
if pkg.find("://") != -1:
old_pkg = os.path.basename(pkg)
@@ -67,7 +101,7 @@ index 8f2538e..8549188 100644
#FIXME - if we're in update and we have deltas enabled
# check the presto data for this pkg and write its info back out
# to our deltafile
-@@ -584,12 +564,12 @@ class MetaDataGenerator:
+@@ -584,12 +565,12 @@ class MetaDataGenerator:
po = None
if isinstance(pkg, YumAvailablePackage):
po = pkg
@@ -82,7 +116,7 @@ index 8f2538e..8549188 100644
self.read_pkgs.append(pkg)
if po:
-@@ -609,7 +589,7 @@ class MetaDataGenerator:
+@@ -609,7 +590,7 @@ class MetaDataGenerator:
# open the files they created and write them out to our metadata
# add up the total pkg counts and return that value
worker_tmp_path = tempfile.mkdtemp()
@@ -91,7 +125,7 @@ index 8f2538e..8549188 100644
worker_cmd_dict = {}
worker_jobs = {}
base_worker_cmdline = [self.conf.worker_cmd,
-@@ -617,7 +597,8 @@ class MetaDataGenerator:
+@@ -617,7 +598,8 @@ class MetaDataGenerator:
'--pkgoptions=_collapse_libc_requires=%s' % self.conf.collapse_glibc_requires,
'--pkgoptions=_cachedir=%s' % self.conf.cachedir,
'--pkgoptions=_baseurl=%s' % self.conf.baseurl,
@@ -101,7 +135,7 @@ index 8f2538e..8549188 100644
if self.conf.quiet:
base_worker_cmdline.append('--quiet')
-@@ -660,7 +641,12 @@ class MetaDataGenerator:
+@@ -660,7 +642,12 @@ class MetaDataGenerator:
if line:
self.callback.errorlog('Worker %s: %s' % (num, line.rstrip()))
@@ -115,7 +149,7 @@ index 8f2538e..8549188 100644
if not self.conf.quiet:
self.callback.log("Workers Finished")
# finished with workers
-@@ -784,7 +770,6 @@ class MetaDataGenerator:
+@@ -784,7 +771,6 @@ class MetaDataGenerator:
return self._old_package_dict
self._old_package_dict = {}
@@ -123,7 +157,7 @@ index 8f2538e..8549188 100644
for d in self.conf.oldpackage_paths:
for f in self.getFileList(d, '.rpm'):
fp = d + '/' + f
-@@ -874,7 +859,6 @@ class MetaDataGenerator:
+@@ -874,7 +860,6 @@ class MetaDataGenerator:
thisdata = RepoData()
thisdata.type = mdtype
@@ -131,7 +165,7 @@ index 8f2538e..8549188 100644
thisdata.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, sfile))
thisdata.checksum = (self.conf.sumtype, csum)
if compress:
-@@ -1046,7 +1030,7 @@ class MetaDataGenerator:
+@@ -1046,7 +1031,7 @@ class MetaDataGenerator:
if self.conf.additional_metadata:
@@ -140,7 +174,7 @@ index 8f2538e..8549188 100644
mdcontent = self._createRepoDataObject(md_file, md_type)
repomd.repoData[mdcontent.type] = mdcontent
-@@ -1110,12 +1094,6 @@ class MetaDataGenerator:
+@@ -1110,23 +1095,42 @@ class MetaDataGenerator:
raise MDError, _(
'Could not remove old metadata file: %s: %s') % (oldfile, e)
@@ -150,10 +184,93 @@ index 8f2538e..8549188 100644
- except (OSError, IOError), e:
- old_contents = []
-
++ old_to_remove = []
++ old_pr = []
++ old_fl = []
++ old_ot = []
++ old_pr_db = []
++ old_fl_db = []
++ old_ot_db = []
for f in os.listdir(output_old_dir):
oldfile = os.path.join(output_old_dir, f)
finalfile = os.path.join(output_final_dir, f)
-@@ -1240,7 +1218,6 @@ class SplitMetaDataGenerator(MetaDataGenerator):
+- if f.find('-') != -1 and f.split('-')[1] in ('primary.sqlite.bz2',
+- 'filelists.sqlite.bz2', 'primary.xml.gz','other.sqlite.bz2',
+- 'other.xml.gz','filelists.xml.gz'):
+- os.remove(oldfile) # kill off the old ones
+- continue
++
++ for (end,lst) in (('-primary.sqlite.bz2', old_pr_db), ('-primary.xml.gz', old_pr),
++ ('-filelists.sqlite.bz2', old_fl_db), ('-filelists.xml.gz', old_fl),
++ ('-other.sqlite.bz2', old_ot_db), ('-other.xml.gz', old_ot)):
++ if f.endswith(end):
++ lst.append(oldfile)
++ break
++
++ # make a list of the old metadata files we don't want to remove.
++ for lst in (old_pr, old_fl, old_ot, old_pr_db, old_fl_db, old_ot_db):
++ sortlst = sorted(lst, key=lambda x: os.path.getmtime(x),
++ reverse=True)
++ for thisf in sortlst[self.conf.retain_old_md:]:
++ old_to_remove.append(thisf)
++
++ for f in os.listdir(output_old_dir):
++ oldfile = os.path.join(output_old_dir, f)
++ finalfile = os.path.join(output_final_dir, f)
++
+ if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2',
+- 'primary.sqlite.bz2'):
+- os.remove(oldfile)
++ 'primary.sqlite.bz2') or oldfile in old_to_remove:
++ try:
++ os.remove(oldfile)
++ except (OSError, IOError), e:
++ raise MDError, _(
++ 'Could not remove old metadata file: %s: %s') % (oldfile, e)
+ continue
+
+ if os.path.exists(finalfile):
+@@ -1147,14 +1151,11 @@ class MetaDataGenerator:
+ msg += _('Error was %s') % e
+ raise MDError, msg
+
+- try:
+- os.rmdir(output_old_dir)
+- except OSError, e:
+- self.errorlog(_('Could not remove old metadata dir: %s')
+- % self.conf.olddir)
+- self.errorlog(_('Error was %s') % e)
+- self.errorlog(_('Please clean up this directory manually.'))
++ self._cleanup_tmp_repodata_dir()
++ self._write_out_read_pkgs_list()
++
+
++ def _write_out_read_pkgs_list(self):
+ # write out the read_pkgs_list file with self.read_pkgs
+ if self.conf.read_pkgs_list:
+ try:
+@@ -1167,6 +1168,20 @@ class MetaDataGenerator:
+ % self.conf.read_pkgs_list)
+ self.errorlog(_('Error was %s') % e)
+
++ def _cleanup_tmp_repodata_dir(self):
++ output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir)
++ output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir)
++ for dirbase in (self.conf.olddir, self.conf.tempdir):
++ dirpath = os.path.join(self.conf.outputdir, dirbase)
++ if os.path.exists(dirpath):
++ try:
++ os.rmdir(dirpath)
++ except OSError, e:
++ self.errorlog(_('Could not remove temp metadata dir: %s')
++ % dirbase)
++ self.errorlog(_('Error was %s') % e)
++ self.errorlog(_('Please clean up this directory manually.'))
++
+ def setup_sqlite_dbs(self, initdb=True):
+ """sets up the sqlite dbs w/table schemas and db_infos"""
+ destdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
+@@ -1240,7 +1255,6 @@ class SplitMetaDataGenerator(MetaDataGenerator):
self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
try:
self.openMetadataDocs()
@@ -161,6 +278,61 @@ index 8f2538e..8549188 100644
for mydir in self.conf.directories:
self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
self.writeMetadataDocs(filematrix[mydir], mydir)
+diff --git a/createrepo/merge.py b/createrepo/merge.py
+index b3b2ea1..1ac43bb 100644
+--- a/createrepo/merge.py
++++ b/createrepo/merge.py
+@@ -24,6 +24,7 @@ from yum.misc import unique, getCacheDir
+ import yum.update_md
+ import rpmUtils.arch
+ import operator
++from utils import MDError
+ import createrepo
+ import tempfile
+
+@@ -84,6 +85,8 @@ class RepoMergeBase:
+ # in the repolist
+ count = 0
+ for r in self.repolist:
++ if r[0] == '/':
++ r = 'file://' + r # just fix the file repos, this is silly.
+ count +=1
+ rid = 'repo%s' % count
+ n = self.yumbase.add_enable_repo(rid, baseurls=[r],
+@@ -92,7 +95,10 @@ class RepoMergeBase:
+ n._merge_rank = count
+
+ #setup our sacks
+- self.yumbase._getSacks(archlist=self.archlist)
++ try:
++ self.yumbase._getSacks(archlist=self.archlist)
++ except yum.Errors.RepoError, e:
++ raise MDError, "Could not setup merge repo pkgsack: %s" % e
+
+ myrepos = self.yumbase.repos.listEnabled()
+
+@@ -102,11 +108,16 @@ class RepoMergeBase:
+ def write_metadata(self, outputdir=None):
+ mytempdir = tempfile.mkdtemp()
+ if self.groups:
+- comps_fn = mytempdir + '/groups.xml'
+- compsfile = open(comps_fn, 'w')
+- compsfile.write(self.yumbase.comps.xml())
+- compsfile.close()
+- self.mdconf.groupfile=comps_fn
++ try:
++ comps_fn = mytempdir + '/groups.xml'
++ compsfile = open(comps_fn, 'w')
++ compsfile.write(self.yumbase.comps.xml())
++ compsfile.close()
++ except yum.Errors.GroupsError, e:
++ # groups not being available shouldn't be a fatal error
++ pass
++ else:
++ self.mdconf.groupfile=comps_fn
+
+ if self.updateinfo:
+ ui_fn = mytempdir + '/updateinfo.xml'
diff --git a/createrepo/readMetadata.py b/createrepo/readMetadata.py
index 27d3690..88e5d95 100644
--- a/createrepo/readMetadata.py
@@ -443,6 +615,231 @@ index 27d3690..88e5d95 100644
+ print po.xml_dump_filelists_metadata()
+ print po.xml_dump_other_metadata()
+
+diff --git a/docs/createrepo.8 b/docs/createrepo.8
+index e3c4c3b..96b5bf8 100644
+--- a/docs/createrepo.8
++++ b/docs/createrepo.8
+@@ -53,7 +53,8 @@ gullible).
+ Don't generate repo metadata, if their timestamps are newer than its rpms.
+ This option decreases the processing time drastically again, if you happen
+ to run it on an unmodified repo, but it is (currently) mutual exclusive
+-with the --split option.
++with the --split option. NOTE: This command will not notice when
++packages have been removed from repo. Use --update to handle that.
+ .br
+ .IP "\fB\--split\fP"
+ Run in split media mode. Rather than pass a single directory, take a set of
+diff --git a/genpkgmetadata.py b/genpkgmetadata.py
+index 8c98191..512420b 100755
+--- a/genpkgmetadata.py
++++ b/genpkgmetadata.py
+@@ -100,6 +100,8 @@ def parse_args(args, conf):
+ parser.add_option("--simple-md-filenames", dest="simple_md_filenames",
+ help="do not include the file's checksum in the filename, helps with proxies",
+ default=False, action="store_true")
++ parser.add_option("--retain-old-md", default=0, type='int', dest='retain_old_md',
++ help="keep around the latest (by timestamp) N copies of the old repodata")
+ parser.add_option("--distro", default=[], action="append",
+ help="distro tag and optional cpeid: --distro" "'cpeid,textname'")
+ parser.add_option("--content", default=[], dest='content_tags',
+@@ -240,6 +242,7 @@ def main(args):
+ if mdgen.checkTimeStamps():
+ if mdgen.conf.verbose:
+ print _('repo is up to date')
++ mdgen._cleanup_tmp_repodata_dir()
+ sys.exit(0)
+
+ if conf.profile:
+diff --git a/mergerepo.py b/mergerepo.py
+index 05e5f5e..069a70b 100755
+--- a/mergerepo.py
++++ b/mergerepo.py
+@@ -18,6 +18,7 @@
+
+ import sys
+ import createrepo.merge
++from createrepo.utils import MDError
+ from optparse import OptionParser
+
+ #TODO:
+@@ -77,9 +78,12 @@ def main(args):
+ rmbase.groups = False
+ if opts.noupdateinfo:
+ rmbase.updateinfo = False
+-
+- rmbase.merge_repos()
+- rmbase.write_metadata()
+-
++ try:
++ rmbase.merge_repos()
++ rmbase.write_metadata()
++ except MDError, e:
++ print >> sys.stderr, "Could not merge repos: %s" % e
++ sys.exit(1)
++
+ if __name__ == "__main__":
+ main(sys.argv[1:])
+diff --git a/modifyrepo.py b/modifyrepo.py
+index 17094a4..c3370e8 100755
+--- a/modifyrepo.py
++++ b/modifyrepo.py
+@@ -1,11 +1,15 @@
+ #!/usr/bin/python
+-# This tools is used to insert arbitrary metadata into an RPM repository.
++# This tool is used to manipulate arbitrary metadata in a RPM repository.
+ # Example:
+ # ./modifyrepo.py updateinfo.xml myrepo/repodata
++# or
++# ./modifyrepo.py --remove updateinfo.xml myrepo/repodata
+ # or in Python:
+ # >>> from modifyrepo import RepoMetadata
+ # >>> repomd = RepoMetadata('myrepo/repodata')
+ # >>> repomd.add('updateinfo.xml')
++# or
++# >>> repomd.remove('updateinfo.xml')
+ #
+ # This program is free software; you can redistribute it and/or modify
+ # it under the terms of the GNU General Public License as published by
+@@ -20,6 +24,7 @@
+ # (C) Copyright 2006 Red Hat, Inc.
+ # Luke Macken <lmacken at redhat.com>
+ # modified by Seth Vidal 2008
++# modified by Daniel Mach 2011
+
+ import os
+ import sys
+@@ -49,6 +54,35 @@ class RepoMetadata:
+ except RepoMDError, e:
+ raise MDError, 'Could not parse %s' % self.repomdxml
+
++ def _get_mdtype(self, mdname, mdtype=None):
++ """ Get mdtype from existing mdtype or from a mdname. """
++ if mdtype:
++ return mdtype
++ return mdname.split('.')[0]
++
++ def _print_repodata(self, repodata):
++ """ Print repodata details. """
++ print " type =", repodata.type
++ print " location =", repodata.location[1]
++ print " checksum =", repodata.checksum[1]
++ print " timestamp =", repodata.timestamp
++ print " open-checksum =", repodata.openchecksum[1]
++
++ def _write_repomd(self):
++ """ Write the updated repomd.xml. """
++ outmd = file(self.repomdxml, 'w')
++ outmd.write(self.repoobj.dump_xml())
++ outmd.close()
++ print "Wrote:", self.repomdxml
++
++ def _remove_repodata_file(self, repodata):
++ """ Remove a file specified in repodata location """
++ try:
++ os.remove(repodata.location[1])
++ except OSError, ex:
++ if ex.errno != 2:
++ # continue on a missing file
++ raise MDError("could not remove file %s" % repodata.location[1])
+
+ def add(self, metadata, mdtype=None):
+ """ Insert arbitrary metadata into this repository.
+@@ -78,9 +112,8 @@ class RepoMetadata:
+ ## Compress the metadata and move it into the repodata
+ if not mdname.endswith('.gz'):
+ mdname += '.gz'
+- if not mdtype:
+- mdtype = mdname.split('.')[0]
+-
++ mdtype = self._get_mdtype(mdname, mdtype)
++
+ destmd = os.path.join(self.repodir, mdname)
+ newmd = GzipFile(filename=destmd, mode='wb')
+ newmd.write(md)
+@@ -91,11 +124,8 @@ class RepoMetadata:
+ csum, destmd = checksum_and_rename(destmd, self.checksum_type)
+ base_destmd = os.path.basename(destmd)
+
+-
+- ## Remove any stale metadata
+- if mdtype in self.repoobj.repoData:
+- del self.repoobj.repoData[mdtype]
+-
++ # Remove any stale metadata
++ old_rd = self.repoobj.repoData.pop(mdtype, None)
+
+ new_rd = RepoData()
+ new_rd.type = mdtype
+@@ -105,18 +135,28 @@ class RepoMetadata:
+ new_rd.size = str(os.stat(destmd).st_size)
+ new_rd.timestamp = str(os.stat(destmd).st_mtime)
+ self.repoobj.repoData[new_rd.type] = new_rd
+-
+- print " type =", new_rd.type
+- print " location =", new_rd.location[1]
+- print " checksum =", new_rd.checksum[1]
+- print " timestamp =", new_rd.timestamp
+- print " open-checksum =", new_rd.openchecksum[1]
+-
+- ## Write the updated repomd.xml
+- outmd = file(self.repomdxml, 'w')
+- outmd.write(self.repoobj.dump_xml())
+- outmd.close()
+- print "Wrote:", self.repomdxml
++ self._print_repodata(new_rd)
++ self._write_repomd()
++
++ if old_rd is not None and old_rd.location[1] != new_rd.location[1]:
++ # remove the old file when overwriting metadata
++ # with the same mdtype but different location
++ self._remove_repodata_file(old_rd)
++
++ def remove(self, metadata, mdtype=None):
++ """ Remove metadata from this repository. """
++ mdname = metadata
++ mdtype = self._get_mdtype(mdname, mdtype)
++
++ old_rd = self.repoobj.repoData.pop(mdtype, None)
++ if old_rd is None:
++ print "Metadata not found: %s" % mdtype
++ return
++
++ self._remove_repodata_file(old_rd)
++ print "Removed:"
++ self._print_repodata(old_rd)
++ self._write_repomd()
+
+
+ def main(args):
+@@ -124,7 +164,9 @@ def main(args):
+ # query options
+ parser.add_option("--mdtype", dest='mdtype',
+ help="specific datatype of the metadata, will be derived from the filename if not specified")
+- parser.usage = "modifyrepo [options] <input_metadata> <output repodata>"
++ parser.add_option("--remove", action="store_true",
++ help="remove specified file from repodata")
++ parser.usage = "modifyrepo [options] [--remove] <input_metadata> <output repodata>"
+
+ (opts, argsleft) = parser.parse_args(args)
+ if len(argsleft) != 2:
+@@ -137,6 +179,17 @@ def main(args):
+ except MDError, e:
+ print "Could not access repository: %s" % str(e)
+ return 1
++
++ # remove
++ if opts.remove:
++ try:
++ repomd.remove(metadata)
++ except MDError, ex:
++ print "Could not remove metadata: %s" % (metadata, str(ex))
++ return 1
++ return
++
++ # add
+ try:
+ repomd.add(metadata, mdtype=opts.mdtype)
+ except MDError, e:
diff --git a/worker.py b/worker.py
index eb35ef7..ab78d90 100755
--- a/worker.py
diff --git a/createrepo.spec b/createrepo.spec
index 3b10489..e27cfe8 100644
--- a/createrepo.spec
+++ b/createrepo.spec
@@ -3,7 +3,7 @@
Summary: Creates a common metadata repository
Name: createrepo
Version: 0.9.9
-Release: 5%{?dist}
+Release: 6%{?dist}
License: GPLv2
Group: System Environment/Base
Source: %{name}-%{version}.tar.gz
@@ -47,6 +47,10 @@ rm -rf $RPM_BUILD_ROOT
%{python_sitelib}/createrepo
%changelog
+* Fri Jul 29 2011 Seth Vidal <skvidal at fedoraproject.org> - 0.9.9-6
+- latest upstream
+- fixes bugs: 713747, 581632, 581628
+
* Wed Jul 20 2011 Seth Vidal <skvidal at fedoraproject.org> - 0.9.9-5
- new patch to fix us breaking certain pungi configs
More information about the scm-commits
mailing list