[createrepo/f15] latest createrepo back to f15
Seth Vidal
skvidal at fedoraproject.org
Fri Jul 29 19:53:57 UTC 2011
commit 753ba82417dd230032c42e9e5fc04b96ea1f47ff
Author: Seth Vidal <skvidal at fedoraproject.org>
Date: Fri Jul 29 15:53:45 2011 -0400
latest createrepo back to f15
createrepo-head.patch | 839 ++++++++++++++++++++++++++++++++++++++++++++++++-
createrepo.spec | 18 +-
2 files changed, 846 insertions(+), 11 deletions(-)
---
diff --git a/createrepo-head.patch b/createrepo-head.patch
index 4673323..14e40e6 100644
--- a/createrepo-head.patch
+++ b/createrepo-head.patch
@@ -1,8 +1,141 @@
+diff --git a/createrepo.bash b/createrepo.bash
+index 54ac8b2..4222fa0 100644
+--- a/createrepo.bash
++++ b/createrepo.bash
+@@ -30,6 +30,10 @@ _cr_createrepo()
+ COMPREPLY=( $( compgen -f -o plusdirs -X '!*.rpm' -- "$2" ) )
+ return 0
+ ;;
++ --retain-old-md)
++ COMPREPLY=( $( compgen -W '0 1 2 3 4 5 6 7 8 9' -- "$2" ) )
++ return 0
++ ;;
+ --num-deltas)
+ COMPREPLY=( $( compgen -W '1 2 3 4 5 6 7 8 9' -- "$2" ) )
+ return 0
+@@ -42,8 +46,8 @@ _cr_createrepo()
+ --cachedir --checkts --no-database --update --update-md-path
+ --skip-stat --split --pkglist --includepkg --outputdir
+ --skip-symlinks --changelog-limit --unique-md-filenames
+- --simple-md-filenames --distro --content --repo --revision --deltas
+- --oldpackagedirs --num-deltas --read-pkgs-list
++ --simple-md-filenames --retain-old-md --distro --content --repo
++ --revision --deltas --oldpackagedirs --num-deltas --read-pkgs-list
+ --max-delta-rpm-size --workers' -- "$2" ) )
+ else
+ COMPREPLY=( $( compgen -d -- "$2" ) )
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
-index 8f2538e..07abc27 100644
+index 8f2538e..30f7422 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
-@@ -660,7 +660,12 @@ class MetaDataGenerator:
+@@ -27,11 +27,11 @@ import stat
+ import fcntl
+ import subprocess
+
+-from yum import misc, Errors, to_unicode
+-from yum.repoMDObject import RepoMD, RepoMDError, RepoData
++from yum import misc, Errors
++from yum.repoMDObject import RepoMD, RepoData
+ from yum.sqlutils import executeSQL
+ from yum.packageSack import MetaSack
+-from yum.packages import YumAvailablePackage, YumLocalPackage
++from yum.packages import YumAvailablePackage
+
+ import rpmUtils.transaction
+ from utils import _, errorprint, MDError
+@@ -110,6 +110,7 @@ class MetaDataConfig(object):
+ self.worker_cmd = '/usr/share/createrepo/worker.py'
+
+ #self.worker_cmd = './worker.py' # helpful when testing
++ self.retain_old_md = 0
+
+ class SimpleMDCallBack(object):
+ def errorlog(self, thing):
+@@ -530,39 +531,19 @@ class MetaDataGenerator:
+ old_pkg = pkg
+ if pkg.find("://") != -1:
+ old_pkg = os.path.basename(pkg)
+- nodes = self.oldData.getNodes(old_pkg)
+- if nodes is not None: # we have a match in the old metadata
++ old_po = self.oldData.getNodes(old_pkg)
++ if old_po: # we have a match in the old metadata
+ if self.conf.verbose:
+ self.callback.log(_("Using data from old metadata for %s")
+ % pkg)
+- (primarynode, filenode, othernode) = nodes
+-
+- for node, outfile in ((primarynode, self.primaryfile),
+- (filenode, self.flfile),
+- (othernode, self.otherfile)):
+- if node is None:
+- break
+-
+- if self.conf.baseurl:
+- anode = node.children
+- while anode is not None:
+- if anode.type != "element":
+- anode = anode.next
+- continue
+- if anode.name == "location":
+- anode.setProp('xml:base', self.conf.baseurl)
+- anode = anode.next
+-
+- output = node.serialize('UTF-8', self.conf.pretty)
+- if output:
+- outfile.write(output)
+- else:
+- if self.conf.verbose:
+- self.callback.log(_("empty serialize on write to" \
+- "%s in %s") % (outfile, pkg))
+- outfile.write('\n')
+-
+- self.oldData.freeNodes(pkg)
++
++ if self.conf.baseurl: # if we have a baseurl set, reset the one
++ # in the old pkg
++ old_po.basepath = self.conf.baseurl
++ self.primaryfile.write(old_po.xml_dump_primary_metadata())
++ self.flfile.write(old_po.xml_dump_filelists_metadata())
++ self.otherfile.write(old_po.xml_dump_other_metadata())
++
+ #FIXME - if we're in update and we have deltas enabled
+ # check the presto data for this pkg and write its info back out
+ # to our deltafile
+@@ -584,12 +565,12 @@ class MetaDataGenerator:
+ po = None
+ if isinstance(pkg, YumAvailablePackage):
+ po = pkg
+- self.read_pkgs.append(po.localpath)
++ self.read_pkgs.append(po.localPkg())
+
+ # if we're dealing with remote pkgs - pitch it over to doing
+ # them one at a time, for now.
+ elif pkg.find('://') != -1:
+- po = self.read_in_package(pkgfile, pkgpath=pkgpath, reldir=reldir)
++ po = self.read_in_package(pkg, pkgpath=pkgpath, reldir=reldir)
+ self.read_pkgs.append(pkg)
+
+ if po:
+@@ -609,7 +590,7 @@ class MetaDataGenerator:
+ # open the files they created and write them out to our metadata
+ # add up the total pkg counts and return that value
+ worker_tmp_path = tempfile.mkdtemp()
+- worker_chunks = utils.split_list_into_equal_chunks(pkgfiles, self.conf.workers)
++ worker_chunks = split_list_into_equal_chunks(pkgfiles, self.conf.workers)
+ worker_cmd_dict = {}
+ worker_jobs = {}
+ base_worker_cmdline = [self.conf.worker_cmd,
+@@ -617,7 +598,8 @@ class MetaDataGenerator:
+ '--pkgoptions=_collapse_libc_requires=%s' % self.conf.collapse_glibc_requires,
+ '--pkgoptions=_cachedir=%s' % self.conf.cachedir,
+ '--pkgoptions=_baseurl=%s' % self.conf.baseurl,
+- '--globalopts=clog_limit=%s' % self.conf.changelog_limit,]
++ '--globalopts=clog_limit=%s' % self.conf.changelog_limit,
++ '--globalopts=sumtype=%s' % self.conf.sumtype, ]
+
+ if self.conf.quiet:
+ base_worker_cmdline.append('--quiet')
+@@ -660,7 +642,12 @@ class MetaDataGenerator:
if line:
self.callback.errorlog('Worker %s: %s' % (num, line.rstrip()))
@@ -16,12 +149,708 @@ index 8f2538e..07abc27 100644
if not self.conf.quiet:
self.callback.log("Workers Finished")
# finished with workers
+@@ -784,7 +771,6 @@ class MetaDataGenerator:
+ return self._old_package_dict
+
+ self._old_package_dict = {}
+- opl = []
+ for d in self.conf.oldpackage_paths:
+ for f in self.getFileList(d, '.rpm'):
+ fp = d + '/' + f
+@@ -874,7 +860,6 @@ class MetaDataGenerator:
+
+ thisdata = RepoData()
+ thisdata.type = mdtype
+- baseloc = None
+ thisdata.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, sfile))
+ thisdata.checksum = (self.conf.sumtype, csum)
+ if compress:
+@@ -1046,7 +1031,7 @@ class MetaDataGenerator:
+
+
+ if self.conf.additional_metadata:
+- for md_type, mdfile in self.conf.additional_metadata.items():
++ for md_type, md_file in self.conf.additional_metadata.items():
+ mdcontent = self._createRepoDataObject(md_file, md_type)
+ repomd.repoData[mdcontent.type] = mdcontent
+
+@@ -1110,23 +1095,42 @@ class MetaDataGenerator:
+ raise MDError, _(
+ 'Could not remove old metadata file: %s: %s') % (oldfile, e)
+
+- # Move everything else back from olddir (eg. repoview files)
+- try:
+- old_contents = os.listdir(output_old_dir)
+- except (OSError, IOError), e:
+- old_contents = []
+-
++ old_to_remove = []
++ old_pr = []
++ old_fl = []
++ old_ot = []
++ old_pr_db = []
++ old_fl_db = []
++ old_ot_db = []
+ for f in os.listdir(output_old_dir):
+ oldfile = os.path.join(output_old_dir, f)
+ finalfile = os.path.join(output_final_dir, f)
+- if f.find('-') != -1 and f.split('-')[1] in ('primary.sqlite.bz2',
+- 'filelists.sqlite.bz2', 'primary.xml.gz','other.sqlite.bz2',
+- 'other.xml.gz','filelists.xml.gz'):
+- os.remove(oldfile) # kill off the old ones
+- continue
++
++ for (end,lst) in (('-primary.sqlite.bz2', old_pr_db), ('-primary.xml.gz', old_pr),
++ ('-filelists.sqlite.bz2', old_fl_db), ('-filelists.xml.gz', old_fl),
++ ('-other.sqlite.bz2', old_ot_db), ('-other.xml.gz', old_ot)):
++ if f.endswith(end):
++ lst.append(oldfile)
++ break
++
++ # make a list of the old metadata files we don't want to remove.
++ for lst in (old_pr, old_fl, old_ot, old_pr_db, old_fl_db, old_ot_db):
++ sortlst = sorted(lst, key=lambda x: os.path.getmtime(x),
++ reverse=True)
++ for thisf in sortlst[self.conf.retain_old_md:]:
++ old_to_remove.append(thisf)
++
++ for f in os.listdir(output_old_dir):
++ oldfile = os.path.join(output_old_dir, f)
++ finalfile = os.path.join(output_final_dir, f)
++
+ if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2',
+- 'primary.sqlite.bz2'):
+- os.remove(oldfile)
++ 'primary.sqlite.bz2') or oldfile in old_to_remove:
++ try:
++ os.remove(oldfile)
++ except (OSError, IOError), e:
++ raise MDError, _(
++ 'Could not remove old metadata file: %s: %s') % (oldfile, e)
+ continue
+
+ if os.path.exists(finalfile):
+@@ -1147,14 +1151,11 @@ class MetaDataGenerator:
+ msg += _('Error was %s') % e
+ raise MDError, msg
+
+- try:
+- os.rmdir(output_old_dir)
+- except OSError, e:
+- self.errorlog(_('Could not remove old metadata dir: %s')
+- % self.conf.olddir)
+- self.errorlog(_('Error was %s') % e)
+- self.errorlog(_('Please clean up this directory manually.'))
++ self._cleanup_tmp_repodata_dir()
++ self._write_out_read_pkgs_list()
++
+
++ def _write_out_read_pkgs_list(self):
+ # write out the read_pkgs_list file with self.read_pkgs
+ if self.conf.read_pkgs_list:
+ try:
+@@ -1167,6 +1168,20 @@ class MetaDataGenerator:
+ % self.conf.read_pkgs_list)
+ self.errorlog(_('Error was %s') % e)
+
++ def _cleanup_tmp_repodata_dir(self):
++ output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir)
++ output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir)
++ for dirbase in (self.conf.olddir, self.conf.tempdir):
++ dirpath = os.path.join(self.conf.outputdir, dirbase)
++ if os.path.exists(dirpath):
++ try:
++ os.rmdir(dirpath)
++ except OSError, e:
++ self.errorlog(_('Could not remove temp metadata dir: %s')
++ % dirbase)
++ self.errorlog(_('Error was %s') % e)
++ self.errorlog(_('Please clean up this directory manually.'))
++
+ def setup_sqlite_dbs(self, initdb=True):
+ """sets up the sqlite dbs w/table schemas and db_infos"""
+ destdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
+@@ -1240,7 +1255,6 @@ class SplitMetaDataGenerator(MetaDataGenerator):
+ self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
+ try:
+ self.openMetadataDocs()
+- original_basedir = self.conf.basedir
+ for mydir in self.conf.directories:
+ self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
+ self.writeMetadataDocs(filematrix[mydir], mydir)
+diff --git a/createrepo/merge.py b/createrepo/merge.py
+index b3b2ea1..1ac43bb 100644
+--- a/createrepo/merge.py
++++ b/createrepo/merge.py
+@@ -24,6 +24,7 @@ from yum.misc import unique, getCacheDir
+ import yum.update_md
+ import rpmUtils.arch
+ import operator
++from utils import MDError
+ import createrepo
+ import tempfile
+
+@@ -84,6 +85,8 @@ class RepoMergeBase:
+ # in the repolist
+ count = 0
+ for r in self.repolist:
++ if r[0] == '/':
++ r = 'file://' + r # just fix the file repos, this is silly.
+ count +=1
+ rid = 'repo%s' % count
+ n = self.yumbase.add_enable_repo(rid, baseurls=[r],
+@@ -92,7 +95,10 @@ class RepoMergeBase:
+ n._merge_rank = count
+
+ #setup our sacks
+- self.yumbase._getSacks(archlist=self.archlist)
++ try:
++ self.yumbase._getSacks(archlist=self.archlist)
++ except yum.Errors.RepoError, e:
++ raise MDError, "Could not setup merge repo pkgsack: %s" % e
+
+ myrepos = self.yumbase.repos.listEnabled()
+
+@@ -102,11 +108,16 @@ class RepoMergeBase:
+ def write_metadata(self, outputdir=None):
+ mytempdir = tempfile.mkdtemp()
+ if self.groups:
+- comps_fn = mytempdir + '/groups.xml'
+- compsfile = open(comps_fn, 'w')
+- compsfile.write(self.yumbase.comps.xml())
+- compsfile.close()
+- self.mdconf.groupfile=comps_fn
++ try:
++ comps_fn = mytempdir + '/groups.xml'
++ compsfile = open(comps_fn, 'w')
++ compsfile.write(self.yumbase.comps.xml())
++ compsfile.close()
++ except yum.Errors.GroupsError, e:
++ # groups not being available shouldn't be a fatal error
++ pass
++ else:
++ self.mdconf.groupfile=comps_fn
+
+ if self.updateinfo:
+ ui_fn = mytempdir + '/updateinfo.xml'
+diff --git a/createrepo/readMetadata.py b/createrepo/readMetadata.py
+index 27d3690..88e5d95 100644
+--- a/createrepo/readMetadata.py
++++ b/createrepo/readMetadata.py
+@@ -16,11 +16,25 @@
+ # Copyright 2006 Red Hat
+
+ import os
+-import libxml2
+ import stat
+ from utils import errorprint, _
+
+-from yum import repoMDObject
++import yum
++from yum import misc
++from yum.Errors import YumBaseError
++import tempfile
++class CreaterepoPkgOld(yum.sqlitesack.YumAvailablePackageSqlite):
++ # special for special people like us.
++ def _return_remote_location(self):
++
++ if self.basepath:
++ msg = """<location xml:base="%s" href="%s"/>\n""" % (
++ misc.to_xml(self.basepath, attrib=True),
++ misc.to_xml(self.relativepath, attrib=True))
++ else:
++ msg = """<location href="%s"/>\n""" % misc.to_xml(self.relativepath, attrib=True)
++
++ return msg
+
+
+ class MetadataIndex(object):
+@@ -30,178 +44,73 @@ class MetadataIndex(object):
+ opts = {}
+ self.opts = opts
+ self.outputdir = outputdir
++ realpath = os.path.realpath(outputdir)
+ repodatadir = self.outputdir + '/repodata'
+- myrepomdxml = repodatadir + '/repomd.xml'
+- if os.path.exists(myrepomdxml):
+- repomd = repoMDObject.RepoMD('garbageid', myrepomdxml)
+- b = repomd.getData('primary').location[1]
+- f = repomd.getData('filelists').location[1]
+- o = repomd.getData('other').location[1]
+- basefile = os.path.join(self.outputdir, b)
+- filelistfile = os.path.join(self.outputdir, f)
+- otherfile = os.path.join(self.outputdir, o)
+- else:
+- basefile = filelistfile = otherfile = ""
+-
+- self.files = {'base' : basefile,
+- 'filelist' : filelistfile,
+- 'other' : otherfile}
+- self.scan()
++ self._repo = yum.yumRepo.YumRepository('garbageid')
++ self._repo.baseurl = 'file://' + realpath
++ self._repo.basecachedir = tempfile.mkdtemp(dir='/var/tmp', prefix="createrepo")
++ self._repo.metadata_expire = 1
++ self._repo.gpgcheck = 0
++ self._repo.repo_gpgcheck = 0
++ self._repo._sack = yum.sqlitesack.YumSqlitePackageSack(CreaterepoPkgOld)
++ self.pkg_tups_by_path = {}
++ try:
++ self.scan()
++ except YumBaseError, e:
++ print "Could not find valid repo at: %s" % self.outputdir
++
+
+ def scan(self):
+- """Read in and index old repo data"""
+- self.basenodes = {}
+- self.filesnodes = {}
+- self.othernodes = {}
+- self.pkg_ids = {}
++ """Read in old repodata"""
+ if self.opts.get('verbose'):
+ print _("Scanning old repo data")
+- for fn in self.files.values():
+- if not os.path.exists(fn):
+- #cannot scan
+- errorprint(_("Warning: Old repodata file missing: %s") % fn)
+- return
+- root = libxml2.parseFile(self.files['base']).getRootElement()
+- self._scanPackageNodes(root, self._handleBase)
+- if self.opts.get('verbose'):
+- print _("Indexed %i base nodes" % len(self.basenodes))
+- root = libxml2.parseFile(self.files['filelist']).getRootElement()
+- self._scanPackageNodes(root, self._handleFiles)
+- if self.opts.get('verbose'):
+- print _("Indexed %i filelist nodes" % len(self.filesnodes))
+- root = libxml2.parseFile(self.files['other']).getRootElement()
+- self._scanPackageNodes(root, self._handleOther)
+- if self.opts.get('verbose'):
+- print _("Indexed %i other nodes" % len(self.othernodes))
+- #reverse index pkg ids to track references
+- self.pkgrefs = {}
+- for relpath, pkgid in self.pkg_ids.iteritems():
+- self.pkgrefs.setdefault(pkgid,[]).append(relpath)
+-
+- def _scanPackageNodes(self, root, handler):
+- node = root.children
+- while node is not None:
+- if node.type != "element":
+- node = node.next
++ self._repo.sack.populate(self._repo, 'all', None, False)
++ for thispo in self._repo.sack:
++ mtime = thispo.filetime
++ size = thispo.size
++ relpath = thispo.relativepath
++ do_stat = self.opts.get('do_stat', True)
++ if mtime is None:
++ print _("mtime missing for %s") % relpath
+ continue
+- if node.name == "package":
+- handler(node)
+- node = node.next
+-
+- def _handleBase(self, node):
+- top = node
+- node = node.children
+- pkgid = None
+- mtime = None
+- size = None
+- relpath = None
+- do_stat = self.opts.get('do_stat', True)
+- while node is not None:
+- if node.type != "element":
+- node = node.next
++ if size is None:
++ print _("size missing for %s") % relpath
+ continue
+- if node.name == "checksum":
+- pkgid = node.content
+- elif node.name == "time":
+- mtime = int(node.prop('file'))
+- elif node.name == "size":
+- size = int(node.prop('package'))
+- elif node.name == "location":
+- relpath = node.prop('href')
+- node = node.next
+- if relpath is None:
+- print _("Incomplete data for node")
+- return
+- if pkgid is None:
+- print _("pkgid missing for %s") % relpath
+- return
+- if mtime is None:
+- print _("mtime missing for %s") % relpath
+- return
+- if size is None:
+- print _("size missing for %s") % relpath
+- return
+- if do_stat:
+- filepath = os.path.join(self.opts['pkgdir'], relpath)
+- try:
+- st = os.stat(filepath)
+- except OSError:
+- #file missing -- ignore
+- return
+- if not stat.S_ISREG(st.st_mode):
+- #ignore non files
+- return
+- #check size and mtime
+- if st.st_size != size:
+- if self.opts.get('verbose'):
+- print _("Size (%i -> %i) changed for file %s") % (size,st.st_size,filepath)
+- return
+- if int(st.st_mtime) != mtime:
+- if self.opts.get('verbose'):
+- print _("Modification time changed for %s") % filepath
+- return
+- #otherwise we index
+- self.basenodes[relpath] = top
+- self.pkg_ids[relpath] = pkgid
+-
+- def _handleFiles(self, node):
+- pkgid = node.prop('pkgid')
+- if pkgid:
+- self.filesnodes[pkgid] = node
+-
+- def _handleOther(self, node):
+- pkgid = node.prop('pkgid')
+- if pkgid:
+- self.othernodes[pkgid] = node
++ if do_stat:
++ filepath = os.path.join(self.opts['pkgdir'], relpath)
++ try:
++ st = os.stat(filepath)
++ except OSError:
++ #file missing -- ignore
++ continue
++ if not stat.S_ISREG(st.st_mode):
++ #ignore non files
++ continue
++ #check size and mtime
++ if st.st_size != size:
++ if self.opts.get('verbose'):
++ print _("Size (%i -> %i) changed for file %s") % (size,st.st_size,filepath)
++ continue
++ if int(st.st_mtime) != mtime:
++ if self.opts.get('verbose'):
++ print _("Modification time changed for %s") % filepath
++ continue
++
++ self.pkg_tups_by_path[relpath] = thispo.pkgtup
+
+- def getNodes(self, relpath):
+- """Return base, filelist, and other nodes for file, if they exist
+
+- Returns a tuple of nodes, or None if not found
++
++ def getNodes(self, relpath):
++ """return a package object based on relative path of pkg
+ """
+- bnode = self.basenodes.get(relpath,None)
+- if bnode is None:
+- return None
+- pkgid = self.pkg_ids.get(relpath,None)
+- if pkgid is None:
+- print _("No pkgid found for: %s") % relpath
+- return None
+- fnode = self.filesnodes.get(pkgid,None)
+- if fnode is None:
+- return None
+- onode = self.othernodes.get(pkgid,None)
+- if onode is None:
+- return None
+- return bnode, fnode, onode
+-
+- def freeNodes(self,relpath):
+- #causing problems
+- """Free up nodes corresponding to file, if possible"""
+- bnode = self.basenodes.get(relpath,None)
+- if bnode is None:
+- print "Missing node for %s" % relpath
+- return
+- bnode.unlinkNode()
+- bnode.freeNode()
+- del self.basenodes[relpath]
+- pkgid = self.pkg_ids.get(relpath,None)
+- if pkgid is None:
+- print _("No pkgid found for: %s") % relpath
++ if relpath in self.pkg_tups_by_path:
++ pkgtup = self.pkg_tups_by_path[relpath]
++ return self._repo.sack.searchPkgTuple(pkgtup)[0]
++ else:
++ print _("No pkg found for: %s") % relpath
+ return None
+- del self.pkg_ids[relpath]
+- dups = self.pkgrefs.get(pkgid)
+- dups.remove(relpath)
+- if len(dups):
+- #still referenced
+- return
+- del self.pkgrefs[pkgid]
+- for nodes in self.filesnodes, self.othernodes:
+- node = nodes.get(pkgid)
+- if node is not None:
+- node.unlinkNode()
+- node.freeNode()
+- del nodes[pkgid]
+
++
+
+ if __name__ == "__main__":
+ cwd = os.getcwd()
+@@ -209,9 +118,9 @@ if __name__ == "__main__":
+ 'pkgdir': cwd}
+
+ idx = MetadataIndex(cwd, opts)
+- for fn in idx.basenodes.keys():
+- a,b,c, = idx.getNodes(fn)
+- a.serialize()
+- b.serialize()
+- c.serialize()
+- idx.freeNodes(fn)
++ for fn in idx.pkg_tups_by_path:
++ po = idx.getNodes(fn)
++ print po.xml_dump_primary_metadata()
++ print po.xml_dump_filelists_metadata()
++ print po.xml_dump_other_metadata()
++
+diff --git a/docs/createrepo.8 b/docs/createrepo.8
+index e3c4c3b..96b5bf8 100644
+--- a/docs/createrepo.8
++++ b/docs/createrepo.8
+@@ -53,7 +53,8 @@ gullible).
+ Don't generate repo metadata, if their timestamps are newer than its rpms.
+ This option decreases the processing time drastically again, if you happen
+ to run it on an unmodified repo, but it is (currently) mutual exclusive
+-with the --split option.
++with the --split option. NOTE: This command will not notice when
++packages have been removed from repo. Use --update to handle that.
+ .br
+ .IP "\fB\--split\fP"
+ Run in split media mode. Rather than pass a single directory, take a set of
+diff --git a/genpkgmetadata.py b/genpkgmetadata.py
+index 8c98191..512420b 100755
+--- a/genpkgmetadata.py
++++ b/genpkgmetadata.py
+@@ -100,6 +100,8 @@ def parse_args(args, conf):
+ parser.add_option("--simple-md-filenames", dest="simple_md_filenames",
+ help="do not include the file's checksum in the filename, helps with proxies",
+ default=False, action="store_true")
++ parser.add_option("--retain-old-md", default=0, type='int', dest='retain_old_md',
++ help="keep around the latest (by timestamp) N copies of the old repodata")
+ parser.add_option("--distro", default=[], action="append",
+ help="distro tag and optional cpeid: --distro" "'cpeid,textname'")
+ parser.add_option("--content", default=[], dest='content_tags',
+@@ -240,6 +242,7 @@ def main(args):
+ if mdgen.checkTimeStamps():
+ if mdgen.conf.verbose:
+ print _('repo is up to date')
++ mdgen._cleanup_tmp_repodata_dir()
+ sys.exit(0)
+
+ if conf.profile:
+diff --git a/mergerepo.py b/mergerepo.py
+index 05e5f5e..069a70b 100755
+--- a/mergerepo.py
++++ b/mergerepo.py
+@@ -18,6 +18,7 @@
+
+ import sys
+ import createrepo.merge
++from createrepo.utils import MDError
+ from optparse import OptionParser
+
+ #TODO:
+@@ -77,9 +78,12 @@ def main(args):
+ rmbase.groups = False
+ if opts.noupdateinfo:
+ rmbase.updateinfo = False
+-
+- rmbase.merge_repos()
+- rmbase.write_metadata()
+-
++ try:
++ rmbase.merge_repos()
++ rmbase.write_metadata()
++ except MDError, e:
++ print >> sys.stderr, "Could not merge repos: %s" % e
++ sys.exit(1)
++
+ if __name__ == "__main__":
+ main(sys.argv[1:])
+diff --git a/modifyrepo.py b/modifyrepo.py
+index 17094a4..c3370e8 100755
+--- a/modifyrepo.py
++++ b/modifyrepo.py
+@@ -1,11 +1,15 @@
+ #!/usr/bin/python
+-# This tools is used to insert arbitrary metadata into an RPM repository.
++# This tool is used to manipulate arbitrary metadata in a RPM repository.
+ # Example:
+ # ./modifyrepo.py updateinfo.xml myrepo/repodata
++# or
++# ./modifyrepo.py --remove updateinfo.xml myrepo/repodata
+ # or in Python:
+ # >>> from modifyrepo import RepoMetadata
+ # >>> repomd = RepoMetadata('myrepo/repodata')
+ # >>> repomd.add('updateinfo.xml')
++# or
++# >>> repomd.remove('updateinfo.xml')
+ #
+ # This program is free software; you can redistribute it and/or modify
+ # it under the terms of the GNU General Public License as published by
+@@ -20,6 +24,7 @@
+ # (C) Copyright 2006 Red Hat, Inc.
+ # Luke Macken <lmacken at redhat.com>
+ # modified by Seth Vidal 2008
++# modified by Daniel Mach 2011
+
+ import os
+ import sys
+@@ -49,6 +54,35 @@ class RepoMetadata:
+ except RepoMDError, e:
+ raise MDError, 'Could not parse %s' % self.repomdxml
+
++ def _get_mdtype(self, mdname, mdtype=None):
++ """ Get mdtype from existing mdtype or from a mdname. """
++ if mdtype:
++ return mdtype
++ return mdname.split('.')[0]
++
++ def _print_repodata(self, repodata):
++ """ Print repodata details. """
++ print " type =", repodata.type
++ print " location =", repodata.location[1]
++ print " checksum =", repodata.checksum[1]
++ print " timestamp =", repodata.timestamp
++ print " open-checksum =", repodata.openchecksum[1]
++
++ def _write_repomd(self):
++ """ Write the updated repomd.xml. """
++ outmd = file(self.repomdxml, 'w')
++ outmd.write(self.repoobj.dump_xml())
++ outmd.close()
++ print "Wrote:", self.repomdxml
++
++ def _remove_repodata_file(self, repodata):
++ """ Remove a file specified in repodata location """
++ try:
++ os.remove(repodata.location[1])
++ except OSError, ex:
++ if ex.errno != 2:
++ # continue on a missing file
++ raise MDError("could not remove file %s" % repodata.location[1])
+
+ def add(self, metadata, mdtype=None):
+ """ Insert arbitrary metadata into this repository.
+@@ -78,9 +112,8 @@ class RepoMetadata:
+ ## Compress the metadata and move it into the repodata
+ if not mdname.endswith('.gz'):
+ mdname += '.gz'
+- if not mdtype:
+- mdtype = mdname.split('.')[0]
+-
++ mdtype = self._get_mdtype(mdname, mdtype)
++
+ destmd = os.path.join(self.repodir, mdname)
+ newmd = GzipFile(filename=destmd, mode='wb')
+ newmd.write(md)
+@@ -91,11 +124,8 @@ class RepoMetadata:
+ csum, destmd = checksum_and_rename(destmd, self.checksum_type)
+ base_destmd = os.path.basename(destmd)
+
+-
+- ## Remove any stale metadata
+- if mdtype in self.repoobj.repoData:
+- del self.repoobj.repoData[mdtype]
+-
++ # Remove any stale metadata
++ old_rd = self.repoobj.repoData.pop(mdtype, None)
+
+ new_rd = RepoData()
+ new_rd.type = mdtype
+@@ -105,18 +135,28 @@ class RepoMetadata:
+ new_rd.size = str(os.stat(destmd).st_size)
+ new_rd.timestamp = str(os.stat(destmd).st_mtime)
+ self.repoobj.repoData[new_rd.type] = new_rd
+-
+- print " type =", new_rd.type
+- print " location =", new_rd.location[1]
+- print " checksum =", new_rd.checksum[1]
+- print " timestamp =", new_rd.timestamp
+- print " open-checksum =", new_rd.openchecksum[1]
+-
+- ## Write the updated repomd.xml
+- outmd = file(self.repomdxml, 'w')
+- outmd.write(self.repoobj.dump_xml())
+- outmd.close()
+- print "Wrote:", self.repomdxml
++ self._print_repodata(new_rd)
++ self._write_repomd()
++
++ if old_rd is not None and old_rd.location[1] != new_rd.location[1]:
++ # remove the old file when overwriting metadata
++ # with the same mdtype but different location
++ self._remove_repodata_file(old_rd)
++
++ def remove(self, metadata, mdtype=None):
++ """ Remove metadata from this repository. """
++ mdname = metadata
++ mdtype = self._get_mdtype(mdname, mdtype)
++
++ old_rd = self.repoobj.repoData.pop(mdtype, None)
++ if old_rd is None:
++ print "Metadata not found: %s" % mdtype
++ return
++
++ self._remove_repodata_file(old_rd)
++ print "Removed:"
++ self._print_repodata(old_rd)
++ self._write_repomd()
+
+
+ def main(args):
+@@ -124,7 +164,9 @@ def main(args):
+ # query options
+ parser.add_option("--mdtype", dest='mdtype',
+ help="specific datatype of the metadata, will be derived from the filename if not specified")
+- parser.usage = "modifyrepo [options] <input_metadata> <output repodata>"
++ parser.add_option("--remove", action="store_true",
++ help="remove specified file from repodata")
++ parser.usage = "modifyrepo [options] [--remove] <input_metadata> <output repodata>"
+
+ (opts, argsleft) = parser.parse_args(args)
+ if len(argsleft) != 2:
+@@ -137,6 +179,17 @@ def main(args):
+ except MDError, e:
+ print "Could not access repository: %s" % str(e)
+ return 1
++
++ # remove
++ if opts.remove:
++ try:
++ repomd.remove(metadata)
++ except MDError, ex:
++ print "Could not remove metadata: %s" % (metadata, str(ex))
++ return 1
++ return
++
++ # add
+ try:
+ repomd.add(metadata, mdtype=opts.mdtype)
+ except MDError, e:
diff --git a/worker.py b/worker.py
-index eb35ef7..591a922 100755
+index eb35ef7..ab78d90 100755
--- a/worker.py
+++ b/worker.py
-@@ -83,8 +83,10 @@ def main(args):
- external_data=external_data)
+@@ -80,11 +80,14 @@ def main(args):
+ print "reading %s" % (pkgfile)
+
+ pkg = createrepo.yumbased.CreateRepoPackage(ts, package=pkgpath,
+- external_data=external_data)
++ sumtype=globalopts.get('sumtype', None),
++ external_data=external_data)
pri.write(pkg.xml_dump_primary_metadata())
fl.write(pkg.xml_dump_filelists_metadata())
- other.write(pkg.xml_dump_other_metadata(clog_limit=
diff --git a/createrepo.spec b/createrepo.spec
index 42432ed..0177d04 100644
--- a/createrepo.spec
+++ b/createrepo.spec
@@ -3,17 +3,17 @@
Summary: Creates a common metadata repository
Name: createrepo
Version: 0.9.9
-Release: 3%{?dist}
+Release: 4%{?dist}
License: GPLv2
Group: System Environment/Base
Source: %{name}-%{version}.tar.gz
-Patch0: ten-changelog-limit.patch
-Patch1: createrepo-head.patch
+Patch0: createrepo-head.patch
+Patch1: ten-changelog-limit.patch
URL: http://createrepo.baseurl.org/
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
BuildArchitectures: noarch
Requires: python >= 2.1, rpm-python, rpm >= 4.1.1, libxml2-python
-Requires: yum-metadata-parser, yum >= 3.2.29-1, python-deltarpm, deltarpm
+Requires: yum-metadata-parser, yum >= 3.2.29-8, python-deltarpm, deltarpm
BuildRequires: python
%description
@@ -22,8 +22,8 @@ packages.
%prep
%setup -q
-%patch0 -p0
-%patch1 -p1
+%patch0 -p1
+%patch1 -p0
%build
@@ -47,6 +47,12 @@ rm -rf $RPM_BUILD_ROOT
%{python_sitelib}/createrepo
%changelog
+* Fri Jul 29 2011 Seth Vidal <skvidal at fedoraproject.org> - 0.9.9-4
+- latest upstream
+- fixes bugs: 713747, 581632, 581628
+- latest upstream head
+- change --update to use sqlite for old repodata
+
* Tue Feb 08 2011 Fedora Release Engineering <rel-eng at lists.fedoraproject.org> - 0.9.9-3
- Rebuilt for https://fedoraproject.org/wiki/Fedora_15_Mass_Rebuild
More information about the scm-commits
mailing list