[createrepo] latest upstream
Seth Vidal
skvidal at fedoraproject.org
Mon Oct 24 17:48:13 UTC 2011
commit 24bc9937db54ba84e1d9ea4ae76e1feac1eb88ac
Author: Seth Vidal <skvidal at fedoraproject.org>
Date: Mon Oct 24 13:48:11 2011 -0400
latest upstream
createrepo-head.patch | 675 ++++++++++++++++++++++++++++++++++++++++++++-----
createrepo.spec | 6 +-
2 files changed, 620 insertions(+), 61 deletions(-)
---
diff --git a/createrepo-head.patch b/createrepo-head.patch
index 14e40e6..e5a8ebc 100644
--- a/createrepo-head.patch
+++ b/createrepo-head.patch
@@ -1,8 +1,27 @@
diff --git a/createrepo.bash b/createrepo.bash
-index 54ac8b2..4222fa0 100644
+index 54ac8b2..f0695ba 100644
--- a/createrepo.bash
+++ b/createrepo.bash
-@@ -30,6 +30,10 @@ _cr_createrepo()
+@@ -1,11 +1,17 @@
+ # bash completion for createrepo and friends
+
++_cr_compress_type()
++{
++ COMPREPLY=( $( compgen -W "$( ${1:-createrepo} --compress-type=FOO / 2>&1 \
++ | sed -ne 's/,/ /g' -ne 's/.*[Cc]ompression.*://p' )" -- "$2" ) )
++}
++
+ _cr_createrepo()
+ {
+ COMPREPLY=()
+
+ case $3 in
+- --version|-h|--help|-u|--baseurl|--distro|--content|--repo|--workers|\
++ --version|-h|--help|-u|--baseurl|--distro|--content|--repo|\
+ --revision|-x|--excludes|--changelog-limit|--max-delta-rpm-size)
+ return 0
+ ;;
+@@ -30,10 +36,24 @@ _cr_createrepo()
COMPREPLY=( $( compgen -f -o plusdirs -X '!*.rpm' -- "$2" ) )
return 0
;;
@@ -13,22 +32,102 @@ index 54ac8b2..4222fa0 100644
--num-deltas)
COMPREPLY=( $( compgen -W '1 2 3 4 5 6 7 8 9' -- "$2" ) )
return 0
-@@ -42,8 +46,8 @@ _cr_createrepo()
+ ;;
++ --workers)
++ local min=2 max=$( getconf _NPROCESSORS_ONLN 2>/dev/null )
++ [[ -z $max || $max -lt $min ]] && max=$min
++ COMPREPLY=( $( compgen -W "{1..$max}" -- "$2" ) )
++ return 0
++ ;;
++ --compress-type)
++ _cr_compress_type "$1" "$2"
++ return 0
++ ;;
+ esac
+
+ if [[ $2 == -* ]] ; then
+@@ -42,9 +62,9 @@ _cr_createrepo()
--cachedir --checkts --no-database --update --update-md-path
--skip-stat --split --pkglist --includepkg --outputdir
--skip-symlinks --changelog-limit --unique-md-filenames
- --simple-md-filenames --distro --content --repo --revision --deltas
- --oldpackagedirs --num-deltas --read-pkgs-list
+- --max-delta-rpm-size --workers' -- "$2" ) )
+ --simple-md-filenames --retain-old-md --distro --content --repo
+ --revision --deltas --oldpackagedirs --num-deltas --read-pkgs-list
- --max-delta-rpm-size --workers' -- "$2" ) )
++ --max-delta-rpm-size --workers --xz --compress-type' -- "$2" ) )
else
COMPREPLY=( $( compgen -d -- "$2" ) )
+ fi
+@@ -63,10 +83,14 @@ _cr_mergerepo()
+ COMPREPLY=( $( compgen -d -- "$2" ) )
+ return 0
+ ;;
++ --compress-type)
++ _cr_compress_type "" "$2"
++ return 0
++ ;;
+ esac
+
+ COMPREPLY=( $( compgen -W '--version --help --repo --archlist --no-database
+- --outputdir --nogroups --noupdateinfo' -- "$2" ) )
++ --outputdir --nogroups --noupdateinfo --compress-type' -- "$2" ) )
+ } &&
+ complete -F _cr_mergerepo -o filenames mergerepo mergerepo.py
+
+@@ -78,17 +102,22 @@ _cr_modifyrepo()
+ --version|-h|--help|--mdtype)
+ return 0
+ ;;
++ --compress-type)
++ _cr_compress_type "" "$2"
++ return 0
++ ;;
+ esac
+
+ if [[ $2 == -* ]] ; then
+- COMPREPLY=( $( compgen -W '--version --help --mdtype' -- "$2" ) )
++ COMPREPLY=( $( compgen -W '--version --help --mdtype --remove
++ --compress --compress-type' -- "$2" ) )
+ return 0
+ fi
+
+ local i argnum=1
+ for (( i=1; i < ${#COMP_WORDS[@]}-1; i++ )) ; do
+ if [[ ${COMP_WORDS[i]} != -* &&
+- ${COMP_WORDS[i-1]} != @(=|--mdtype) ]]; then
++ ${COMP_WORDS[i-1]} != @(=|--@(md|compress-)type) ]]; then
+ argnum=$(( argnum+1 ))
+ fi
+ done
+diff --git a/createrepo.spec b/createrepo.spec
+index 1e491cd..eea7092 100644
+--- a/createrepo.spec
++++ b/createrepo.spec
+@@ -11,7 +11,7 @@ URL: http://createrepo.baseurl.org/
+ BuildRoot: %{_tmppath}/%{name}-%{version}root
+ BuildArchitectures: noarch
+ Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python
+-Requires: yum-metadata-parser, yum >= 3.2.29, python-deltarpm
++Requires: yum-metadata-parser, yum >= 3.2.29, python-deltarpm, pyliblzma
+
+ %description
+ This utility will generate a common metadata repository from a directory of
+@@ -43,6 +43,9 @@ make DESTDIR=$RPM_BUILD_ROOT sysconfdir=%{_sysconfdir} install
+ %{python_sitelib}/createrepo
+
+ %changelog
++* Fri Sep 9 2011 Seth Vidal <skvidal at fedoraproject.org>
++- add lzma dep
++
+ * Wed Jan 26 2011 Seth Vidal <skvidal at fedoraproject.org>
+ - bump to 0.9.9
+ - add worker.py
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
-index 8f2538e..30f7422 100644
+index 8f2538e..61694f9 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
-@@ -27,11 +27,11 @@ import stat
+@@ -27,14 +27,14 @@ import stat
import fcntl
import subprocess
@@ -42,16 +141,125 @@ index 8f2538e..30f7422 100644
+from yum.packages import YumAvailablePackage
import rpmUtils.transaction
- from utils import _, errorprint, MDError
-@@ -110,6 +110,7 @@ class MetaDataConfig(object):
+-from utils import _, errorprint, MDError
++from utils import _, errorprint, MDError, lzma, _available_compression
+ import readMetadata
+ try:
+ import sqlite3 as sqlite
+@@ -46,7 +46,7 @@ try:
+ except ImportError:
+ pass
+
+-from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, \
++from utils import _gzipOpen, compressFile, compressOpen, checkAndMakeDir, GzipFile, \
+ checksum_and_rename, split_list_into_equal_chunks
+ import deltarpms
+
+@@ -74,7 +74,7 @@ class MetaDataConfig(object):
+ self.deltadir = None
+ self.delta_relative = 'drpms/'
+ self.oldpackage_paths = [] # where to look for the old packages -
+- self.deltafile = 'prestodelta.xml.gz'
++ self.deltafile = 'prestodelta.xml'
+ self.num_deltas = 1 # number of older versions to delta (max)
+ self.max_delta_rpm_size = 100000000
+ self.update_md_path = None
+@@ -86,9 +86,9 @@ class MetaDataConfig(object):
+ self.skip_symlinks = False
+ self.pkglist = []
+ self.database_only = False
+- self.primaryfile = 'primary.xml.gz'
+- self.filelistsfile = 'filelists.xml.gz'
+- self.otherfile = 'other.xml.gz'
++ self.primaryfile = 'primary.xml'
++ self.filelistsfile = 'filelists.xml'
++ self.otherfile = 'other.xml'
+ self.repomdfile = 'repomd.xml'
+ self.tempdir = '.repodata'
+ self.finaldir = 'repodata'
+@@ -108,8 +108,10 @@ class MetaDataConfig(object):
+ self.collapse_glibc_requires = True
+ self.workers = 1 # number of workers to fork off to grab metadata from the pkgs
self.worker_cmd = '/usr/share/createrepo/worker.py'
-
+-
#self.worker_cmd = './worker.py' # helpful when testing
+ self.retain_old_md = 0
++ self.compress_type = 'gz'
++
class SimpleMDCallBack(object):
def errorlog(self, thing):
-@@ -530,39 +531,19 @@ class MetaDataGenerator:
+@@ -145,6 +147,14 @@ class MetaDataGenerator:
+ if not self.conf.directory and not self.conf.directories:
+ raise MDError, "No directory given on which to run."
+
++ if not self.conf.compress_type:
++ self.conf.compress_type = 'gz'
++
++ if self.conf.compress_type not in utils._available_compression:
++ raise MDError, "Compression %s not available: Please choose from: %s" \
++ % (self.conf.compress_type, ', '.join(utils._available_compression))
++
++
+ if not self.conf.directories: # just makes things easier later
+ self.conf.directories = [self.conf.directory]
+ if not self.conf.directory: # ensure we have both in the config object
+@@ -410,9 +420,11 @@ class MetaDataGenerator:
+
+ def _setupPrimary(self):
+ # setup the primary metadata file
++ # FIXME - make this be conf.compress_type once y-m-p is fixed
++ fpz = self.conf.primaryfile + '.' + 'gz'
+ primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
+- self.conf.primaryfile)
+- fo = _gzipOpen(primaryfilepath, 'w')
++ fpz)
++ fo = compressOpen(primaryfilepath, 'w', 'gz')
+ fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+ fo.write('<metadata xmlns="http://linux.duke.edu/metadata/common"' \
+ ' xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">' %
+@@ -421,9 +433,11 @@ class MetaDataGenerator:
+
+ def _setupFilelists(self):
+ # setup the filelist file
++ # FIXME - make this be conf.compress_type once y-m-p is fixed
++ fpz = self.conf.filelistsfile + '.' + 'gz'
+ filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir,
+- self.conf.filelistsfile)
+- fo = _gzipOpen(filelistpath, 'w')
++ fpz)
++ fo = compressOpen(filelistpath, 'w', 'gz')
+ fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+ fo.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists"' \
+ ' packages="%s">' % self.pkgcount)
+@@ -431,9 +445,11 @@ class MetaDataGenerator:
+
+ def _setupOther(self):
+ # setup the other file
++ # FIXME - make this be conf.compress_type once y-m-p is fixed
++ fpz = self.conf.otherfile + '.' + 'gz'
+ otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
+- self.conf.otherfile)
+- fo = _gzipOpen(otherfilepath, 'w')
++ fpz)
++ fo = compressOpen(otherfilepath, 'w', 'gz')
+ fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+ fo.write('<otherdata xmlns="http://linux.duke.edu/metadata/other"' \
+ ' packages="%s">' %
+@@ -442,9 +458,10 @@ class MetaDataGenerator:
+
+ def _setupDelta(self):
+ # setup the other file
++ fpz = self.conf.deltafile + '.' + self.conf.compress_type
+ deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
+- self.conf.deltafile)
+- fo = _gzipOpen(deltafilepath, 'w')
++ fpz)
++ fo = compressOpen(deltafilepath, 'w', self.conf.compress_type)
+ fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+ fo.write('<prestodelta>\n')
+ return fo
+@@ -530,39 +547,19 @@ class MetaDataGenerator:
old_pkg = pkg
if pkg.find("://") != -1:
old_pkg = os.path.basename(pkg)
@@ -101,7 +309,7 @@ index 8f2538e..30f7422 100644
#FIXME - if we're in update and we have deltas enabled
# check the presto data for this pkg and write its info back out
# to our deltafile
-@@ -584,12 +565,12 @@ class MetaDataGenerator:
+@@ -584,12 +581,12 @@ class MetaDataGenerator:
po = None
if isinstance(pkg, YumAvailablePackage):
po = pkg
@@ -116,16 +324,18 @@ index 8f2538e..30f7422 100644
self.read_pkgs.append(pkg)
if po:
-@@ -609,7 +590,7 @@ class MetaDataGenerator:
+@@ -608,8 +605,8 @@ class MetaDataGenerator:
+ # waitfor the workers to finish and as each one comes in
# open the files they created and write them out to our metadata
# add up the total pkg counts and return that value
- worker_tmp_path = tempfile.mkdtemp()
+- worker_tmp_path = tempfile.mkdtemp()
- worker_chunks = utils.split_list_into_equal_chunks(pkgfiles, self.conf.workers)
++ self._worker_tmp_path = tempfile.mkdtemp() # setting this in the base object so we can clean it up later
+ worker_chunks = split_list_into_equal_chunks(pkgfiles, self.conf.workers)
worker_cmd_dict = {}
worker_jobs = {}
base_worker_cmdline = [self.conf.worker_cmd,
-@@ -617,7 +598,8 @@ class MetaDataGenerator:
+@@ -617,7 +614,8 @@ class MetaDataGenerator:
'--pkgoptions=_collapse_libc_requires=%s' % self.conf.collapse_glibc_requires,
'--pkgoptions=_cachedir=%s' % self.conf.cachedir,
'--pkgoptions=_baseurl=%s' % self.conf.baseurl,
@@ -135,7 +345,30 @@ index 8f2538e..30f7422 100644
if self.conf.quiet:
base_worker_cmdline.append('--quiet')
-@@ -660,7 +642,12 @@ class MetaDataGenerator:
+@@ -626,15 +624,20 @@ class MetaDataGenerator:
+ base_worker_cmdline.append('--verbose')
+
+ for worker_num in range(self.conf.workers):
++ pkl = self._worker_tmp_path + '/pkglist-%s' % worker_num
++ f = open(pkl, 'w')
++ f.write('\n'.join(worker_chunks[worker_num]))
++ f.close()
++
+ # make the worker directory
+ workercmdline = []
+ workercmdline.extend(base_worker_cmdline)
+- thisdir = worker_tmp_path + '/' + str(worker_num)
++ thisdir = self._worker_tmp_path + '/' + str(worker_num)
+ if checkAndMakeDir(thisdir):
+ workercmdline.append('--tmpmdpath=%s' % thisdir)
+ else:
+ raise MDError, "Unable to create worker path: %s" % thisdir
+- workercmdline.extend(worker_chunks[worker_num])
++ workercmdline.append('--pkglist=%s/pkglist-%s' % (self._worker_tmp_path, worker_num))
+ worker_cmd_dict[worker_num] = workercmdline
+
+
+@@ -660,7 +663,12 @@ class MetaDataGenerator:
if line:
self.callback.errorlog('Worker %s: %s' % (num, line.rstrip()))
@@ -149,7 +382,16 @@ index 8f2538e..30f7422 100644
if not self.conf.quiet:
self.callback.log("Workers Finished")
# finished with workers
-@@ -784,7 +771,6 @@ class MetaDataGenerator:
+@@ -671,7 +679,7 @@ class MetaDataGenerator:
+ for (fn, fo) in (('primary.xml', self.primaryfile),
+ ('filelists.xml', self.flfile),
+ ('other.xml', self.otherfile)):
+- fnpath = worker_tmp_path + '/' + str(num) + '/' + fn
++ fnpath = self._worker_tmp_path + '/' + str(num) + '/' + fn
+ if os.path.exists(fnpath):
+ fo.write(open(fnpath, 'r').read())
+
+@@ -784,7 +792,6 @@ class MetaDataGenerator:
return self._old_package_dict
self._old_package_dict = {}
@@ -157,7 +399,38 @@ index 8f2538e..30f7422 100644
for d in self.conf.oldpackage_paths:
for f in self.getFileList(d, '.rpm'):
fp = d + '/' + f
-@@ -874,7 +860,6 @@ class MetaDataGenerator:
+@@ -833,7 +840,7 @@ class MetaDataGenerator:
+ return ' '.join(results)
+
+ def _createRepoDataObject(self, mdfile, mdtype, compress=True,
+- compress_type='gzip', attribs={}):
++ compress_type=None, attribs={}):
+ """return random metadata as RepoData object to be added to RepoMD
+ mdfile = complete path to file
+ mdtype = the metadata type to use
+@@ -843,15 +850,13 @@ class MetaDataGenerator:
+ sfile = os.path.basename(mdfile)
+ fo = open(mdfile, 'r')
+ outdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
++ if not compress_type:
++ compress_type = self.conf.compress_type
+ if compress:
+- if compress_type == 'gzip':
+- sfile = '%s.gz' % sfile
+- outfn = os.path.join(outdir, sfile)
+- output = GzipFile(filename = outfn, mode='wb')
+- elif compress_type == 'bzip2':
+- sfile = '%s.bz2' % sfile
+- outfn = os.path.join(outdir, sfile)
+- output = BZ2File(filename = outfn, mode='wb')
++ sfile = '%s.%s' % (sfile, compress_type)
++ outfn = os.path.join(outdir, sfile)
++ output = compressOpen(outfn, mode='wb', compress_type=compress_type)
++
+ else:
+ outfn = os.path.join(outdir, sfile)
+ output = open(outfn, 'w')
+@@ -874,7 +879,6 @@ class MetaDataGenerator:
thisdata = RepoData()
thisdata.type = mdtype
@@ -165,7 +438,49 @@ index 8f2538e..30f7422 100644
thisdata.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, sfile))
thisdata.checksum = (self.conf.sumtype, csum)
if compress:
-@@ -1046,7 +1031,7 @@ class MetaDataGenerator:
+@@ -925,9 +929,13 @@ class MetaDataGenerator:
+ rp = sqlitecachec.RepodataParserSqlite(repopath, repomd.repoid, None)
+
+ for (rpm_file, ftype) in workfiles:
++ # when we fix y-m-p and non-gzipped xml files - then we can make this just add
++ # self.conf.compress_type
++ if ftype in ('other', 'filelists', 'primary'):
++ rpm_file = rpm_file + '.' + 'gz'
+ complete_path = os.path.join(repopath, rpm_file)
+
+- zfo = _gzipOpen(complete_path)
++ zfo = compressOpen(complete_path)
+ # This is misc.checksum() done locally so we can get the size too.
+ data = misc.Checksums([sumtype])
+ while data.read(zfo, 2**16):
+@@ -968,12 +976,13 @@ class MetaDataGenerator:
+
+ # rename from silly name to not silly name
+ os.rename(tmp_result_path, resultpath)
+- compressed_name = '%s.bz2' % good_name
++ ext = self.conf.compress_type
++ compressed_name = '%s.%s' % (good_name, ext)
+ result_compressed = os.path.join(repopath, compressed_name)
+ db_csums[ftype] = misc.checksum(sumtype, resultpath)
+
+ # compress the files
+- bzipFile(resultpath, result_compressed)
++ compressFile(resultpath, result_compressed, self.conf.compress_type)
+ # csum the compressed file
+ db_compressed_sums[ftype] = misc.checksum(sumtype,
+ result_compressed)
+@@ -983,8 +992,8 @@ class MetaDataGenerator:
+ os.unlink(resultpath)
+
+ if self.conf.unique_md_filenames:
+- csum_compressed_name = '%s-%s.bz2' % (
+- db_compressed_sums[ftype], good_name)
++ csum_compressed_name = '%s-%s.%s' % (
++ db_compressed_sums[ftype], good_name, ext)
+ csum_result_compressed = os.path.join(repopath,
+ csum_compressed_name)
+ os.rename(result_compressed, csum_result_compressed)
+@@ -1046,7 +1055,7 @@ class MetaDataGenerator:
if self.conf.additional_metadata:
@@ -174,7 +489,7 @@ index 8f2538e..30f7422 100644
mdcontent = self._createRepoDataObject(md_file, md_type)
repomd.repoData[mdcontent.type] = mdcontent
-@@ -1110,23 +1095,42 @@ class MetaDataGenerator:
+@@ -1110,23 +1119,43 @@ class MetaDataGenerator:
raise MDError, _(
'Could not remove old metadata file: %s: %s') % (oldfile, e)
@@ -199,11 +514,15 @@ index 8f2538e..30f7422 100644
- 'other.xml.gz','filelists.xml.gz'):
- os.remove(oldfile) # kill off the old ones
- continue
+- if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2',
+- 'primary.sqlite.bz2'):
+- os.remove(oldfile)
+
-+ for (end,lst) in (('-primary.sqlite.bz2', old_pr_db), ('-primary.xml.gz', old_pr),
-+ ('-filelists.sqlite.bz2', old_fl_db), ('-filelists.xml.gz', old_fl),
-+ ('-other.sqlite.bz2', old_ot_db), ('-other.xml.gz', old_ot)):
-+ if f.endswith(end):
++ for (end,lst) in (('-primary.sqlite', old_pr_db), ('-primary.xml', old_pr),
++ ('-filelists.sqlite', old_fl_db), ('-filelists.xml', old_fl),
++ ('-other.sqlite', old_ot_db), ('-other.xml', old_ot)):
++ fn = '.'.join(f.split('.')[:-1])
++ if fn.endswith(end):
+ lst.append(oldfile)
+ break
+
@@ -217,11 +536,9 @@ index 8f2538e..30f7422 100644
+ for f in os.listdir(output_old_dir):
+ oldfile = os.path.join(output_old_dir, f)
+ finalfile = os.path.join(output_final_dir, f)
-+
- if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2',
-- 'primary.sqlite.bz2'):
-- os.remove(oldfile)
-+ 'primary.sqlite.bz2') or oldfile in old_to_remove:
++ fn = '.'.join(f.split('.')[:-1])
++ if fn in ('filelists.sqlite', 'other.sqlite.',
++ 'primary.sqlite') or oldfile in old_to_remove:
+ try:
+ os.remove(oldfile)
+ except (OSError, IOError), e:
@@ -230,7 +547,7 @@ index 8f2538e..30f7422 100644
continue
if os.path.exists(finalfile):
-@@ -1147,14 +1151,11 @@ class MetaDataGenerator:
+@@ -1147,14 +1176,19 @@ class MetaDataGenerator:
msg += _('Error was %s') % e
raise MDError, msg
@@ -242,14 +559,22 @@ index 8f2538e..30f7422 100644
- self.errorlog(_('Error was %s') % e)
- self.errorlog(_('Please clean up this directory manually.'))
+ self._cleanup_tmp_repodata_dir()
++ self._cleanup_update_tmp_dir()
+ self._write_out_read_pkgs_list()
-+
++
++ def _cleanup_update_tmp_dir(self):
++ if not self.conf.update:
++ return
++
++ shutil.rmtree(self.oldData._repo.basecachedir, ignore_errors=True)
++ shutil.rmtree(self.oldData._repo.base_persistdir, ignore_errors=True)
++
+ def _write_out_read_pkgs_list(self):
# write out the read_pkgs_list file with self.read_pkgs
if self.conf.read_pkgs_list:
try:
-@@ -1167,6 +1168,20 @@ class MetaDataGenerator:
+@@ -1167,6 +1201,23 @@ class MetaDataGenerator:
% self.conf.read_pkgs_list)
self.errorlog(_('Error was %s') % e)
@@ -266,11 +591,14 @@ index 8f2538e..30f7422 100644
+ % dirbase)
+ self.errorlog(_('Error was %s') % e)
+ self.errorlog(_('Please clean up this directory manually.'))
++ # our worker tmp path
++ if hasattr(self, '_worker_tmp_path') and os.path.exists(self._worker_tmp_path):
++ shutil.rmtree(self._worker_tmp_path, ignore_errors=True)
+
def setup_sqlite_dbs(self, initdb=True):
"""sets up the sqlite dbs w/table schemas and db_infos"""
destdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
-@@ -1240,7 +1255,6 @@ class SplitMetaDataGenerator(MetaDataGenerator):
+@@ -1240,7 +1291,6 @@ class SplitMetaDataGenerator(MetaDataGenerator):
self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
try:
self.openMetadataDocs()
@@ -334,7 +662,7 @@ index b3b2ea1..1ac43bb 100644
if self.updateinfo:
ui_fn = mytempdir + '/updateinfo.xml'
diff --git a/createrepo/readMetadata.py b/createrepo/readMetadata.py
-index 27d3690..88e5d95 100644
+index 27d3690..54863cb 100644
--- a/createrepo/readMetadata.py
+++ b/createrepo/readMetadata.py
@@ -16,11 +16,25 @@
@@ -365,7 +693,7 @@ index 27d3690..88e5d95 100644
class MetadataIndex(object):
-@@ -30,178 +44,73 @@ class MetadataIndex(object):
+@@ -30,178 +44,72 @@ class MetadataIndex(object):
opts = {}
self.opts = opts
self.outputdir = outputdir
@@ -390,6 +718,7 @@ index 27d3690..88e5d95 100644
+ self._repo = yum.yumRepo.YumRepository('garbageid')
+ self._repo.baseurl = 'file://' + realpath
+ self._repo.basecachedir = tempfile.mkdtemp(dir='/var/tmp', prefix="createrepo")
++ self._repo.base_persistdir = tempfile.mkdtemp(dir='/var/tmp', prefix="createrepo-p")
+ self._repo.metadata_expire = 1
+ self._repo.gpgcheck = 0
+ self._repo.repo_gpgcheck = 0
@@ -538,12 +867,12 @@ index 27d3690..88e5d95 100644
+ continue
+
+ self.pkg_tups_by_path[relpath] = thispo.pkgtup
++
- def getNodes(self, relpath):
- """Return base, filelist, and other nodes for file, if they exist
- Returns a tuple of nodes, or None if not found
-+
+ def getNodes(self, relpath):
+ """return a package object based on relative path of pkg
"""
@@ -575,12 +904,7 @@ index 27d3690..88e5d95 100644
- pkgid = self.pkg_ids.get(relpath,None)
- if pkgid is None:
- print _("No pkgid found for: %s") % relpath
-+ if relpath in self.pkg_tups_by_path:
-+ pkgtup = self.pkg_tups_by_path[relpath]
-+ return self._repo.sack.searchPkgTuple(pkgtup)[0]
-+ else:
-+ print _("No pkg found for: %s") % relpath
- return None
+- return None
- del self.pkg_ids[relpath]
- dups = self.pkgrefs.get(pkgid)
- dups.remove(relpath)
@@ -594,12 +918,16 @@ index 27d3690..88e5d95 100644
- node.unlinkNode()
- node.freeNode()
- del nodes[pkgid]
++ if relpath in self.pkg_tups_by_path:
++ pkgtup = self.pkg_tups_by_path[relpath]
++ return self._repo.sack.searchPkgTuple(pkgtup)[0]
++ return None
+
if __name__ == "__main__":
cwd = os.getcwd()
-@@ -209,9 +118,9 @@ if __name__ == "__main__":
+@@ -209,9 +117,9 @@ if __name__ == "__main__":
'pkgdir': cwd}
idx = MetadataIndex(cwd, opts)
@@ -615,8 +943,91 @@ index 27d3690..88e5d95 100644
+ print po.xml_dump_filelists_metadata()
+ print po.xml_dump_other_metadata()
+
+diff --git a/createrepo/utils.py b/createrepo/utils.py
+index 995c3b9..c816640 100644
+--- a/createrepo/utils.py
++++ b/createrepo/utils.py
+@@ -23,6 +23,12 @@ import bz2
+ import gzip
+ from gzip import write32u, FNAME
+ from yum import misc
++_available_compression = ['gz', 'bz2']
++try:
++ import lzma
++ _available_compression.append('xz')
++except ImportError:
++ lzma = None
+
+ def errorprint(stuff):
+ print >> sys.stderr, stuff
+@@ -69,6 +75,65 @@ def bzipFile(source, dest):
+ s_fn.close()
+
+
++def xzFile(source, dest):
++ if not 'xz' in _available_compression:
++ raise MDError, "Cannot use xz for compression, library/module is not available"
++
++ s_fn = open(source, 'rb')
++ destination = lzma.LZMAFile(dest, 'w')
++
++ while True:
++ data = s_fn.read(1024000)
++
++ if not data: break
++ destination.write(data)
++
++ destination.close()
++ s_fn.close()
++
++def gzFile(source, dest):
++
++ s_fn = open(source, 'rb')
++ destination = GzipFile(dest, 'w')
++
++ while True:
++ data = s_fn.read(1024000)
++
++ if not data: break
++ destination.write(data)
++
++ destination.close()
++ s_fn.close()
++
++
++
++def compressFile(source, dest, compress_type):
++ """Compress an existing file using any compression type from source to dest"""
++
++ if compress_type == 'xz':
++ xzFile(source, dest)
++ elif compress_type == 'bz2':
++ bzipFile(source, dest)
++ elif compress_type == 'gz':
++ gzFile(source, dest)
++ else:
++ raise MDError, "Unknown compression type %s" % compress_type
++
++def compressOpen(fn, mode='rb', compress_type=None):
++
++ if not compress_type:
++ # we are readonly and we don't give a compress_type - then guess based on the file extension
++ compress_type = fn.split('.')[-1]
++
++ if compress_type == 'xz':
++ return lzma.LZMAFile(fn, mode)
++ elif compress_type == 'bz2':
++ return bz2.BZ2File(fn, mode)
++ elif compress_type == 'gz':
++ return _gzipOpen(fn, mode)
++ else:
++ raise MDError, "Unknown compression type %s" % compress_type
++
+ def returnFD(filename):
+ try:
+ fdno = os.open(filename, os.O_RDONLY)
diff --git a/docs/createrepo.8 b/docs/createrepo.8
-index e3c4c3b..96b5bf8 100644
+index e3c4c3b..4734392 100644
--- a/docs/createrepo.8
+++ b/docs/createrepo.8
@@ -53,7 +53,8 @@ gullible).
@@ -629,8 +1040,25 @@ index e3c4c3b..96b5bf8 100644
.br
.IP "\fB\--split\fP"
Run in split media mode. Rather than pass a single directory, take a set of
+@@ -104,7 +105,15 @@ Tells createrepo to generate deltarpms and the delta metadata
+ paths to look for older pkgs to delta against. Can be specified multiple times
+ .IP "\fB\--num-deltas\fP int"
+ the number of older versions to make deltas against. Defaults to 1
+-
++.IP "\fB\--read-pkgs-list\fP READ_PKGS_LIST
++output the paths to the pkgs actually read useful with --update
++.IP "\fB\--max-delta-rpm-size\fP MAX_DELTA_RPM_SIZE
++max size of an rpm that to run deltarpm against (in bytes)
++.IP "\fB\--workers\fP WORKERS
++number of workers to spawn to read rpms
++.IP "\fB\--xz\fP
++use xz for repodata compression
++.IP
+
+ .SH "EXAMPLES"
+ Here is an example of a repository with a groups file. Note that the
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
-index 8c98191..512420b 100755
+index 8c98191..af0ecb4 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -100,6 +100,8 @@ def parse_args(args, conf):
@@ -642,7 +1070,36 @@ index 8c98191..512420b 100755
parser.add_option("--distro", default=[], action="append",
help="distro tag and optional cpeid: --distro" "'cpeid,textname'")
parser.add_option("--content", default=[], dest='content_tags',
-@@ -240,6 +242,7 @@ def main(args):
+@@ -119,10 +121,15 @@ def parse_args(args, conf):
+ parser.add_option("--max-delta-rpm-size", default=100000000,
+ dest='max_delta_rpm_size', type='int',
+ help="max size of an rpm that to run deltarpm against (in bytes)")
+-
+ parser.add_option("--workers", default=1,
+ dest='workers', type='int',
+ help="number of workers to spawn to read rpms")
++ parser.add_option("--xz", default=False,
++ action="store_true",
++ help="use xz for repodata compression")
++ parser.add_option("--compress-type", default=None, dest="compress_type",
++ help="which compression type to use")
++
+
+ (opts, argsleft) = parser.parse_args(args)
+ if len(argsleft) > 1 and not opts.split:
+@@ -155,6 +162,11 @@ def parse_args(args, conf):
+
+ if opts.nodatabase:
+ opts.database = False
++
++ # xz is just a shorthand for compress_type
++ if opts.xz and not opts.compress_type:
++ opts.compress_type='xz'
++
+
+ # let's switch over to using the conf object - put all the opts into it
+ for opt in parser.option_list:
+@@ -240,6 +252,7 @@ def main(args):
if mdgen.checkTimeStamps():
if mdgen.conf.verbose:
print _('repo is up to date')
@@ -651,7 +1108,7 @@ index 8c98191..512420b 100755
if conf.profile:
diff --git a/mergerepo.py b/mergerepo.py
-index 05e5f5e..069a70b 100755
+index 05e5f5e..80cb1a8 100755
--- a/mergerepo.py
+++ b/mergerepo.py
@@ -18,6 +18,7 @@
@@ -662,7 +1119,17 @@ index 05e5f5e..069a70b 100755
from optparse import OptionParser
#TODO:
-@@ -77,9 +78,12 @@ def main(args):
+@@ -47,6 +48,9 @@ def parse_args(args):
+ help="Do not merge group(comps) metadata")
+ parser.add_option("", "--noupdateinfo", default=False, action="store_true",
+ help="Do not merge updateinfo metadata")
++ parser.add_option("--compress-type", default=None, dest="compress_type",
++ help="which compression type to use")
++
+ (opts, argsleft) = parser.parse_args(args)
+
+ if len(opts.repos) < 2:
+@@ -77,9 +81,14 @@ def main(args):
rmbase.groups = False
if opts.noupdateinfo:
rmbase.updateinfo = False
@@ -670,6 +1137,8 @@ index 05e5f5e..069a70b 100755
- rmbase.merge_repos()
- rmbase.write_metadata()
-
++ if opts.compress_type:
++ rmbase.mdconf.compress_type = opts.compress_type
+ try:
+ rmbase.merge_repos()
+ rmbase.write_metadata()
@@ -680,7 +1149,7 @@ index 05e5f5e..069a70b 100755
if __name__ == "__main__":
main(sys.argv[1:])
diff --git a/modifyrepo.py b/modifyrepo.py
-index 17094a4..c3370e8 100755
+index 17094a4..153ad4d 100755
--- a/modifyrepo.py
+++ b/modifyrepo.py
@@ -1,11 +1,15 @@
@@ -700,7 +1169,7 @@ index 17094a4..c3370e8 100755
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
-@@ -20,6 +24,7 @@
+@@ -20,11 +24,12 @@
# (C) Copyright 2006 Red Hat, Inc.
# Luke Macken <lmacken at redhat.com>
# modified by Seth Vidal 2008
@@ -708,7 +1177,22 @@ index 17094a4..c3370e8 100755
import os
import sys
-@@ -49,6 +54,35 @@ class RepoMetadata:
+ from createrepo import __version__
+-from createrepo.utils import checksum_and_rename, GzipFile, MDError
++from createrepo.utils import checksum_and_rename, compressOpen, MDError
+ from yum.misc import checksum
+
+ from yum.repoMDObject import RepoMD, RepoMDError, RepoData
+@@ -39,6 +44,8 @@ class RepoMetadata:
+ self.repodir = os.path.abspath(repo)
+ self.repomdxml = os.path.join(self.repodir, 'repomd.xml')
+ self.checksum_type = 'sha256'
++ self.compress = False
++ self.compress_type='xz'
+
+ if not os.path.exists(self.repomdxml):
+ raise MDError, '%s not found' % self.repomdxml
+@@ -49,6 +56,35 @@ class RepoMetadata:
except RepoMDError, e:
raise MDError, 'Could not parse %s' % self.repomdxml
@@ -744,19 +1228,44 @@ index 17094a4..c3370e8 100755
def add(self, metadata, mdtype=None):
""" Insert arbitrary metadata into this repository.
-@@ -78,9 +112,8 @@ class RepoMetadata:
+@@ -63,8 +99,8 @@ class RepoMetadata:
+ mdname = 'updateinfo.xml'
+ elif isinstance(metadata, str):
+ if os.path.exists(metadata):
+- if metadata.endswith('.gz'):
+- oldmd = GzipFile(filename=metadata, mode='rb')
++ if metadata.split('.')[-1] in ('gz', 'bz2', 'xz'):
++ oldmd = compressOpen(metadata, mode='rb')
+ else:
+ oldmd = file(metadata, 'r')
+ md = oldmd.read()
+@@ -75,14 +111,19 @@ class RepoMetadata:
+ else:
+ raise MDError, 'invalid metadata type'
+
++ do_compress = False
## Compress the metadata and move it into the repodata
- if not mdname.endswith('.gz'):
- mdname += '.gz'
+- if not mdname.endswith('.gz'):
+- mdname += '.gz'
- if not mdtype:
- mdtype = mdname.split('.')[0]
-
++ if self.compress or not mdname.split('.')[-1] in ('gz', 'bz2', 'xz'):
++ do_compress = True
++ mdname += '.' + self.compress_type
+ mdtype = self._get_mdtype(mdname, mdtype)
+
destmd = os.path.join(self.repodir, mdname)
- newmd = GzipFile(filename=destmd, mode='wb')
+- newmd = GzipFile(filename=destmd, mode='wb')
++ if do_compress:
++ newmd = compressOpen(destmd, mode='wb', compress_type=self.compress_type)
++ else:
++ newmd = open(destmd, 'wb')
++
newmd.write(md)
-@@ -91,11 +124,8 @@ class RepoMetadata:
+ newmd.close()
+ print "Wrote:", destmd
+@@ -91,11 +132,8 @@ class RepoMetadata:
csum, destmd = checksum_and_rename(destmd, self.checksum_type)
base_destmd = os.path.basename(destmd)
@@ -770,7 +1279,7 @@ index 17094a4..c3370e8 100755
new_rd = RepoData()
new_rd.type = mdtype
-@@ -105,18 +135,28 @@ class RepoMetadata:
+@@ -105,18 +143,28 @@ class RepoMetadata:
new_rd.size = str(os.stat(destmd).st_size)
new_rd.timestamp = str(os.stat(destmd).st_mtime)
self.repoobj.repoData[new_rd.type] = new_rd
@@ -811,22 +1320,30 @@ index 17094a4..c3370e8 100755
def main(args):
-@@ -124,7 +164,9 @@ def main(args):
+@@ -124,7 +172,13 @@ def main(args):
# query options
parser.add_option("--mdtype", dest='mdtype',
help="specific datatype of the metadata, will be derived from the filename if not specified")
- parser.usage = "modifyrepo [options] <input_metadata> <output repodata>"
+ parser.add_option("--remove", action="store_true",
+ help="remove specified file from repodata")
++ parser.add_option("--compress", action="store_true", default=False,
++ help="compress the new repodata before adding it to the repo")
++ parser.add_option("--compress-type", dest='compress_type', default='xz',
++ help="compression format to use")
+ parser.usage = "modifyrepo [options] [--remove] <input_metadata> <output repodata>"
(opts, argsleft) = parser.parse_args(args)
if len(argsleft) != 2:
-@@ -137,6 +179,17 @@ def main(args):
+@@ -137,11 +191,27 @@ def main(args):
except MDError, e:
print "Could not access repository: %s" % str(e)
return 1
+
++
++ repomd.compress = opts.compress
++ repomd.compress_type = opts.compress_type
++
+ # remove
+ if opts.remove:
+ try:
@@ -840,11 +1357,49 @@ index 17094a4..c3370e8 100755
try:
repomd.add(metadata, mdtype=opts.mdtype)
except MDError, e:
+ print "Could not add metadata from file %s: %s" % (metadata, str(e))
+ return 1
++
+
+ if __name__ == '__main__':
+ ret = main(sys.argv[1:])
diff --git a/worker.py b/worker.py
-index eb35ef7..ab78d90 100755
+index eb35ef7..23c87a3 100755
--- a/worker.py
+++ b/worker.py
-@@ -80,11 +80,14 @@ def main(args):
+@@ -5,6 +5,7 @@ import yum
+ import createrepo
+ import os
+ import rpmUtils
++import re
+ from optparse import OptionParser
+
+
+@@ -23,6 +24,8 @@ def main(args):
+ parser = OptionParser()
+ parser.add_option('--tmpmdpath', default=None,
+ help="path where the outputs should be dumped for this worker")
++ parser.add_option('--pkglist', default=None,
++ help="file to read the pkglist from in lieu of all of them on the cli")
+ parser.add_option("--pkgoptions", default=[], action='append',
+ help="pkgoptions in the format of key=value")
+ parser.add_option("--quiet", default=False, action='store_true',
+@@ -68,7 +71,13 @@ def main(args):
+ fl = open(opts.tmpmdpath + '/filelists.xml' , 'w')
+ other = open(opts.tmpmdpath + '/other.xml' , 'w')
+
+-
++ if opts.pkglist:
++ for line in open(opts.pkglist,'r').readlines():
++ line = line.strip()
++ if re.match('^\s*\#.*', line) or re.match('^\s*$', line):
++ continue
++ pkgs.append(line)
++
+ for pkgfile in pkgs:
+ pkgpath = reldir + '/' + pkgfile
+ if not os.path.exists(pkgpath):
+@@ -80,11 +89,14 @@ def main(args):
print "reading %s" % (pkgfile)
pkg = createrepo.yumbased.CreateRepoPackage(ts, package=pkgpath,
diff --git a/createrepo.spec b/createrepo.spec
index e27cfe8..cbd1a61 100644
--- a/createrepo.spec
+++ b/createrepo.spec
@@ -3,7 +3,7 @@
Summary: Creates a common metadata repository
Name: createrepo
Version: 0.9.9
-Release: 6%{?dist}
+Release: 7%{?dist}
License: GPLv2
Group: System Environment/Base
Source: %{name}-%{version}.tar.gz
@@ -47,6 +47,10 @@ rm -rf $RPM_BUILD_ROOT
%{python_sitelib}/createrepo
%changelog
+* Mon Oct 24 2011 Seth Vidal <skvidal at fedoraproject.org> - 0.9.9-7
+- latest upstream
+- --compress-type among other deals.
+
* Fri Jul 29 2011 Seth Vidal <skvidal at fedoraproject.org> - 0.9.9-6
- latest upstream
- fixes bugs: 713747, 581632, 581628
More information about the scm-commits
mailing list