[createrepo/f16] latest head
Seth Vidal
skvidal at fedoraproject.org
Thu Jan 5 17:24:22 UTC 2012
commit 4c8c02529884488b737e14f7b115b231b90e3d87
Author: Seth Vidal <skvidal at fedoraproject.org>
Date: Thu Jan 5 12:23:46 2012 -0500
latest head
createrepo-head.patch | 157 ++++++++++++++++++++++++++++++++++++-------------
createrepo.spec | 6 ++-
2 files changed, 120 insertions(+), 43 deletions(-)
---
diff --git a/createrepo-head.patch b/createrepo-head.patch
index ee14d25..6827238 100644
--- a/createrepo-head.patch
+++ b/createrepo-head.patch
@@ -124,7 +124,7 @@ index 1e491cd..eea7092 100644
- bump to 0.9.9
- add worker.py
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
-index 8f2538e..014765a 100644
+index 8f2538e..3f55700 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -27,14 +27,14 @@ import stat
@@ -146,16 +146,18 @@ index 8f2538e..014765a 100644
import readMetadata
try:
import sqlite3 as sqlite
-@@ -46,7 +46,7 @@ try:
+@@ -46,8 +46,9 @@ try:
except ImportError:
pass
-from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, \
+from utils import _gzipOpen, compressFile, compressOpen, checkAndMakeDir, GzipFile, \
checksum_and_rename, split_list_into_equal_chunks
++from utils import num_cpus_online
import deltarpms
-@@ -74,7 +74,7 @@ class MetaDataConfig(object):
+ __version__ = '0.9.9'
+@@ -74,7 +75,7 @@ class MetaDataConfig(object):
self.deltadir = None
self.delta_relative = 'drpms/'
self.oldpackage_paths = [] # where to look for the old packages -
@@ -164,7 +166,7 @@ index 8f2538e..014765a 100644
self.num_deltas = 1 # number of older versions to delta (max)
self.max_delta_rpm_size = 100000000
self.update_md_path = None
-@@ -86,9 +86,9 @@ class MetaDataConfig(object):
+@@ -86,9 +87,9 @@ class MetaDataConfig(object):
self.skip_symlinks = False
self.pkglist = []
self.database_only = False
@@ -177,22 +179,32 @@ index 8f2538e..014765a 100644
self.repomdfile = 'repomd.xml'
self.tempdir = '.repodata'
self.finaldir = 'repodata'
-@@ -108,8 +108,10 @@ class MetaDataConfig(object):
+@@ -108,8 +109,10 @@ class MetaDataConfig(object):
self.collapse_glibc_requires = True
self.workers = 1 # number of workers to fork off to grab metadata from the pkgs
self.worker_cmd = '/usr/share/createrepo/worker.py'
-
#self.worker_cmd = './worker.py' # helpful when testing
+ self.retain_old_md = 0
-+ self.compress_type = 'gz'
++ self.compress_type = 'compat'
+
class SimpleMDCallBack(object):
def errorlog(self, thing):
-@@ -145,6 +147,14 @@ class MetaDataGenerator:
+@@ -141,10 +144,23 @@ class MetaDataGenerator:
+ self.files = []
+ self.rpmlib_reqs = {}
+ self.read_pkgs = []
++ self.compat_compress = False
+
if not self.conf.directory and not self.conf.directories:
raise MDError, "No directory given on which to run."
-
+-
++
++ if self.conf.compress_type == 'compat':
++ self.compat_compress = True
++ self.conf.compress_type = None
++
+ if not self.conf.compress_type:
+ self.conf.compress_type = 'gz'
+
@@ -204,7 +216,7 @@ index 8f2538e..014765a 100644
if not self.conf.directories: # just makes things easier later
self.conf.directories = [self.conf.directory]
if not self.conf.directory: # ensure we have both in the config object
-@@ -410,9 +420,11 @@ class MetaDataGenerator:
+@@ -410,9 +426,11 @@ class MetaDataGenerator:
def _setupPrimary(self):
# setup the primary metadata file
@@ -218,7 +230,7 @@ index 8f2538e..014765a 100644
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<metadata xmlns="http://linux.duke.edu/metadata/common"' \
' xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">' %
-@@ -421,9 +433,11 @@ class MetaDataGenerator:
+@@ -421,9 +439,11 @@ class MetaDataGenerator:
def _setupFilelists(self):
# setup the filelist file
@@ -232,7 +244,7 @@ index 8f2538e..014765a 100644
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists"' \
' packages="%s">' % self.pkgcount)
-@@ -431,9 +445,11 @@ class MetaDataGenerator:
+@@ -431,9 +451,11 @@ class MetaDataGenerator:
def _setupOther(self):
# setup the other file
@@ -246,7 +258,7 @@ index 8f2538e..014765a 100644
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<otherdata xmlns="http://linux.duke.edu/metadata/other"' \
' packages="%s">' %
-@@ -442,9 +458,10 @@ class MetaDataGenerator:
+@@ -442,9 +464,10 @@ class MetaDataGenerator:
def _setupDelta(self):
# setup the other file
@@ -259,7 +271,7 @@ index 8f2538e..014765a 100644
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<prestodelta>\n')
return fo
-@@ -530,39 +547,19 @@ class MetaDataGenerator:
+@@ -530,39 +553,19 @@ class MetaDataGenerator:
old_pkg = pkg
if pkg.find("://") != -1:
old_pkg = os.path.basename(pkg)
@@ -309,7 +321,7 @@ index 8f2538e..014765a 100644
#FIXME - if we're in update and we have deltas enabled
# check the presto data for this pkg and write its info back out
# to our deltafile
-@@ -584,12 +581,12 @@ class MetaDataGenerator:
+@@ -584,12 +587,12 @@ class MetaDataGenerator:
po = None
if isinstance(pkg, YumAvailablePackage):
po = pkg
@@ -324,18 +336,20 @@ index 8f2538e..014765a 100644
self.read_pkgs.append(pkg)
if po:
-@@ -608,8 +605,8 @@ class MetaDataGenerator:
+@@ -608,8 +611,10 @@ class MetaDataGenerator:
# waitfor the workers to finish and as each one comes in
# open the files they created and write them out to our metadata
# add up the total pkg counts and return that value
- worker_tmp_path = tempfile.mkdtemp()
- worker_chunks = utils.split_list_into_equal_chunks(pkgfiles, self.conf.workers)
+ self._worker_tmp_path = tempfile.mkdtemp() # setting this in the base object so we can clean it up later
++ if self.conf.workers < 1:
++ self.conf.workers = num_cpus_online()
+ worker_chunks = split_list_into_equal_chunks(pkgfiles, self.conf.workers)
worker_cmd_dict = {}
worker_jobs = {}
base_worker_cmdline = [self.conf.worker_cmd,
-@@ -617,7 +614,8 @@ class MetaDataGenerator:
+@@ -617,7 +622,8 @@ class MetaDataGenerator:
'--pkgoptions=_collapse_libc_requires=%s' % self.conf.collapse_glibc_requires,
'--pkgoptions=_cachedir=%s' % self.conf.cachedir,
'--pkgoptions=_baseurl=%s' % self.conf.baseurl,
@@ -345,7 +359,7 @@ index 8f2538e..014765a 100644
if self.conf.quiet:
base_worker_cmdline.append('--quiet')
-@@ -626,15 +624,20 @@ class MetaDataGenerator:
+@@ -626,15 +632,20 @@ class MetaDataGenerator:
base_worker_cmdline.append('--verbose')
for worker_num in range(self.conf.workers):
@@ -368,7 +382,7 @@ index 8f2538e..014765a 100644
worker_cmd_dict[worker_num] = workercmdline
-@@ -660,7 +663,12 @@ class MetaDataGenerator:
+@@ -660,7 +671,12 @@ class MetaDataGenerator:
if line:
self.callback.errorlog('Worker %s: %s' % (num, line.rstrip()))
@@ -382,7 +396,7 @@ index 8f2538e..014765a 100644
if not self.conf.quiet:
self.callback.log("Workers Finished")
# finished with workers
-@@ -671,15 +679,19 @@ class MetaDataGenerator:
+@@ -671,15 +687,19 @@ class MetaDataGenerator:
for (fn, fo) in (('primary.xml', self.primaryfile),
('filelists.xml', self.flfile),
('other.xml', self.otherfile)):
@@ -405,7 +419,7 @@ index 8f2538e..014765a 100644
self.read_pkgs.append(pkgfile)
return self.current_pkg
-@@ -784,7 +796,6 @@ class MetaDataGenerator:
+@@ -784,7 +804,6 @@ class MetaDataGenerator:
return self._old_package_dict
self._old_package_dict = {}
@@ -413,7 +427,7 @@ index 8f2538e..014765a 100644
for d in self.conf.oldpackage_paths:
for f in self.getFileList(d, '.rpm'):
fp = d + '/' + f
-@@ -833,7 +844,7 @@ class MetaDataGenerator:
+@@ -833,7 +852,7 @@ class MetaDataGenerator:
return ' '.join(results)
def _createRepoDataObject(self, mdfile, mdtype, compress=True,
@@ -422,7 +436,7 @@ index 8f2538e..014765a 100644
"""return random metadata as RepoData object to be added to RepoMD
mdfile = complete path to file
mdtype = the metadata type to use
-@@ -843,15 +854,13 @@ class MetaDataGenerator:
+@@ -843,15 +862,13 @@ class MetaDataGenerator:
sfile = os.path.basename(mdfile)
fo = open(mdfile, 'r')
outdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
@@ -444,7 +458,7 @@ index 8f2538e..014765a 100644
else:
outfn = os.path.join(outdir, sfile)
output = open(outfn, 'w')
-@@ -874,7 +883,6 @@ class MetaDataGenerator:
+@@ -874,7 +891,6 @@ class MetaDataGenerator:
thisdata = RepoData()
thisdata.type = mdtype
@@ -452,7 +466,7 @@ index 8f2538e..014765a 100644
thisdata.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, sfile))
thisdata.checksum = (self.conf.sumtype, csum)
if compress:
-@@ -925,9 +933,14 @@ class MetaDataGenerator:
+@@ -925,9 +941,14 @@ class MetaDataGenerator:
rp = sqlitecachec.RepodataParserSqlite(repopath, repomd.repoid, None)
for (rpm_file, ftype) in workfiles:
@@ -469,34 +483,41 @@ index 8f2538e..014765a 100644
# This is misc.checksum() done locally so we can get the size too.
data = misc.Checksums([sumtype])
while data.read(zfo, 2**16):
-@@ -968,12 +981,13 @@ class MetaDataGenerator:
-
+@@ -966,14 +987,20 @@ class MetaDataGenerator:
+ good_name = '%s.sqlite' % ftype
+ resultpath = os.path.join(repopath, good_name)
+
++ # compat compression for rhel5 compatibility from fedora :(
++ compress_type = self.conf.compress_type
++ if self.compat_compress:
++ compress_type = 'bz2'
++
# rename from silly name to not silly name
os.rename(tmp_result_path, resultpath)
- compressed_name = '%s.bz2' % good_name
-+ ext = self.conf.compress_type
-+ compressed_name = '%s.%s' % (good_name, ext)
++ compressed_name = '%s.%s' % (good_name, compress_type)
result_compressed = os.path.join(repopath, compressed_name)
db_csums[ftype] = misc.checksum(sumtype, resultpath)
# compress the files
- bzipFile(resultpath, result_compressed)
-+ compressFile(resultpath, result_compressed, self.conf.compress_type)
++
++ compressFile(resultpath, result_compressed, compress_type)
# csum the compressed file
db_compressed_sums[ftype] = misc.checksum(sumtype,
result_compressed)
-@@ -983,8 +997,8 @@ class MetaDataGenerator:
+@@ -983,8 +1010,8 @@ class MetaDataGenerator:
os.unlink(resultpath)
if self.conf.unique_md_filenames:
- csum_compressed_name = '%s-%s.bz2' % (
- db_compressed_sums[ftype], good_name)
+ csum_compressed_name = '%s-%s.%s' % (
-+ db_compressed_sums[ftype], good_name, ext)
++ db_compressed_sums[ftype], good_name, compress_type)
csum_result_compressed = os.path.join(repopath,
csum_compressed_name)
os.rename(result_compressed, csum_result_compressed)
-@@ -1020,7 +1034,13 @@ class MetaDataGenerator:
+@@ -1020,7 +1047,13 @@ class MetaDataGenerator:
data.openchecksum = (sumtype, uncsum)
if self.conf.unique_md_filenames:
@@ -511,7 +532,7 @@ index 8f2538e..014765a 100644
orig_file = os.path.join(repopath, rpm_file)
dest_file = os.path.join(repopath, res_file)
os.rename(orig_file, dest_file)
-@@ -1046,7 +1066,7 @@ class MetaDataGenerator:
+@@ -1046,7 +1079,7 @@ class MetaDataGenerator:
if self.conf.additional_metadata:
@@ -520,7 +541,7 @@ index 8f2538e..014765a 100644
mdcontent = self._createRepoDataObject(md_file, md_type)
repomd.repoData[mdcontent.type] = mdcontent
-@@ -1110,23 +1130,43 @@ class MetaDataGenerator:
+@@ -1110,23 +1143,43 @@ class MetaDataGenerator:
raise MDError, _(
'Could not remove old metadata file: %s: %s') % (oldfile, e)
@@ -578,7 +599,7 @@ index 8f2538e..014765a 100644
continue
if os.path.exists(finalfile):
-@@ -1147,14 +1187,19 @@ class MetaDataGenerator:
+@@ -1147,14 +1200,19 @@ class MetaDataGenerator:
msg += _('Error was %s') % e
raise MDError, msg
@@ -592,8 +613,8 @@ index 8f2538e..014765a 100644
+ self._cleanup_tmp_repodata_dir()
+ self._cleanup_update_tmp_dir()
+ self._write_out_read_pkgs_list()
-
+
+
+ def _cleanup_update_tmp_dir(self):
+ if not self.conf.update:
+ return
@@ -605,7 +626,7 @@ index 8f2538e..014765a 100644
# write out the read_pkgs_list file with self.read_pkgs
if self.conf.read_pkgs_list:
try:
-@@ -1167,6 +1212,23 @@ class MetaDataGenerator:
+@@ -1167,6 +1225,23 @@ class MetaDataGenerator:
% self.conf.read_pkgs_list)
self.errorlog(_('Error was %s') % e)
@@ -629,7 +650,7 @@ index 8f2538e..014765a 100644
def setup_sqlite_dbs(self, initdb=True):
"""sets up the sqlite dbs w/table schemas and db_infos"""
destdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
-@@ -1240,7 +1302,6 @@ class SplitMetaDataGenerator(MetaDataGenerator):
+@@ -1240,7 +1315,6 @@ class SplitMetaDataGenerator(MetaDataGenerator):
self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
try:
self.openMetadataDocs()
@@ -975,7 +996,7 @@ index 27d3690..54863cb 100644
+ print po.xml_dump_other_metadata()
+
diff --git a/createrepo/utils.py b/createrepo/utils.py
-index 995c3b9..c5cec64 100644
+index 995c3b9..84a43ba 100644
--- a/createrepo/utils.py
+++ b/createrepo/utils.py
@@ -23,6 +23,12 @@ import bz2
@@ -991,7 +1012,36 @@ index 995c3b9..c5cec64 100644
def errorprint(stuff):
print >> sys.stderr, stuff
-@@ -69,6 +75,67 @@ def bzipFile(source, dest):
+@@ -34,22 +40,14 @@ def _(args):
+
+ class GzipFile(gzip.GzipFile):
+ def _write_gzip_header(self):
++ # Generate a header that is easily reproduced with gzip -9 -n on
++ # an unix-like system
+ self.fileobj.write('\037\213') # magic header
+ self.fileobj.write('\010') # compression method
+- if hasattr(self, 'name'):
+- fname = self.name[:-3]
+- else:
+- fname = self.filename[:-3]
+- flags = 0
+- if fname:
+- flags = FNAME
+- self.fileobj.write(chr(flags))
+- write32u(self.fileobj, long(0))
+- self.fileobj.write('\002')
+- self.fileobj.write('\377')
+- if fname:
+- self.fileobj.write(fname + '\000')
+-
++ self.fileobj.write('\000') # flags
++ write32u(self.fileobj, long(0)) # timestamp
++ self.fileobj.write('\002') # max compression
++ self.fileobj.write('\003') # UNIX
+
+ def _gzipOpen(filename, mode="rb", compresslevel=9):
+ return GzipFile(filename, mode, compresslevel)
+@@ -69,6 +67,67 @@ def bzipFile(source, dest):
s_fn.close()
@@ -1059,6 +1109,29 @@ index 995c3b9..c5cec64 100644
def returnFD(filename):
try:
fdno = os.open(filename, os.O_RDONLY)
+@@ -133,6 +192,22 @@ def split_list_into_equal_chunks(seq, num_chunks):
+
+ return out
+
++def num_cpus_online(unknown=1):
++ if not hasattr(os, "sysconf"):
++ return unknown
++
++ if not os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
++ return unknown
++
++ ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
++ try:
++ if int(ncpus) > 0:
++ return ncpus
++ except:
++ pass
++
++ return unknown
++
+
+ class MDError(Exception):
+ def __init__(self, value=None):
diff --git a/docs/createrepo.8 b/docs/createrepo.8
index e3c4c3b..4734392 100644
--- a/docs/createrepo.8
@@ -1091,7 +1164,7 @@ index e3c4c3b..4734392 100644
.SH "EXAMPLES"
Here is an example of a repository with a groups file. Note that the
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
-index 8c98191..af0ecb4 100755
+index 8c98191..ce8e452 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -100,6 +100,8 @@ def parse_args(args, conf):
@@ -1114,7 +1187,7 @@ index 8c98191..af0ecb4 100755
+ parser.add_option("--xz", default=False,
+ action="store_true",
+ help="use xz for repodata compression")
-+ parser.add_option("--compress-type", default=None, dest="compress_type",
++ parser.add_option("--compress-type", default='compat', dest="compress_type",
+ help="which compression type to use")
+
@@ -1126,7 +1199,7 @@ index 8c98191..af0ecb4 100755
opts.database = False
+
+ # xz is just a shorthand for compress_type
-+ if opts.xz and not opts.compress_type:
++ if opts.xz and opts.compress_type == 'compat':
+ opts.compress_type='xz'
+
diff --git a/createrepo.spec b/createrepo.spec
index 52a3a4c..8cdf05a 100644
--- a/createrepo.spec
+++ b/createrepo.spec
@@ -3,7 +3,7 @@
Summary: Creates a common metadata repository
Name: createrepo
Version: 0.9.9
-Release: 9%{?dist}
+Release: 10%{?dist}
License: GPLv2
Group: System Environment/Base
Source: %{name}-%{version}.tar.gz
@@ -47,6 +47,10 @@ rm -rf $RPM_BUILD_ROOT
%{python_sitelib}/createrepo
%changelog
+* Thu Jan 5 2012 Seth Vidal <skvidal at fedoraproject.org> - 0.9.9-10
+- update to latest head
+- fix for generating repos for rhel5 on fedora
+
* Fri Oct 28 2011 Seth Vidal <skvidal at fedoraproject.org> - 0.9.9-9
- 3rd time is the charm
- fix it so prestodelta's get made with the right name and don't traceback
More information about the scm-commits
mailing list