spacewalk-commits February 2012

spacewalk-commits@lists.fedorahosted.org

13 participants
595 discussions

2 commits - spacewalk/admin
by Jan Pazdziora 03 Feb '12

03 Feb '12

spacewalk/admin/Makefile.admin | 21 +++++++++++++++------ spacewalk/admin/service-list | 4 ++++ spacewalk/admin/spacewalk-admin.spec | 1 + spacewalk/admin/spacewalk-service | 3 +++ 4 files changed, 23 insertions(+), 6 deletions(-) New commits: commit 5ed8779d0e1e4b8e4b73c4f273bdbd934a08472b Author: Jan Pazdziora <jpazdziora(a)redhat.com> Date: Fri Feb 3 14:22:45 2012 +0100 784158 - make it possible to customize list of services managed by the spacewalk-service command. diff --git a/spacewalk/admin/Makefile.admin b/spacewalk/admin/Makefile.admin index feb6f8a..0c2c976 100644 --- a/spacewalk/admin/Makefile.admin +++ b/spacewalk/admin/Makefile.admin @@ -19,12 +19,14 @@ BINDIR = /usr/bin SBINDIR = /usr/sbin +CONFDIR = /etc/rhn GPGKEYDIR = /usr/share/rhn VERSION = $(shell echo `awk '{ print $$1 }' version`) PERL_DEST = $(PREFIX)/$(BINDIR) SBIN_DEST = $(PREFIX)/$(SBINDIR) +CONF_DEST = $(PREFIX)/$(CONFDIR) GPGKEY_DEST = $(PREFIX)/$(GPGKEYDIR) PERL_SCRIPTS = rhn-config-satellite.pl \ @@ -36,23 +38,30 @@ PERL_SCRIPTS = rhn-config-satellite.pl \ SBIN_SCRIPTS = rhn-sat-restart-silent spacewalk-service +CONF_FILES = service-list + GPGKEY_FILES = RHN-GPG-KEY BIN_INSTALL = install -m 755 +CONF_INSTALL = install -m 644 GPGKEY_INSTALL = install -m 644 DIR_INSTALL = install -d -m 755 # install scripts all: -install: $(PERL_SCRIPTS) $(PERL_DEST) $(GPGKEY_FILES) $(GPGKEY_DEST) $(SBIN_SCRIPTS) $(SBIN_DEST) +install: $(PERL_SCRIPTS) $(PERL_DEST) $(CONF_FILES) $(CONF_DEST) $(GPGKEY_FILES) $(GPGKEY_DEST) $(SBIN_SCRIPTS) $(SBIN_DEST) $(BIN_INSTALL) $(PERL_SCRIPTS) $(PERL_DEST) + $(CONF_INSTALL) $(CONF_FILES) $(CONF_DEST) $(GPGKEY_INSTALL) $(GPGKEY_FILES) $(GPGKEY_DEST) $(BIN_INSTALL) $(SBIN_SCRIPTS) $(SBIN_DEST) $(PERL_DEST): $(DIR_INSTALL) $@ +$(CONF_DEST): + $(DIR_INSTALL) $@ + $(GPGKEY_DEST): $(DIR_INSTALL) $@ diff --git a/spacewalk/admin/service-list b/spacewalk/admin/service-list new file mode 100644 index 0000000..82379be --- /dev/null +++ b/spacewalk/admin/service-list @@ -0,0 +1,4 @@ +# Customize list of services to be managed using the +# spacewalk-service command here. +# For example: +# SERVICES="your-service $SERVICES" diff --git a/spacewalk/admin/spacewalk-admin.spec b/spacewalk/admin/spacewalk-admin.spec index ebd6d7a..bef7655 100644 --- a/spacewalk/admin/spacewalk-admin.spec +++ b/spacewalk/admin/spacewalk-admin.spec @@ -68,6 +68,7 @@ rm -rf $RPM_BUILD_ROOT %{_mandir}/man8/spacewalk-service.8* %{_mandir}/man8/rhn-sat-restart-silent.8* %{_mandir}/man8/rhn-config-satellite.pl.8* +%config(noreplace) %{_sysconfdir}/rhn/service-list %changelog * Fri Feb 03 2012 Jan Pazdziora 1.7.2-1 diff --git a/spacewalk/admin/spacewalk-service b/spacewalk/admin/spacewalk-service index 45feaef..1345a31 100755 --- a/spacewalk/admin/spacewalk-service +++ b/spacewalk/admin/spacewalk-service @@ -34,6 +34,9 @@ fi SERVICES="jabberd $DB_SERVICE osa-dispatcher $TOMCAT httpd Monitoring MonitoringScout rhn-search cobblerd taskomatic" +if [ -f /etc/rhn/service-list ] ; then + . /etc/rhn/service-list +fi . /etc/init.d/functions commit 4fc407bfc5419a583905e8f9943f0e28a22151fc Author: Jan Pazdziora <jpazdziora(a)redhat.com> Date: Fri Feb 3 14:43:52 2012 +0100 Purging trailing whitespaces. diff --git a/spacewalk/admin/Makefile.admin b/spacewalk/admin/Makefile.admin index afaf29b..feb6f8a 100644 --- a/spacewalk/admin/Makefile.admin +++ b/spacewalk/admin/Makefile.admin @@ -38,8 +38,8 @@ SBIN_SCRIPTS = rhn-sat-restart-silent spacewalk-service GPGKEY_FILES = RHN-GPG-KEY -BIN_INSTALL = install -m 755 -GPGKEY_INSTALL = install -m 644 +BIN_INSTALL = install -m 755 +GPGKEY_INSTALL = install -m 644 DIR_INSTALL = install -d -m 755 # install scripts @@ -48,12 +48,12 @@ all: install: $(PERL_SCRIPTS) $(PERL_DEST) $(GPGKEY_FILES) $(GPGKEY_DEST) $(SBIN_SCRIPTS) $(SBIN_DEST) $(BIN_INSTALL) $(PERL_SCRIPTS) $(PERL_DEST) $(GPGKEY_INSTALL) $(GPGKEY_FILES) $(GPGKEY_DEST) - $(BIN_INSTALL) $(SBIN_SCRIPTS) $(SBIN_DEST) + $(BIN_INSTALL) $(SBIN_SCRIPTS) $(SBIN_DEST) -$(PERL_DEST): +$(PERL_DEST): $(DIR_INSTALL) $@ -$(GPGKEY_DEST): +$(GPGKEY_DEST): $(DIR_INSTALL) $@ $(SBIN_DEST):

1 0

Makefile
by Jan Pazdziora 03 Feb '12

03 Feb '12

Makefile | 1 - 1 file changed, 1 deletion(-) New commits: commit 43480b07ee49baca2e527765765392a294bbeded Author: Jan Pazdziora <jpazdziora(a)redhat.com> Date: Fri Feb 3 14:14:24 2012 +0100 Packages of the Spacewalk project are built using the tito tool. diff --git a/Makefile b/Makefile deleted file mode 120000 index 29e219d..0000000 --- a/Makefile +++ /dev/null @@ -1 +0,0 @@ -rel-eng/Makefile \ No newline at end of file

1 0

Changes to 'refs/tags/spacewalk-backend-1.7.13-1'
by Michael Mraka 03 Feb '12

03 Feb '12

Tag 'spacewalk-backend-1.7.13-1' created by Michael Mraka <michael.mraka(a)redhat.com> at 2012-02-03 10:29 +0000 Tagging package [spacewalk-backend] version [1.7.13-1] in directory [backend/]. Changes since MessageQueue-3.26.7-1: Michael Mraka (26): created virtual class for RPM/MPM/DEB packages rewritten package_push handler to use package object compute checksum while saving payload remember payload_size added header_start, header_end create proper package object created RPM_Package moved InvalidPackageError to rhn_pkg removed commented out code moved _read_bytes() to A_Package implemented read_header() for MPM_Package replaced stream_copy() with _stream_copy() from super class removed tempfile usage from MPM_Package.write() implemented MPM_Package.save_payload() removed unused write_temp_file() rewritten get_package_header() with A_Package() fixed push_package() to use A_Package removed od CVS style id tag created DEB_Package spacewalk-repo-sync updated to use new package object generic code to compute checksum of package modified RPM/MPM/DEB package for payload_checksum version and release should be strings, epoch should be None or string merged duplicated h.execute() call simplified checksum_exists condition Automatic commit of package [spacewalk-backend] release [1.7.13-1]. --- backend/common/rhn_deb.py | 49 +++-- backend/common/rhn_mpm.py | 117 +++++------- backend/common/rhn_pkg.py | 114 ++++++++++- backend/common/rhn_rpm.py | 95 +++++++++ backend/satellite_tools/reposync.py | 42 +--- backend/server/handlers/app/packages.py | 42 ++-- backend/server/rhnPackageUpload.py | 98 ++++------ backend/spacewalk-backend.spec | 17 + backend/upload_server/handlers/package_push/package_push.py | 38 +-- rel-eng/packages/spacewalk-backend | 2 10 files changed, 389 insertions(+), 225 deletions(-) ---

1 0

backend/spacewalk-backend.spec rel-eng/packages
by Michael Mraka 03 Feb '12

03 Feb '12

backend/spacewalk-backend.spec | 17 ++++++++++++++++- rel-eng/packages/spacewalk-backend | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) New commits: commit f5b6b580999260976c151a184a18d262edb529f9 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Feb 3 11:29:10 2012 +0100 Automatic commit of package [spacewalk-backend] release [1.7.13-1]. diff --git a/backend/spacewalk-backend.spec b/backend/spacewalk-backend.spec index 9115079..cdfe2c0 100644 --- a/backend/spacewalk-backend.spec +++ b/backend/spacewalk-backend.spec @@ -12,7 +12,7 @@ Name: spacewalk-backend Summary: Common programs needed to be installed on the Spacewalk servers/proxies Group: Applications/Internet License: GPLv2 and Python -Version: 1.7.12 +Version: 1.7.13 Release: 1%{?dist} URL: https://fedorahosted.org/spacewalk Source0: https://fedorahosted.org/releases/s/p/spacewalk/%{name}-%{version}.tar.gz @@ -675,6 +675,21 @@ rm -f %{rhnconf}/rhnSecret.py* # $Id$ %changelog +* Fri Feb 03 2012 Michael Mraka <michael.mraka(a)redhat.com> 1.7.13-1 +- simplified checksum_exists condition +- merged duplicated h.execute() call +- version and release should be strings, epoch should be None or string +- modified RPM/MPM/DEB package for payload_checksum +- generic code to compute checksum of package +- spacewalk-repo-sync updated to use new package object +- created DEB_Package +- moved InvalidPackageError to rhn_pkg +- created RPM_Package +- create proper package object +- compute checksum while saving payload +- rewritten package_push handler to use package object +- created virtual class for RPM/MPM/DEB packages + * Thu Feb 02 2012 Jan Pazdziora 1.7.12-1 - Call the test entitlement count check for satellite-sync as well. diff --git a/rel-eng/packages/spacewalk-backend b/rel-eng/packages/spacewalk-backend index 6e649bc..b8c89a4 100644 --- a/rel-eng/packages/spacewalk-backend +++ b/rel-eng/packages/spacewalk-backend @@ -1 +1 @@ -1.7.12-1 backend/ +1.7.13-1 backend/

1 0

25 commits - backend/common backend/satellite_tools backend/server backend/upload_server
by Michael Mraka 03 Feb '12

03 Feb '12

backend/common/rhn_deb.py | 49 +++-- backend/common/rhn_mpm.py | 117 +++++------- backend/common/rhn_pkg.py | 114 ++++++++++- backend/common/rhn_rpm.py | 95 +++++++++ backend/satellite_tools/reposync.py | 42 +--- backend/server/handlers/app/packages.py | 42 ++-- backend/server/rhnPackageUpload.py | 98 ++++------ backend/upload_server/handlers/package_push/package_push.py | 38 +-- 8 files changed, 372 insertions(+), 223 deletions(-) New commits: commit 31e01e204aa9bbe13b0cce78cc80affe584fe75d Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Feb 3 11:12:06 2012 +0100 simplified checksum_exists condition diff --git a/backend/server/handlers/app/packages.py b/backend/server/handlers/app/packages.py index 4363155..b31d347 100644 --- a/backend/server/handlers/app/packages.py +++ b/backend/server/handlers/app/packages.py @@ -306,18 +306,10 @@ class Packages(RPC_Base): # Source package - no reason to continue continue _checksum_sql_filter = "" - checksum_exists = 0 if 'md5sum' in package: # for old rhnpush compatibility package['checksum_type'] = 'md5' package['checksum'] = package['md5sum'] - if package.has_key('checksum') and CFG.ENABLE_NVREA: - checksum_exists = 1 - _checksum_sql_filter = """and c.checksum = :checksum - and c.checksum_type = :checksum_type""" - - h = rhnSQL.prepare(self._get_pkg_info_query % \ - _checksum_sql_filter) exec_args = { 'pkg_name': package['name'], 'pkg_epoch': package['epoch'], @@ -326,9 +318,15 @@ class Packages(RPC_Base): 'pkg_arch': package['arch'], 'orgid': org_id } - if checksum_exists: + + if package.has_key('checksum') and CFG.ENABLE_NVREA: + _checksum_sql_filter = """and c.checksum = :checksum + and c.checksum_type = :checksum_type""" exec_args.update({'checksum_type': package['checksum_type'], 'checksum': package['checksum']}) + + h = rhnSQL.prepare(self._get_pkg_info_query % \ + _checksum_sql_filter) h.execute(**exec_args) row = h.fetchone_dict() commit a10b358478ac635ea2b837093235b7f72391f524 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Feb 3 11:09:40 2012 +0100 merged duplicated h.execute() call diff --git a/backend/server/handlers/app/packages.py b/backend/server/handlers/app/packages.py index 0b22eff..4363155 100644 --- a/backend/server/handlers/app/packages.py +++ b/backend/server/handlers/app/packages.py @@ -318,21 +318,18 @@ class Packages(RPC_Base): h = rhnSQL.prepare(self._get_pkg_info_query % \ _checksum_sql_filter) + exec_args = { + 'pkg_name': package['name'], + 'pkg_epoch': package['epoch'], + 'pkg_version': package['version'], + 'pkg_rel': package['release'], + 'pkg_arch': package['arch'], + 'orgid': org_id + } if checksum_exists: - h.execute(pkg_name=package['name'], \ - pkg_epoch=package['epoch'], \ - pkg_version=package['version'], \ - pkg_rel=package['release'],pkg_arch=package['arch'], \ - orgid = org_id, \ - checksum_type = package['checksum_type'], \ - checksum = package['checksum']) - else: - h.execute(pkg_name=package['name'], \ - pkg_epoch=package['epoch'], \ - pkg_version=package['version'], \ - pkg_rel=package['release'], \ - pkg_arch=package['arch'], orgid = org_id ) - + exec_args.update({'checksum_type': package['checksum_type'], + 'checksum': package['checksum']}) + h.execute(**exec_args) row = h.fetchone_dict() package['checksum_type'] = row['checksum_type'] commit 1bd9bd534e087b848da90a13f616fe274418cfaa Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Feb 3 11:07:16 2012 +0100 version and release should be strings, epoch should be None or string diff --git a/backend/server/handlers/app/packages.py b/backend/server/handlers/app/packages.py index 1d39487..0b22eff 100644 --- a/backend/server/handlers/app/packages.py +++ b/backend/server/handlers/app/packages.py @@ -293,6 +293,15 @@ class Packages(RPC_Base): for k in package_keys: if not package.has_key(k): raise Exception("Missing key %s" % k) + if k == 'epoch': + if package[k] is not None: + if package[k] == '': + package[k] = None + else: + package[k] = str(package[k]) + else: + package[k] = str(package[k]) + if package['arch'] == 'src' or package['arch'] == 'nosrc': # Source package - no reason to continue continue @@ -309,13 +318,9 @@ class Packages(RPC_Base): h = rhnSQL.prepare(self._get_pkg_info_query % \ _checksum_sql_filter) - pkg_epoch = None - if package['epoch'] is not None and package['epoch'] != '': - pkg_epoch = str(package['epoch']) - if checksum_exists: h.execute(pkg_name=package['name'], \ - pkg_epoch=pkg_epoch, \ + pkg_epoch=package['epoch'], \ pkg_version=package['version'], \ pkg_rel=package['release'],pkg_arch=package['arch'], \ orgid = org_id, \ @@ -323,7 +328,7 @@ class Packages(RPC_Base): checksum = package['checksum']) else: h.execute(pkg_name=package['name'], \ - pkg_epoch=pkg_epoch, \ + pkg_epoch=package['epoch'], \ pkg_version=package['version'], \ pkg_rel=package['release'], \ pkg_arch=package['arch'], orgid = org_id ) commit 90a0fda79fd98ae3277d7036800438fb243263cb Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Wed Feb 1 13:51:17 2012 +0100 modified RPM/MPM/DEB package for payload_checksum diff --git a/backend/common/rhn_deb.py b/backend/common/rhn_deb.py index 308c530..e532874 100644 --- a/backend/common/rhn_deb.py +++ b/backend/common/rhn_deb.py @@ -159,8 +159,10 @@ class DEB_Package(A_Package): def save_payload(self, output_stream): hash = checksum.hashlib.new(self.checksum_type) - output_start = output_stream.tell() + if output_stream: + output_start = output_stream.tell() self._stream_copy(self.header_data, output_stream, hash) self.checksum = hash.hexdigest() - self.payload_stream = output_stream - self.payload_size = output_stream.tell() - output_start + if output_stream: + self.payload_stream = output_stream + self.payload_size = output_stream.tell() - output_start diff --git a/backend/common/rhn_mpm.py b/backend/common/rhn_mpm.py index bb15eb6..e4a0451 100644 --- a/backend/common/rhn_mpm.py +++ b/backend/common/rhn_mpm.py @@ -241,21 +241,24 @@ class MPM_Package(A_Package): def _encode_payload(self, stream, hash=None): assert(self.payload_stream is not None) - start = stream.tell() - if self.payload_flags & MPM_PAYLOAD_COMPRESSED_GZIP: + if stream: + start = stream.tell() + if stream and self.payload_flags & MPM_PAYLOAD_COMPRESSED_GZIP: f = gzip.GzipFile(None, "wb", 9, stream) self._stream_copy(self.payload_stream, f, hash) f.close() else: self._stream_copy(self.payload_stream, stream, hash) - self.payload_size = stream.tell() - start + if stream: + self.payload_size = stream.tell() - start def save_payload(self, output_stream): self.payload_stream = self.input_stream hash = checksum.hashlib.new(self.header.checksum_type()) self._encode_payload(output_stream, hash) self.checksum = hash.hexdigest() - self.payload_stream = output_stream + if output_stream: + self.payload_stream = output_stream def _replace_null(obj): diff --git a/backend/common/rhn_rpm.py b/backend/common/rhn_rpm.py index abba819..5c822db 100644 --- a/backend/common/rhn_rpm.py +++ b/backend/common/rhn_rpm.py @@ -206,13 +206,16 @@ class RPM_Package(A_Package): def save_payload(self, output_stream): hash = checksum.hashlib.new(self.checksum_type) - output_start = output_stream.tell() + if output_stream: + output_start = output_stream.tell() self.header_data.seek(0,0) self._stream_copy(self.header_data, output_stream, hash) self._stream_copy(self.input_stream, output_stream, hash) self.checksum = hash.hexdigest() - self.payload_stream = output_stream - self.payload_size = output_stream.tell() - output_start + self.header_data.close() + if output_stream: + self.payload_stream = output_stream + self.payload_size = output_stream.tell() - output_start def get_header_byte_range(package_file): """ commit c5818703826db1220403c9a5473729cabb44bb32 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Wed Feb 1 13:29:45 2012 +0100 generic code to compute checksum of package diff --git a/backend/common/rhn_pkg.py b/backend/common/rhn_pkg.py index ae55836..fd952f8 100644 --- a/backend/common/rhn_pkg.py +++ b/backend/common/rhn_pkg.py @@ -74,11 +74,20 @@ class A_Package: def save_payload(self, output_stream): """saves payload to output_stream""" hash = checksum.hashlib.new(self.checksum_type) - output_start = output_stream.tell() + if output_stream: + output_start = output_stream.tell() self._stream_copy(self.input_stream, output_stream, hash) self.checksum = hash.hexdigest() - self.payload_stream = output_stream - self.payload_size = output_stream.tell() - output_start + if output_stream: + self.payload_stream = output_stream + self.payload_size = output_stream.tell() - output_start + + def payload_checksum(self): + # just read and compute checksum + start = self.input_stream.tell() + self.save_payload(None) + self.payload_size = self.input_stream.tell() - start + self.payload_stream = self.input_stream def _stream_copy(self, source, dest, hash=None): @@ -87,7 +96,8 @@ class A_Package: buf = source.read(BUFFER_SIZE) if not buf: break - dest.write(buf) + if dest: + dest.write(buf) if hash: hash.update(buf) commit d7bf414c6810d1ca5fb101932f303297095cd6e5 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Tue Jan 31 10:59:35 2012 +0100 spacewalk-repo-sync updated to use new package object diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py index 18646c6..f68545e 100644 --- a/backend/satellite_tools/reposync.py +++ b/backend/satellite_tools/reposync.py @@ -21,7 +21,7 @@ import time from datetime import datetime from spacewalk.server import rhnPackage, rhnSQL, rhnChannel, rhnPackageUpload -from spacewalk.common import rhnLog +from spacewalk.common import rhnLog, rhn_pkg from spacewalk.common.rhnLog import log_debug from spacewalk.common.checksum import getFileChecksum from spacewalk.common.rhnConfig import CFG, initCFG @@ -359,9 +359,8 @@ class RepoSync(object): pack.path = localpath = plug.get_package(pack) pack.load_checksum_from_header() if to_download: - self.upload_package(pack) + pack.upload_package(self.channel) finally_remove(localpath) - pack.payload_stream.close() if to_link: self.associate_package(pack) except KeyboardInterrupt: @@ -380,17 +379,6 @@ class RepoSync(object): return 1 return 0 - def upload_package(self, package): - rel_package_path = rhnPackageUpload.relative_path_from_header( - package.header, self.channel['org_id'], - package.checksum_type, package.checksum) - package_dict, diff_level = rhnPackageUpload.push_package(package.header, - package.path, package.payload_stream, package.checksum_type, package.checksum, - force=False, - header_start=package.header_start, header_end=package.header_end, - relative_path=rel_package_path, - org_id=self.channel['org_id']) - def associate_package(self, pack): caller = "server.app.yumreposync" backend = SQLBackend() @@ -399,8 +387,8 @@ class RepoSync(object): package['version'] = pack.version package['release'] = pack.release package['arch'] = pack.arch - package['checksum'] = pack.checksum - package['checksum_type'] = pack.checksum_type + package['checksum'] = pack.a_pkg.checksum + package['checksum_type'] = pack.a_pkg.checksum_type package['channels'] = [{'label':self.channel_label, 'id':self.channel['id']}] package['org_id'] = self.channel['org_id'] @@ -560,10 +548,8 @@ class ContentPackage: self.path = None self.file = None - self.header = None - self.payload_stream = None - self.header_start = None - self.header_end = None + + self.a_pkg = None def setNVREA(self, name, version, release, epoch, arch): self.name = name @@ -582,8 +568,16 @@ class ContentPackage: if self.path is None: raise rhnFault(50, "Unable to load package", explain=0) self.file = open(self.path, 'rb') - self.header, self.payload_stream, self.header_start, self.header_end = \ - rhnPackageUpload.load_package(self.file) - self.checksum_type = self.header.checksum_type() - self.checksum = getFileChecksum(self.checksum_type, file=self.file) + self.a_pkg = rhn_pkg.package_from_stream(self.file, packaging='rpm') + self.a_pkg.read_header() + self.a_pkg.payload_checksum() self.file.close() + + def upload_package(self, channel): + rel_package_path = rhnPackageUpload.relative_path_from_header( + self.a_pkg.header, channel['org_id'], + self.a_pkg.checksum_type, self.a_pkg.checksum) + package_dict, diff_level = rhnPackageUpload.push_package(self.a_pkg, + force=False, + relative_path=rel_package_path, + org_id=channel['org_id']) commit d0f9cf64f2122d70cd29003b00b02f32d717ab33 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Tue Jan 31 10:04:26 2012 +0100 created DEB_Package diff --git a/backend/common/rhn_deb.py b/backend/common/rhn_deb.py index 49515ae..308c530 100644 --- a/backend/common/rhn_deb.py +++ b/backend/common/rhn_deb.py @@ -20,9 +20,12 @@ import os import gzip import sys +import tempfile from debian import debfile -from rhn_pkg import InvalidPackageError + +import checksum +from rhn_pkg import A_Package, InvalidPackageError DEB_CHECKSUM_TYPE = 'md5' # FIXME: this should be a configuration option @@ -44,9 +47,9 @@ def load(filename=None, file=None, fd=None): f = os.fdopen(os.dup(fd), "r") f.seek(0, 0) - return load_deb(f, filename) + return load_deb(f) -def load_deb(stream, filename): +def load_deb(stream): # Dup the file descriptor, we don't want it to get closed before we read # the payload @@ -59,7 +62,7 @@ def load_deb(stream, filename): try: #header = rhn_deb.get_package_header(file=stream) - header = deb_Header(filename) + header = deb_Header(stream) except: raise InvalidPackageError, None, sys.exc_info()[2] stream.seek(0, 0) @@ -68,7 +71,7 @@ def load_deb(stream, filename): class deb_Header: "Wrapper class for an deb header - we need to store a flag is_source" - def __init__(self, name): + def __init__(self, stream): self.hdr = {} self.packaging = 'deb' self.signatures = [] @@ -80,7 +83,7 @@ class deb_Header: self.deb = None try: - self.deb = debfile.DebFile(name) + self.deb = debfile.DebFile(stream.name) except Exception, e: raise InvalidPackageError(e), None, sys.exc_info()[2] @@ -139,3 +142,25 @@ class deb_Header: return self.deb.debcontrol().get_as_string(name) return None + +class DEB_Package(A_Package): + def __init__(self, input_stream = None): + A_Package.__init__(self, input_stream) + self.header_data = tempfile.NamedTemporaryFile() + self.checksum_type = DEB_CHECKSUM_TYPE + + def read_header(self): + self._stream_copy(self.input_stream, self.header_data) + try: + self.header_data.seek(0,0) + self.header = deb_Header(self.header_data) + except: + raise InvalidPackageError, None, sys.exc_info()[2] + + def save_payload(self, output_stream): + hash = checksum.hashlib.new(self.checksum_type) + output_start = output_stream.tell() + self._stream_copy(self.header_data, output_stream, hash) + self.checksum = hash.hexdigest() + self.payload_stream = output_stream + self.payload_size = output_stream.tell() - output_start commit f375719cb3665de57df1b363bcf96532f160ed97 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Tue Jan 31 09:46:25 2012 +0100 removed od CVS style id tag diff --git a/backend/common/rhn_deb.py b/backend/common/rhn_deb.py index 51250e0..49515ae 100644 --- a/backend/common/rhn_deb.py +++ b/backend/common/rhn_deb.py @@ -15,8 +15,6 @@ # # Meta-package manager # -# $Id$ -# # Author: Lukas Durfina <lukas.durfina(a)gmail.com> import os commit 4f1c5716c4806ed808cee2f811e67b2c7e6cdeb1 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 16:52:21 2012 +0100 fixed push_package() to use A_Package diff --git a/backend/server/rhnPackageUpload.py b/backend/server/rhnPackageUpload.py index bf392a3..ebe774e 100644 --- a/backend/server/rhnPackageUpload.py +++ b/backend/server/rhnPackageUpload.py @@ -18,7 +18,7 @@ import os import sys import tempfile -from spacewalk.common import rhn_mpm, rhn_deb +from spacewalk.common import rhn_mpm, rhn_deb, rhn_pkg from spacewalk.common.rhnLog import log_debug from spacewalk.common.rhnConfig import CFG from spacewalk.common.rhnException import rhnFault @@ -155,17 +155,12 @@ def push_package(a_pkg, org_id=None, force=None, channels=[], relative_path=None orig_path = os.path.join(CFG.MOUNT_POINT, orig_path) log_debug(4, "Original package", orig_path) - # Determine the type of packaging that was used to create the package. - packaging = 'rpm' - if hasattr(a_pkg.header, 'packaging'): - packaging = a_pkg.header.packaging - # MPMs do not store their headers on disk, so we must avoid performing # operations which rely on information only contained in the headers # (such as header signatures). - if os.path.exists(orig_path) and packaging != 'mpm': - oh = rhn_mpm.get_package_header(orig_path) - _diff_header_sigs(header, oh, pdict['diff']['diff']) + if os.path.exists(orig_path) and a_pkg.header.packaging != 'mpm': + oh = rhn_pkg.get_package_header(orig_path) + _diff_header_sigs(a_pkg.header, oh, pdict['diff']['diff']) return pdict, package.diff.level commit bc3931ca16913edbc4e58b64a1a3f7052e8702ec Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 16:48:06 2012 +0100 rewritten get_package_header() with A_Package() diff --git a/backend/common/rhn_pkg.py b/backend/common/rhn_pkg.py index 99dd6ee..ae55836 100644 --- a/backend/common/rhn_pkg.py +++ b/backend/common/rhn_pkg.py @@ -13,25 +13,30 @@ # in this software or its documentation. # -import rhn_rpm -import rhn_mpm -import rhn_deb import checksum -def rhn_pkg(filename=None, file=None, fd=None): - if filename: - if filename.endswith('.deb'): - return rhn_deb - elif filename.endswith('.rpm'): - return rhn_rpm - else: - return rhn_mpm - # XXX recognize backend also by file/fd - return rhn_rpm - def get_package_header(filename=None, file=None, fd=None): - return rhn_pkg(filename=filename, file=file, fd=fd) \ - .get_package_header(filename=filename, file=file, fd=fd) + if filename is not None: + stream = open(filename) + need_close = True + elif file is not None: + stream = file + else: + stream = os.fdopen(os.dup(fd), "r") + need_close = True + + if stream.name.endswith('.deb'): + packaging = 'deb' + elif stream.name.endswith('.rpm'): + packaging = 'rpm' + else: + packaging = 'mpm' + + a_pkg = package_from_stream(stream, packaging) + a_pkg.read_header() + if need_close: + stream.close() + return a_pkg.header def package_from_stream(stream, packaging): if packaging == 'deb': commit 8d65a541b9967a3aba1c4cdd741224fae62dbdcf Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Thu Jan 12 12:59:06 2012 +0100 removed unused write_temp_file() diff --git a/backend/server/rhnPackageUpload.py b/backend/server/rhnPackageUpload.py index 395b368..bf392a3 100644 --- a/backend/server/rhnPackageUpload.py +++ b/backend/server/rhnPackageUpload.py @@ -33,26 +33,6 @@ from spacewalk.server.rhnLib import get_package_path, \ from spacewalk.server.rhnServer import server_packages -def write_temp_file(req, buffer_size, dir, prefix, packaging=None): - """ Write request to temporary file (write max. buffer_size at once). - Returns the file object. - """ - suffix = '' - if packaging == 'deb': - suffix = '.deb' - if not os.path.isdir(dir): - os.makedirs(dir) - t = tempfile.NamedTemporaryFile( - dir=dir, prefix=prefix + '.', - suffix=suffix) - while 1: - buf = req.read(buffer_size) - if not buf: - break - t.write(buf) - t.seek(0, 0) - return t - def authenticate(username, password, channels=[], null_org=None, force=None): log_debug(4, username, force) authobj = userAuth.UserAuth() commit 4b7d7a5757195f4f102458489462110db63f5531 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 16:09:35 2012 +0100 implemented MPM_Package.save_payload() diff --git a/backend/common/rhn_mpm.py b/backend/common/rhn_mpm.py index 77a2a47..bb15eb6 100644 --- a/backend/common/rhn_mpm.py +++ b/backend/common/rhn_mpm.py @@ -23,6 +23,8 @@ import sys import fileutils from types import ListType, TupleType + +import checksum from rhn_pkg import A_Package, InvalidPackageError MPM_CHECKSUM_TYPE = 'md5' # FIXME: this should be a configuration option @@ -131,6 +133,7 @@ class MPM_Package(A_Package): def __init__(self, input_stream = None): A_Package.__init__(self, input_stream) self.header_flags = MPM_HEADER_COMPRESSED_GZIP + self.header_size = 0 self.payload_flags = 0 assert(len(self._magic) == 16) self._buffer_size = 16384 @@ -211,12 +214,12 @@ class MPM_Package(A_Package): raise Exception() output_stream.seek(128, 0) - header_size = self._encode_header(output_stream) - payload_size = self._encode_payload(output_stream) + self._encode_header(output_stream) + self._encode_payload(output_stream) # now we know header and payload size so rewind back and write lead lead_arr = (self._magic, 1, "\0" * 3, self.header_flags, - self.payload_flags, header_size, payload_size, '\0' * 92) + self.payload_flags, self.header_size, self.payload_size, '\0' * 92) # lead lead = apply(struct.pack, (self._lead_format, ) + lead_arr) output_stream.seek(0, 0) @@ -234,20 +237,25 @@ class MPM_Package(A_Package): else: stream.write(data) stream.flush() - end = stream.tell() - return (end - start) + self.header_size = stream.tell() - start - def _encode_payload(self, stream): + def _encode_payload(self, stream, hash=None): assert(self.payload_stream is not None) start = stream.tell() if self.payload_flags & MPM_PAYLOAD_COMPRESSED_GZIP: f = gzip.GzipFile(None, "wb", 9, stream) - self._stream_copy(self.payload_stream, f) + self._stream_copy(self.payload_stream, f, hash) f.close() else: - self._stream_copy(self.payload_stream, stream) - end = stream.tell() - return (end - start) + self._stream_copy(self.payload_stream, stream, hash) + self.payload_size = stream.tell() - start + + def save_payload(self, output_stream): + self.payload_stream = self.input_stream + hash = checksum.hashlib.new(self.header.checksum_type()) + self._encode_payload(output_stream, hash) + self.checksum = hash.hexdigest() + self.payload_stream = output_stream def _replace_null(obj): commit 3174afc23727797c0a35414fe5ac7e246a94c4e1 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 15:52:11 2012 +0100 removed tempfile usage from MPM_Package.write() diff --git a/backend/common/rhn_mpm.py b/backend/common/rhn_mpm.py index 0b4ad53..77a2a47 100644 --- a/backend/common/rhn_mpm.py +++ b/backend/common/rhn_mpm.py @@ -210,21 +210,23 @@ class MPM_Package(A_Package): if self.header is None: raise Exception() - header_stream, header_size = self._encode_header() - payload_stream, payload_size = self._encode_payload() + output_stream.seek(128, 0) + header_size = self._encode_header(output_stream) + payload_size = self._encode_payload(output_stream) + # now we know header and payload size so rewind back and write lead lead_arr = (self._magic, 1, "\0" * 3, self.header_flags, self.payload_flags, header_size, payload_size, '\0' * 92) # lead lead = apply(struct.pack, (self._lead_format, ) + lead_arr) + output_stream.seek(0, 0) output_stream.write(lead) - self._stream_copy(header_stream, output_stream) - self._stream_copy(payload_stream, output_stream) + output_stream.seek(0, 2) - def _encode_header(self): + def _encode_header(self, stream): assert(self.header is not None) - stream = tempfile.TemporaryFile() data = xmlrpclib.dumps((_replace_null(self.header), )) + start = stream.tell() if self.header_flags & MPM_HEADER_COMPRESSED_GZIP: f = gzip.GzipFile(None, "wb", 9, stream) f.write(data) @@ -232,25 +234,20 @@ class MPM_Package(A_Package): else: stream.write(data) stream.flush() - stream.seek(0, 2) - size = stream.tell() - stream.seek(0, 0) - return stream, size + end = stream.tell() + return (end - start) - def _encode_payload(self): + def _encode_payload(self, stream): assert(self.payload_stream is not None) - stream = tempfile.TemporaryFile() + start = stream.tell() if self.payload_flags & MPM_PAYLOAD_COMPRESSED_GZIP: f = gzip.GzipFile(None, "wb", 9, stream) self._stream_copy(self.payload_stream, f) f.close() else: - stream = self.payload_stream - stream.flush() - stream.seek(0, 2) - size = stream.tell() - stream.seek(0, 0) - return stream, size + self._stream_copy(self.payload_stream, stream) + end = stream.tell() + return (end - start) def _replace_null(obj): commit 9562335d2d4e876e245f5e94f1c487658b112263 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 15:22:24 2012 +0100 replaced stream_copy() with _stream_copy() from super class diff --git a/backend/common/rhn_mpm.py b/backend/common/rhn_mpm.py index 19f5de4..0b4ad53 100644 --- a/backend/common/rhn_mpm.py +++ b/backend/common/rhn_mpm.py @@ -200,7 +200,7 @@ class MPM_Package(A_Package): if payload_flags & MPM_PAYLOAD_COMPRESSED_GZIP: g = gzip.GzipFile(None, "r", 0, payload_stream) t = tempfile.TemporaryFile() - self.stream_copy(g, t) + self._stream_copy(g, t) g.close() payload_stream = t @@ -218,8 +218,8 @@ class MPM_Package(A_Package): # lead lead = apply(struct.pack, (self._lead_format, ) + lead_arr) output_stream.write(lead) - self.stream_copy(header_stream, output_stream) - self.stream_copy(payload_stream, output_stream) + self._stream_copy(header_stream, output_stream) + self._stream_copy(payload_stream, output_stream) def _encode_header(self): assert(self.header is not None) @@ -242,7 +242,7 @@ class MPM_Package(A_Package): stream = tempfile.TemporaryFile() if self.payload_flags & MPM_PAYLOAD_COMPRESSED_GZIP: f = gzip.GzipFile(None, "wb", 9, stream) - self.stream_copy(self.payload_stream, f) + self._stream_copy(self.payload_stream, f) f.close() else: stream = self.payload_stream @@ -252,14 +252,6 @@ class MPM_Package(A_Package): stream.seek(0, 0) return stream, size - def stream_copy(self, source, dest): - "Copies data from the source stream to the destination stream" - while 1: - buf = source.read(self._buffer_size) - if not buf: - break - dest.write(buf) - def _replace_null(obj): if obj is None: commit aba73d53a7eedec0ff6b3e2dd3fe21e3e6fab97e Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 14:53:18 2012 +0100 implemented read_header() for MPM_Package diff --git a/backend/common/rhn_mpm.py b/backend/common/rhn_mpm.py index 9b57ece..19f5de4 100644 --- a/backend/common/rhn_mpm.py +++ b/backend/common/rhn_mpm.py @@ -23,7 +23,7 @@ import sys import fileutils from types import ListType, TupleType -from rhn_pkg import InvalidPackageError +from rhn_pkg import A_Package, InvalidPackageError MPM_CHECKSUM_TYPE = 'md5' # FIXME: this should be a configuration option @@ -125,16 +125,28 @@ class MPM_Header: MPM_HEADER_COMPRESSED_GZIP = 1 MPM_PAYLOAD_COMPRESSED_GZIP = 1 -class MPM_Package: +class MPM_Package(A_Package): _lead_format = '!16sB3s4L92s' _magic = 'mpmpackage012345' - def __init__(self): - self.header = None - self.payload_stream = None + def __init__(self, input_stream = None): + A_Package.__init__(self, input_stream) self.header_flags = MPM_HEADER_COMPRESSED_GZIP self.payload_flags = 0 assert(len(self._magic) == 16) self._buffer_size = 16384 + self.file_size = 0 + + def read_header(self): + arr = self._read_lead(self.input_stream) + magic = arr[0] + if magic != self._magic: + raise InvalidPackageError() + header_len, payload_len = int(arr[5]), int(arr[6]) + self.header_flags, self.payload_flags = arr[3], arr[4] + self.file_size = 128 + header_len + payload_len + header_data = self._read_bytes(self.input_stream, header_len) + self._read_header(header_data, self.header_flags) + self.checksum_type = self.header.checksum_type() def _read_lead(self, stream): # Lead has the following format: @@ -156,23 +168,16 @@ class MPM_Package: def load(self, input_stream): # Clean up self.__init__() - arr = self._read_lead(input_stream) - magic = arr[0] - if magic != self._magic: - raise InvalidPackageError() - header_len, payload_len = int(arr[5]), int(arr[6]) - header_flags, payload_flags = arr[3], arr[4] - file_size = 128 + header_len + payload_len - input_stream.seek(file_size) - if file_size != input_stream.tell(): - raise InvalidPackageError() + self.input_stream = input_stream # Read the header - input_stream.seek(128, 0) - header_data = self._read_bytes(input_stream, header_len) + self.read_header() + payload_stream = fileutils.payload(input_stream.name, input_stream.tell()) + input_stream.seek(self.file_size) + if self.file_size != input_stream.tell(): + raise InvalidPackageError() - self._read_header(header_data, header_flags) - self._read_payload(payload_stream, payload_flags) + self._read_payload(payload_stream, self.payload_flags) def _read_header(self, header_data, header_flags): if header_flags & MPM_HEADER_COMPRESSED_GZIP: commit 7c2b705e9d4aed97cfe649ba1aa475841d891e15 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 14:31:31 2012 +0100 moved _read_bytes() to A_Package diff --git a/backend/common/rhn_mpm.py b/backend/common/rhn_mpm.py index a63540f..9b57ece 100644 --- a/backend/common/rhn_mpm.py +++ b/backend/common/rhn_mpm.py @@ -174,16 +174,6 @@ class MPM_Package: self._read_header(header_data, header_flags) self._read_payload(payload_stream, payload_flags) - def _read_bytes(self, stream, amt): - ret = "" - while amt: - buf = stream.read(min(amt, self._buffer_size)) - if not buf: - return ret - ret = ret + buf - amt = amt - len(buf) - return ret - def _read_header(self, header_data, header_flags): if header_flags & MPM_HEADER_COMPRESSED_GZIP: t = cStringIO.StringIO(header_data) diff --git a/backend/common/rhn_pkg.py b/backend/common/rhn_pkg.py index 3b0d351..99dd6ee 100644 --- a/backend/common/rhn_pkg.py +++ b/backend/common/rhn_pkg.py @@ -86,5 +86,15 @@ class A_Package: if hash: hash.update(buf) + def _read_bytes(self, stream, amt): + ret = "" + while amt: + buf = stream.read(min(amt, BUFFER_SIZE)) + if not buf: + return ret + ret = ret + buf + amt = amt - len(buf) + return ret + class InvalidPackageError(Exception): pass commit 5baa958b65468ed7ee51dacf8ff2b265d98e80b5 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 13:53:27 2012 +0100 removed commented out code diff --git a/backend/common/rhn_deb.py b/backend/common/rhn_deb.py index abf158e..51250e0 100644 --- a/backend/common/rhn_deb.py +++ b/backend/common/rhn_deb.py @@ -141,14 +141,3 @@ class deb_Header: return self.deb.debcontrol().get_as_string(name) return None -""" - def __setitem__(self, name, item): - self.hdr[name] = item - - def __delitem__(self, name): - del self.hdr[name] - - def __getattr__(self, name): - return getattr(self.hdr, name) -""" - commit 21848efdeaa7f534bcb80015dcd3c134439b8c48 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Thu Jan 12 11:42:44 2012 +0100 moved InvalidPackageError to rhn_pkg diff --git a/backend/common/rhn_deb.py b/backend/common/rhn_deb.py index 7daea15..abf158e 100644 --- a/backend/common/rhn_deb.py +++ b/backend/common/rhn_deb.py @@ -24,6 +24,7 @@ import gzip import sys from debian import debfile +from rhn_pkg import InvalidPackageError DEB_CHECKSUM_TYPE = 'md5' # FIXME: this should be a configuration option @@ -151,5 +152,3 @@ class deb_Header: return getattr(self.hdr, name) """ -class InvalidPackageError(Exception): - pass diff --git a/backend/common/rhn_mpm.py b/backend/common/rhn_mpm.py index 49a59e0..a63540f 100644 --- a/backend/common/rhn_mpm.py +++ b/backend/common/rhn_mpm.py @@ -23,6 +23,7 @@ import sys import fileutils from types import ListType, TupleType +from rhn_pkg import InvalidPackageError MPM_CHECKSUM_TYPE = 'md5' # FIXME: this should be a configuration option @@ -82,7 +83,7 @@ def load_rpm(stream): try: header = rhn_rpm.get_package_header(file=stream) - except rhn_rpm.InvalidPackageError, e: + except InvalidPackageError, e: raise InvalidPackageError(*e.args), None, sys.exc_info()[2] except rhn_rpm.error, e: raise InvalidPackageError(e), None, sys.exc_info()[2] @@ -121,9 +122,6 @@ class MPM_Header: def unload(self): return None -class InvalidPackageError(Exception): - pass - MPM_HEADER_COMPRESSED_GZIP = 1 MPM_PAYLOAD_COMPRESSED_GZIP = 1 diff --git a/backend/common/rhn_pkg.py b/backend/common/rhn_pkg.py index fb6a242..3b0d351 100644 --- a/backend/common/rhn_pkg.py +++ b/backend/common/rhn_pkg.py @@ -85,3 +85,6 @@ class A_Package: dest.write(buf) if hash: hash.update(buf) + +class InvalidPackageError(Exception): + pass diff --git a/backend/common/rhn_rpm.py b/backend/common/rhn_rpm.py index 8a373b6..abba819 100644 --- a/backend/common/rhn_rpm.py +++ b/backend/common/rhn_rpm.py @@ -20,7 +20,7 @@ import struct import tempfile import checksum -from rhn_pkg import A_Package +from rhn_pkg import A_Package, InvalidPackageError # Expose a bunch of useful constants from rpm error = rpm.error @@ -56,9 +56,6 @@ PGPHASHALGO = { } -class InvalidPackageError(Exception): - pass - class RPM_Header: "Wrapper class for an rpm header - we need to store a flag is_source" def __init__(self, hdr, is_source=None): commit dd8af107343ccfbcfd0d037b4a0695aa08e507dd Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 13:39:29 2012 +0100 created RPM_Package diff --git a/backend/common/rhn_rpm.py b/backend/common/rhn_rpm.py index de11246..8a373b6 100644 --- a/backend/common/rhn_rpm.py +++ b/backend/common/rhn_rpm.py @@ -17,6 +17,10 @@ import os import sys import rpm import struct +import tempfile + +import checksum +from rhn_pkg import A_Package # Expose a bunch of useful constants from rpm error = rpm.error @@ -128,6 +132,91 @@ class RPM_Header: 'signature' : ret, }) +class RPM_Package(A_Package): + def __init__(self, input_stream = None): + A_Package.__init__(self, input_stream) + self.header_data = tempfile.SpooledTemporaryFile() + + def read_header(self): + self._get_header_byte_range() + try: + self.header = get_package_header(file=self.header_data) + except InvalidPackageError, e: + raise InvalidPackageError(*e.args), None, sys.exc_info()[2] + except error, e: + raise InvalidPackageError(e), None, sys.exc_info()[2] + except: + raise InvalidPackageError, None, sys.exc_info()[2] + self.checksum_type = self.header.checksum_type() + + def _get_header_byte_range(self): + """ + Return the start and end bytes of the rpm header object. + Raw header data are then stored in self.header_data. + + For details of the rpm file format, see: + http://www.rpm.org/max-rpm/s1-rpm-file-format-rpm-file-format.html + """ + + lead_size = 96 + struct_lead_size = 16 + # Move past the rpm lead + buf = self._read_bytes(self.input_stream, lead_size) + self.header_data.write(buf) + + buf = self._read_bytes(self.input_stream, struct_lead_size) + self.header_data.write(buf) + + sig_size = self._get_header_struct_size(buf) + + # Now we can find the start of the actual header. + self.header_start = lead_size + sig_size + + buf = self._read_bytes(self.input_stream, sig_size - struct_lead_size) + self.header_data.write(buf) + + buf = self._read_bytes(self.input_stream, struct_lead_size) + self.header_data.write(buf) + + header_size = self._get_header_struct_size(buf) + self.header_end = self.header_start + header_size + + buf = self._read_bytes(self.input_stream, header_size - struct_lead_size) + self.header_data.write(buf) + + def _get_header_struct_size(self, struct_lead): + """ + Compute the size in bytes of the rpm header struct starting at the current + position in package_file. + """ + # Read the number of index entries + header_index = struct_lead[8:12] + (header_index_value, ) = struct.unpack('>I', header_index) + + # Read the the size of the header data store + header_store = struct_lead[12:16] + (header_store_value, ) = struct.unpack('>I', header_store) + + # The total size of the header. Each index entry is 16 bytes long. + header_size = 8 + 4 + 4 + header_index_value * 16 + header_store_value + + # Headers end on an 8-byte boundary. Round out the extra data. + round_out = header_size % 8 + if round_out != 0: + header_size = header_size + (8 - round_out) + + return header_size + + def save_payload(self, output_stream): + hash = checksum.hashlib.new(self.checksum_type) + output_start = output_stream.tell() + self.header_data.seek(0,0) + self._stream_copy(self.header_data, output_stream, hash) + self._stream_copy(self.input_stream, output_stream, hash) + self.checksum = hash.hexdigest() + self.payload_stream = output_stream + self.payload_size = output_stream.tell() - output_start + def get_header_byte_range(package_file): """ Return the start and end bytes of the rpm header object. commit 6f31a35f552a90ed1cd3954fc8ca408187d66d51 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 6 13:29:38 2012 +0100 create proper package object diff --git a/backend/common/rhn_pkg.py b/backend/common/rhn_pkg.py index e22cbdb..fb6a242 100644 --- a/backend/common/rhn_pkg.py +++ b/backend/common/rhn_pkg.py @@ -33,6 +33,19 @@ def get_package_header(filename=None, file=None, fd=None): return rhn_pkg(filename=filename, file=file, fd=fd) \ .get_package_header(filename=filename, file=file, fd=fd) +def package_from_stream(stream, packaging): + if packaging == 'deb': + import rhn_deb + a_pkg = rhn_deb.DEB_Package(stream) + elif packaging == 'rpm': + import rhn_rpm + a_pkg = rhn_rpm.RPM_Package(stream) + elif packaging == 'mpm': + import rhn_mpm + a_pkg = rhn_mpm.MPM_Package(stream) + else: + a_pkg = None + return a_pkg BUFFER_SIZE = 16384 DEFAULT_CHECKSUM_TYPE = 'md5' commit 9cee53a98124ec6444e25134b4f4c90829000749 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 10:29:49 2012 +0100 added header_start, header_end diff --git a/backend/common/rhn_pkg.py b/backend/common/rhn_pkg.py index cb70074..e22cbdb 100644 --- a/backend/common/rhn_pkg.py +++ b/backend/common/rhn_pkg.py @@ -41,6 +41,8 @@ class A_Package: """virtual class that implements shared methods for RPM/MPM/DEB package object""" def __init__(self, input_stream = None): self.header = None + self.header_start = 0 + self.header_end = 0 self.input_stream = input_stream self.checksum_type = DEFAULT_CHECKSUM_TYPE self.checksum = None commit 98071ac628f324cc4d5e635de28e1c2552cbe9b4 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Fri Jan 27 10:25:42 2012 +0100 remember payload_size diff --git a/backend/common/rhn_pkg.py b/backend/common/rhn_pkg.py index 56b0232..cb70074 100644 --- a/backend/common/rhn_pkg.py +++ b/backend/common/rhn_pkg.py @@ -45,6 +45,7 @@ class A_Package: self.checksum_type = DEFAULT_CHECKSUM_TYPE self.checksum = None self.payload_stream = None + self.payload_size = None def read_header(self): """reads header from self.input_file""" @@ -53,9 +54,12 @@ class A_Package: def save_payload(self, output_stream): """saves payload to output_stream""" hash = checksum.hashlib.new(self.checksum_type) + output_start = output_stream.tell() self._stream_copy(self.input_stream, output_stream, hash) self.checksum = hash.hexdigest() self.payload_stream = output_stream + self.payload_size = output_stream.tell() - output_start + def _stream_copy(self, source, dest, hash=None): """copies data from the source stream to the destination stream""" commit 0662c9d9f4d0e53b94fcd4cedb28b1e863fe119f Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Thu Jan 26 22:26:45 2012 +0100 compute checksum while saving payload diff --git a/backend/common/rhn_pkg.py b/backend/common/rhn_pkg.py index a2786b5..56b0232 100644 --- a/backend/common/rhn_pkg.py +++ b/backend/common/rhn_pkg.py @@ -16,6 +16,7 @@ import rhn_rpm import rhn_mpm import rhn_deb +import checksum def rhn_pkg(filename=None, file=None, fd=None): if filename: @@ -34,12 +35,16 @@ def get_package_header(filename=None, file=None, fd=None): BUFFER_SIZE = 16384 +DEFAULT_CHECKSUM_TYPE = 'md5' class A_Package: """virtual class that implements shared methods for RPM/MPM/DEB package object""" def __init__(self, input_stream = None): self.header = None self.input_stream = input_stream + self.checksum_type = DEFAULT_CHECKSUM_TYPE + self.checksum = None + self.payload_stream = None def read_header(self): """reads header from self.input_file""" @@ -47,12 +52,17 @@ class A_Package: def save_payload(self, output_stream): """saves payload to output_stream""" - self._stream_copy(self.input_stream, output_stream) + hash = checksum.hashlib.new(self.checksum_type) + self._stream_copy(self.input_stream, output_stream, hash) + self.checksum = hash.hexdigest() + self.payload_stream = output_stream - def _stream_copy(self, source, dest): + def _stream_copy(self, source, dest, hash=None): """copies data from the source stream to the destination stream""" while True: buf = source.read(BUFFER_SIZE) if not buf: break dest.write(buf) + if hash: + hash.update(buf) commit 2bb06c25da7d7bec7738ebd884d4d86d81c6ecf5 Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Thu Jan 26 22:26:45 2012 +0100 rewritten package_push handler to use package object diff --git a/backend/server/rhnPackageUpload.py b/backend/server/rhnPackageUpload.py index a3b1f95..395b368 100644 --- a/backend/server/rhnPackageUpload.py +++ b/backend/server/rhnPackageUpload.py @@ -129,21 +129,14 @@ def relative_path_from_nevra_without_package_name(nevra, org_id, checksum_type, return get_package_path_without_package_name(nevra, org_id, CFG.PREPENDED_DIR, checksum_type, checksum) -def push_package(header, temp_path, payload_stream, checksum_type, checksum, org_id=None, force=None, - header_start=None, header_end=None, channels=[], relative_path=None): - """Uploads an RPM package - """ - - # Get the payload size - log_debug(3, CFG.MOUNT_POINT, relative_path, force, org_id) - payload_stream.seek(0, 2) - payload_size = payload_stream.tell() - payload_stream.seek(0, 0) +def push_package(a_pkg, org_id=None, force=None, channels=[], relative_path=None): + """Uploads a package""" # First write the package to the filesystem to final location try: - importLib.move_package(temp_path, basedir=CFG.MOUNT_POINT, - relpath=relative_path, checksum_type=checksum_type, checksum=checksum, force=1) + importLib.move_package(a_pkg.payload_stream.name, basedir=CFG.MOUNT_POINT, + relpath=relative_path, + checksum_type=a_pkg.checksum_type, checksum=a_pkg.checksum, force=1) except OSError, e: raise rhnFault(50, "Package upload failed: %s" % e), None, sys.exc_info()[2] except importLib.FileConflictError: @@ -151,10 +144,10 @@ def push_package(header, temp_path, payload_stream, checksum_type, checksum, org except: raise rhnFault(50, "File error"), None, sys.exc_info()[2] - pkg = mpmSource.create_package(header, size=payload_size, - checksum_type=checksum_type, checksum=checksum, - relpath=relative_path, org_id=org_id, header_start=header_start, - header_end=header_end, channels=channels) + pkg = mpmSource.create_package(a_pkg.header, size=a_pkg.payload_size, + checksum_type=a_pkg.checksum_type, checksum=a_pkg.checksum, + relpath=relative_path, org_id=org_id, header_start=a_pkg.header_start, + header_end=a_pkg.header_end, channels=channels) batch = importLib.Collection() batch.append(pkg) @@ -166,7 +159,7 @@ def push_package(header, temp_path, payload_stream, checksum_type, checksum, org else: upload_force = 0 importer = packageImport.packageImporter(batch, backend, - source=header.is_source, caller="server.app.uploadPackage") + source=a_pkg.header.is_source, caller="server.app.uploadPackage") importer.setUploadForce(upload_force) importer.run() @@ -184,8 +177,8 @@ def push_package(header, temp_path, payload_stream, checksum_type, checksum, org # Determine the type of packaging that was used to create the package. packaging = 'rpm' - if hasattr(header, 'packaging'): - packaging = header.packaging + if hasattr(a_pkg.header, 'packaging'): + packaging = a_pkg.header.packaging # MPMs do not store their headers on disk, so we must avoid performing # operations which rely on information only contained in the headers @@ -230,12 +223,12 @@ def push_package(header, temp_path, payload_stream, checksum_type, checksum, org (ps.org_id is null and :org_id is null) ) """ - if header.is_source: + if a_pkg.header.is_source: h_package_table = 'rhnPackageSource' else: h_package_table = 'rhnPackage' h_path = rhnSQL.prepare(h_path_sql % h_package_table) - h_path.execute(ctype=checksum_type, csum=checksum, org_id = org_id) + h_path.execute(ctype=a_pkg.checksum_type, csum=a_pkg.checksum, org_id = org_id) rs_path = h_path.fetchall_dict() path_dict = {} @@ -253,15 +246,17 @@ def push_package(header, temp_path, payload_stream, checksum_type, checksum, org where c.checksum = :csum and c.checksum_type = :ctype) """) - h_upd.execute(path=relative_path, ctype=checksum_type, csum=checksum) + h_upd.execute(path=relative_path, ctype=a_pkg.checksum_type, + csum=a_pkg.checksum) # commit the transactions rhnSQL.commit() - if not header.is_source: + if not a_pkg.header.is_source: # Process Package Key information - server_packages.processPackageKeyAssociations(header, checksum_type, checksum) + server_packages.processPackageKeyAssociations(a_pkg.header, + a_pkg.checksum_type, a_pkg.checksum) - if not header.is_source: + if not a_pkg.header.is_source: errataCache.schedule_errata_cache_update(importer.affected_channels) log_debug(2, "Returning") @@ -289,6 +284,30 @@ def _key_ids(sigs): l.sort() return l +def save_uploaded_package(stream, nevra, org_id, packaging, + checksum_type=None, checksum=None): + a_pkg = rhn_pkg.package_from_stream(stream, packaging=packaging) + a_pkg.read_header() + + temp_dir = os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR, org_id, 'stage') + if not os.path.isdir(temp_dir): + os.makedirs(temp_dir) + temp_stream = tempfile.NamedTemporaryFile(dir = temp_dir, + prefix = '-'.join((nevra[0], nevra[2], nevra[3], nevra[4]))) + a_pkg.save_payload(temp_stream) + + if checksum_type and checksum: + # verify checksum + if not (checksum_type == a_pkg.checksum_type + and checksum == a_pkg.checksum): + log_debug(1, "Mismatching checksums: expected %s:%s got %s:%s" % + (checksum_type, checksum, + a_pkg.checksum_type, a_pkg.checksum)) + raise rhnFault(104, "Mismatching information") + + temp_stream.delete=False + return a_pkg + def load_package(package_stream): if package_stream.name.endswith('.deb'): try: diff --git a/backend/upload_server/handlers/package_push/package_push.py b/backend/upload_server/handlers/package_push/package_push.py index 44d8a5e..d97b26c 100644 --- a/backend/upload_server/handlers/package_push/package_push.py +++ b/backend/upload_server/handlers/package_push/package_push.py @@ -108,38 +108,22 @@ class PackagePush(basePackageUpload.BasePackageUpload): if ret != apache.OK: return ret - temp_stream = rhnPackageUpload.write_temp_file(req, 16384, - os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR, - str(self.org_id), 'stage'), - '-'.join((self.package_name, self.package_version, - self.package_release, self.package_arch)), - self.packaging) - - header, payload_stream, header_start, header_end = \ - rhnPackageUpload.load_package(temp_stream) - - # Sanity check - removed, the package path can no longer be determined - # without the header - checksum_type = header.checksum_type() - checksum = getFileChecksum(checksum_type, file=payload_stream) + a_pkg = rhnPackageUpload.save_uploaded_package(req, + (self.package_name, None, self.package_version, + self.package_release, self.package_arch), + str(self.org_id), + self.packaging, + self.file_checksum_type, self.file_checksum) + self.rel_package_path = rhnPackageUpload.relative_path_from_header( - header, org_id=self.org_id, checksum_type=checksum_type, checksum=checksum) + a_pkg.header, org_id=self.org_id, + checksum_type=a_pkg.checksum_type, checksum=a_pkg.checksum) self.package_path = os.path.join(CFG.MOUNT_POINT, self.rel_package_path) - # Verify the checksum of the bytes we downloaded against the checksum - # presented by rhnpush in the HTTP headers - if not (checksum_type == self.file_checksum_type - and checksum == self.file_checksum): - log_debug(1, "Mismatching checksums: expected", - self.file_checksum_type, self.file_checksum, - "; got:", checksum_type, checksum) - raise rhnFault(104, "Mismatching information") - package_dict, diff_level = rhnPackageUpload.push_package(header, - temp_stream.name, payload_stream, checksum_type, checksum, force=self.force, - header_start=header_start, header_end=header_end, + package_dict, diff_level = rhnPackageUpload.push_package(a_pkg, + force=self.force, relative_path=self.rel_package_path, org_id=self.org_id) - temp_stream.delete=False if diff_level: return self._send_package_diff(req, diff_level, package_dict) commit b9233168fab79474f14d8635d125a2939025f97e Author: Michael Mraka <michael.mraka(a)redhat.com> Date: Thu Jan 26 21:30:23 2012 +0100 created virtual class for RPM/MPM/DEB packages diff --git a/backend/common/rhn_pkg.py b/backend/common/rhn_pkg.py index 3097416..a2786b5 100644 --- a/backend/common/rhn_pkg.py +++ b/backend/common/rhn_pkg.py @@ -31,3 +31,28 @@ def rhn_pkg(filename=None, file=None, fd=None): def get_package_header(filename=None, file=None, fd=None): return rhn_pkg(filename=filename, file=file, fd=fd) \ .get_package_header(filename=filename, file=file, fd=fd) + + +BUFFER_SIZE = 16384 + +class A_Package: + """virtual class that implements shared methods for RPM/MPM/DEB package object""" + def __init__(self, input_stream = None): + self.header = None + self.input_stream = input_stream + + def read_header(self): + """reads header from self.input_file""" + pass + + def save_payload(self, output_stream): + """saves payload to output_stream""" + self._stream_copy(self.input_stream, output_stream) + + def _stream_copy(self, source, dest): + """copies data from the source stream to the destination stream""" + while True: + buf = source.read(BUFFER_SIZE) + if not buf: + break + dest.write(buf)

1 0

Changes to 'refs/tags/MessageQueue-3.26.7-1'
by Miroslav Suchý 03 Feb '12

03 Feb '12

Tag 'MessageQueue-3.26.7-1' created by Miroslav Suchý <msuchy(a)redhat.com> at 2012-02-03 10:17 +0000 Tagging package [MessageQueue] version [3.26.7-1] in directory [monitoring/MessageQueue/]. Changes since spacewalk-admin-1.7.2-1: Miroslav Suchý (1): Automatic commit of package [MessageQueue] release [3.26.7-1]. --- monitoring/MessageQueue/MessageQueue.spec | 5 ++++- rel-eng/packages/MessageQueue | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) ---

1 0

monitoring/MessageQueue rel-eng/packages
by Miroslav Suchý 03 Feb '12

03 Feb '12

monitoring/MessageQueue/MessageQueue.spec | 5 ++++- rel-eng/packages/MessageQueue | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) New commits: commit 67da0378df3409ae26042472dd44909a88b77b02 Author: Miroslav Suchý <msuchy(a)redhat.com> Date: Fri Feb 3 11:17:28 2012 +0100 Automatic commit of package [MessageQueue] release [3.26.7-1]. diff --git a/monitoring/MessageQueue/MessageQueue.spec b/monitoring/MessageQueue/MessageQueue.spec index b3beab1..7cc9314 100644 --- a/monitoring/MessageQueue/MessageQueue.spec +++ b/monitoring/MessageQueue/MessageQueue.spec @@ -7,7 +7,7 @@ %define snmp_qdir %queue_dir/snmp Name: MessageQueue -Version: 3.26.6 +Version: 3.26.7 Release: 1%{?dist} Summary: Message buffer/relay system URL: https://fedorahosted.org/spacewalk @@ -75,6 +75,9 @@ fi rm -rf $RPM_BUILD_ROOT %changelog +* Fri Feb 03 2012 Miroslav Suchý 3.26.7-1 +- If $url is not set, fail with sane message. (msuchy(a)redhat.com) + * Mon Dec 19 2011 Michael Mraka <michael.mraka(a)redhat.com> 3.26.6-1 - 768188 - return mac address of the first available interface diff --git a/rel-eng/packages/MessageQueue b/rel-eng/packages/MessageQueue index 8e18fad..8679dc9 100644 --- a/rel-eng/packages/MessageQueue +++ b/rel-eng/packages/MessageQueue @@ -1 +1 @@ -3.26.6-1 monitoring/MessageQueue/ +3.26.7-1 monitoring/MessageQueue/

1 0

Changes to 'refs/tags/spacewalk-admin-1.7.2-1'
by Jan Pazdziora 03 Feb '12

03 Feb '12

Tag 'spacewalk-admin-1.7.2-1' created by Jan Pazdziora <jpazdziora(a)redhat.com> at 2012-02-03 10:10 +0000 Tagging package [spacewalk-admin] version [1.7.2-1] in directory [spacewalk/admin/]. Changes since oracle-lib-compat-11.2.0.6-1: Jan Pazdziora (1): Automatic commit of package [spacewalk-admin] release [1.7.2-1]. --- rel-eng/packages/spacewalk-admin | 2 +- spacewalk/admin/spacewalk-admin.spec | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) ---

1 0

rel-eng/packages spacewalk/admin
by Jan Pazdziora 03 Feb '12

03 Feb '12

rel-eng/packages/spacewalk-admin | 2 +- spacewalk/admin/spacewalk-admin.spec | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) New commits: commit 461bdfdc3f4d236b0f8c8d844208a112f544e87d Author: Jan Pazdziora <jpazdziora(a)redhat.com> Date: Fri Feb 3 11:10:17 2012 +0100 Automatic commit of package [spacewalk-admin] release [1.7.2-1]. diff --git a/rel-eng/packages/spacewalk-admin b/rel-eng/packages/spacewalk-admin index b3ee305..ffa6177 100644 --- a/rel-eng/packages/spacewalk-admin +++ b/rel-eng/packages/spacewalk-admin @@ -1 +1 @@ -1.7.1-1 spacewalk/admin/ +1.7.2-1 spacewalk/admin/ diff --git a/spacewalk/admin/spacewalk-admin.spec b/spacewalk/admin/spacewalk-admin.spec index 15e80ba..ebd6d7a 100644 --- a/spacewalk/admin/spacewalk-admin.spec +++ b/spacewalk/admin/spacewalk-admin.spec @@ -2,7 +2,7 @@ Summary: Various utility scripts and data files for RHN Satellite installations Name: spacewalk-admin URL: https://fedorahosted.org/spacewalk -Version: 1.7.1 +Version: 1.7.2 Release: 1%{?dist} Source0: https://fedorahosted.org/releases/s/p/spacewalk/%{name}-%{version}.tar.gz License: GPLv2 @@ -70,6 +70,10 @@ rm -rf $RPM_BUILD_ROOT %{_mandir}/man8/rhn-config-satellite.pl.8* %changelog +* Fri Feb 03 2012 Jan Pazdziora 1.7.2-1 +- Revert "cat /var/run/.pid nondeterministically failes with 'cat: write error: + Broken pipe', thus it's better to do it in two separate steps." + * Thu Feb 02 2012 Michael Mraka <michael.mraka(a)redhat.com> 1.7.1-1 - fixed write error: Broken pipe

1 0

Changes to 'refs/tags/oracle-lib-compat-11.2.0.6-1'
by Jan Pazdziora 03 Feb '12

03 Feb '12

Tag 'oracle-lib-compat-11.2.0.6-1' created by Jan Pazdziora <jpazdziora(a)redhat.com> at 2012-02-03 10:09 +0000 Tagging package [oracle-lib-compat] version [11.2.0.6-1] in directory [projects/oracle-lib-compat/]. Changes since spacewalk-utils-1.7.8-1: Jan Pazdziora (3): Avoid cat: write error: Broken pipe when calling tomcat service under trap '' PIPE Revert "cat /var/run/.pid nondeterministically failes with 'cat: write error: Broken pipe', thus it's better to do it in two separate steps." Automatic commit of package [oracle-lib-compat] release [11.2.0.6-1]. Justin Sherrill (3): errata date clone - fixing man page formatting errata date clone - man page fix errata date clone - another man page fix --- projects/oracle-lib-compat/oracle-lib-compat.spec | 6 +++- projects/oracle-lib-compat/tomcat-setenv.sh | 2 - rel-eng/packages/oracle-lib-compat | 2 - spacewalk/admin/spacewalk-service | 3 -- utils/spacewalk-clone-by-date.sgml | 31 +++++++++++++--------- 5 files changed, 27 insertions(+), 17 deletions(-) ---

1 0

← Newer
1
...
48
49
50
51
52
53
54
...
60
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

spacewalk-commits February 2012