There are two copies of the package downloading code. Trivially merge them to ease maintenance.
Signed-off-by: Mark McLoughlin markmc@redhat.com
Index: pungi/pypungi/gather.py =================================================================== --- pungi.orig/pypungi/gather.py +++ pungi/pypungi/gather.py @@ -292,69 +292,69 @@ class Gather(pypungi.PungiBase): if not srpm in self.srpmlist: self.srpmlist.append(srpm)
+ def _link(self, local, target): + try: + os.link(local, target) + except OSError, e: + if e.errno != 18: # EXDEV + self.logger.error('Got an error linking from cache: %s' % e) + raise OSError, e
- def downloadPackages(self): + # Can't hardlink cross file systems + shutil.copy2(local, target) + + def _downloadPackageList(self, polist, relpkgdir): """Cycle through the list of package objects and download them from their respective repos."""
- - downloads = [] - for pkg in self.polist: - downloads.append('%s.%s' % (pkg.name, pkg.arch)) - downloads.sort() - self.logger.info("Download list: %s" % downloads) - - # Package location within destdir, name subject to change/config - pkgdir = os.path.join(self.config.get('default', 'destdir'), self.config.get('default', 'version'), - self.config.get('default', 'flavor'), - self.config.get('default', 'arch'), - self.config.get('default', 'osdir'), - self.config.get('default', 'product_path')) + pkgdir = os.path.join(self.config.get('default', 'destdir'), + self.config.get('default', 'version'), + self.config.get('default', 'flavor'), + relpkgdir)
if not os.path.exists(pkgdir): os.makedirs(pkgdir)
- for pkg in self.polist: - repo = self.ayum.repos.getRepo(pkg.repoid) - remote = pkg.relativepath - local = os.path.basename(remote) - local = os.path.join(self.config.get('default', 'cachedir'), local) - if os.path.exists(local) and self.verifyCachePkg(pkg, local): + for po in polist: + repo = self.ayum.repos.getRepo(po.repoid) + + basename = os.path.basename(po.relativepath) + + local = os.path.join(self.config.get('default', 'cachedir'), basename) + target = os.path.join(pkgdir, basename) + + if os.path.exists(local) and self.verifyCachePkg(po, local): self.logger.debug("%s already exists and appears to be complete" % local) - target = os.path.join(pkgdir, os.path.basename(remote)) if os.path.exists(target): os.remove(target) # avoid traceback after interrupted download - try: - os.link(local, target) - except OSError, e: - if e.errno == 18: - # Can't hardlink cross file systems - shutil.copy2(local, target) - else: - self.logger.error('Got an error linking from cache: %s' % e) - raise OSError, e + self._link(local, target) continue
# Disable cache otherwise things won't download repo.cache = 0 - self.logger.info('Downloading %s' % os.path.basename(remote)) - pkg.localpath = local # Hack: to set the localpath to what we want. + self.logger.info('Downloading %s' % basename) + po.localpath = local # Hack: to set the localpath to what we want.
# do a little dance for file:// repos... - path = repo.getPackage(pkg) + path = repo.getPackage(po) if not os.path.exists(local) or not os.path.samefile(path, local): shutil.copy2(path, local)
- try: - os.link(local, os.path.join(pkgdir, os.path.basename(remote))) - except OSError, e: - if e.errno == 18: - # Can't hardlink cross file systems - shutil.copy2(local, os.path.join(pkgdir, os.path.basename(remote))) - else: - self.logger.error('Got an error linking from cache: %s' % e) - raise OSError, e + self._link(local, target)
+ def downloadPackages(self): + """Download the package objects obtained in getPackageObjects().""" + + downloads = [] + for pkg in self.polist: + downloads.append('%s.%s' % (pkg.name, pkg.arch)) + downloads.sort() + self.logger.info("Download list: %s" % downloads) + + self._downloadPackageList(self.polist, + os.path.join(self.config.get('default', 'arch'), + self.config.get('default', 'osdir'), + self.config.get('default', 'product_path')))
self.logger.info('Finished downloading packages.')
@@ -425,51 +425,4 @@ class Gather(pypungi.PungiBase): sys.exit(1)
# do the downloads - pkgdir = os.path.join(self.config.get('default', 'destdir'), self.config.get('default', 'version'), - self.config.get('default', 'flavor'), 'source', 'SRPMS') - - if not os.path.exists(pkgdir): - os.makedirs(pkgdir) - - for pkg in srpmpolist: - repo = self.ayum.repos.getRepo(pkg.repoid) - remote = pkg.relativepath - local = os.path.basename(remote) - local = os.path.join(self.config.get('default', 'cachedir'), local) - if os.path.exists(local) and self.verifyCachePkg(pkg, local): - self.logger.debug("%s already exists and appears to be complete" % local) - if os.path.exists(os.path.join(pkgdir, os.path.basename(remote))) and self.verifyCachePkg(pkg, os.path.join(pkgdir, os.path.basename(remote))): - self.logger.debug("%s already exists in tree and appears to be complete" % local) - else: - try: - os.link(local, os.path.join(pkgdir, os.path.basename(remote))) - except OSError, e: - if e.errno == 18: - # Can't hardlink cross file systems - shutil.copy2(local, os.path.join(pkgdir, os.path.basename(remote))) - else: - self.logger.error('Got an error linking from cache: %s' % e) - raise OSError, e - - continue - - # Disable cache otherwise things won't download - repo.cache = 0 - self.logger.info('Downloading %s' % os.path.basename(remote)) - pkg.localpath = local # Hack: to set the localpath to what we want. - - # do a little dance for file:// repos... - path = repo.getPackage(pkg) - if not os.path.exists(local) or not os.path.samefile(path, local): - shutil.copy2(path, local) - - try: - os.link(local, os.path.join(pkgdir, os.path.basename(remote))) - except OSError, e: - if e.errno == 18: - # Can't hardlink cross file systems - shutil.copy2(local, target) - else: - self.logger.error('Got an error linking from cache: %s' % e) - raise OSError, e - + self._downloadPackageList(srpmpolist, os.path.join('source', 'SRPMS'))
buildsys@lists.fedoraproject.org
-
Mark McLoughlin