There are two particularly troublesome network calls that can fail if the network has issues. Therefore, add a retries option to allow looping until the call succeeds or too many retries happen.
--- mash/__init__.py | 32 +++++++++++++++++++++++++++----- mash/config.py | 1 + 2 files changed, 28 insertions(+), 5 deletions(-)
diff --git a/mash/__init__.py b/mash/__init__.py index 566b8c1..9325794 100644 --- a/mash/__init__.py +++ b/mash/__init__.py @@ -31,6 +31,8 @@ import yum
import rpmUtils.arch
+import xml.parsers.expat + def nevra(pkg): return '%s-%s:%s-%s.%s' % (pkg['name'],pkg['epoch'],pkg['version'],pkg['release'],pkg['arch'])
@@ -214,9 +216,17 @@ class Mash: self.logger.error("ERROR: can't download %s from signed path %s" % (nevra(pkg), srcurl)) return 1 srcurl = os.path.join(koji.pathinfo.build(z), koji.pathinfo.rpm(pkg)) - try: - result = urlgrabber.grabber.urlgrab(srcurl, cachepath) - except: + tries = 0 + downloaded = False + while tries < self.config.retries and not downloaded: + try: + tries += 1 + result = urlgrabber.grabber.urlgrab(srcurl, cachepath) + downloaded = True + except: + self.logger.warning("WARNING: can't download %s from %s, attempt #%d" % (nevra(pkg), srcurl, tries)) + os.remove(cachepath) + if not downloaded: self.logger.error("ERROR: can't download %s from %s" % (nevra(pkg), srcurl)) return 1
@@ -300,8 +310,20 @@ class Mash: os.makedirs(self.config.cachedir, 0755) # Get package list. This is an expensive operation. self.logger.info("Getting package lists for %s..." % (self.config.tag)) - - (pkglist, buildlist) = self.session.listTaggedRPMS(self.config.tag, inherit = self.config.inherit, latest = self.config.latest, rpmsigs = True) + + tries = 0 + downloaded = False + while tries < self.config.retries and not downloaded: + try: + tries += 1 + (pkglist, buildlist) = self.session.listTaggedRPMS(self.config.tag, inherit = self.config.inherit, latest = self.config.latest, rpmsigs = True) + downloaded = True + except xml.parsers.expat.ExpatError: + self.logger.warning("WARNING: can't listTaggedRPMS from koji, attempt #%d" % (tries)) + if not downloaded: + self.logger.error("ERROR: can't listTaggedRPMS from koji") + sys.exit(1) + # filter by key biglist = PackageList(self.config) for pkg in pkglist: diff --git a/mash/config.py b/mash/config.py index e4e9b95..3da98d1 100644 --- a/mash/config.py +++ b/mash/config.py @@ -100,6 +100,7 @@ class MashDistroConfig(config.BaseConfig): hash_packages = config.BoolOption(False) parent_repos = config.ListOption() previous = None + retries = config.Option(1)
def fixup(self, sect): if not self.name:
buildsys@lists.fedoraproject.org