[python-urlgrabber] Update to latest HEAD.
Zdeněk Pavlas
zpavlas at fedoraproject.org
Tue Sep 4 13:42:14 UTC 2012
commit c0934722e032211cc226a575fd9a170ef34a6c67
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Tue Sep 4 15:42:04 2012 +0200
Update to latest HEAD.
python-urlgrabber.spec | 6 +-
urlgrabber-HEAD.patch | 365 +++++++++++++++++++++++++++++++++++++----------
2 files changed, 292 insertions(+), 79 deletions(-)
---
diff --git a/python-urlgrabber.spec b/python-urlgrabber.spec
index c33f014..2975737 100644
--- a/python-urlgrabber.spec
+++ b/python-urlgrabber.spec
@@ -3,7 +3,7 @@
Summary: A high-level cross-protocol url-grabber
Name: python-urlgrabber
Version: 3.9.1
-Release: 19%{?dist}
+Release: 20%{?dist}
Source0: urlgrabber-%{version}.tar.gz
Patch1: urlgrabber-HEAD.patch
@@ -44,6 +44,10 @@ rm -rf $RPM_BUILD_ROOT
%attr(0755,root,root) %{_libexecdir}/urlgrabber-ext-down
%changelog
+* Tue Sep 4 2012 Zdeněk Pavlas <zpavlas at redhat.com> - 3.9.1-20
+- Update to latest HEAD.
+- Fixed BZ 851178, 854075.
+
* Mon Aug 27 2012 Zdeněk Pavlas <zpavlas at redhat.com> - 3.9.1-19
- timedhosts: defer 1st update until a 1MB+ download. BZ 851178
diff --git a/urlgrabber-HEAD.patch b/urlgrabber-HEAD.patch
index a092d3a..ef304ad 100644
--- a/urlgrabber-HEAD.patch
+++ b/urlgrabber-HEAD.patch
@@ -73,10 +73,10 @@ index 518e512..09cd896 100644
print __doc__
diff --git a/scripts/urlgrabber-ext-down b/scripts/urlgrabber-ext-down
new file mode 100755
-index 0000000..3da55a4
+index 0000000..3dafb12
--- /dev/null
+++ b/scripts/urlgrabber-ext-down
-@@ -0,0 +1,72 @@
+@@ -0,0 +1,75 @@
+#! /usr/bin/python
+# A very simple external downloader
+# Copyright 2011-2012 Zdenek Pavlas
@@ -134,18 +134,21 @@ index 0000000..3da55a4
+ if opts.progress_obj:
+ opts.progress_obj = ProxyProgress()
+ opts.progress_obj._id = cnt
-+ tm = time.time()
++
++ dlsz = dltm = 0
+ try:
+ fo = PyCurlFileObject(opts.url, opts.filename, opts)
+ fo._do_grab()
+ fo.fo.close()
+ size = fo._amount_read
-+ dlsz = size - fo._reget_length
++ if fo._tm_last:
++ dlsz = fo._tm_last[0] - fo._tm_first[0]
++ dltm = fo._tm_last[1] - fo._tm_first[1]
+ ug_err = 'OK'
+ except URLGrabError, e:
-+ size = dlsz = 0
++ size = 0
+ ug_err = '%d %s' % e.args
-+ write('%d %d %d %.3f %s\n', opts._id, size, dlsz, time.time() - tm, ug_err)
++ write('%d %d %d %.3f %s\n', opts._id, size, dlsz, dltm, ug_err)
+
+if __name__ == '__main__':
+ main()
@@ -233,7 +236,7 @@ index 3e5f3b7..8eeaeda 100644
return (fb,lb)
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
-index e090e90..daa478d 100644
+index e090e90..01218b0 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -49,11 +49,26 @@ GENERAL ARGUMENTS (kwargs)
@@ -678,7 +681,7 @@ index e090e90..daa478d 100644
if scheme == 'file' and not opts.copy_local:
# just return the name of the local file - don't make a
# copy currently
-@@ -950,41 +1114,49 @@ class URLGrabber:
+@@ -950,41 +1114,51 @@ class URLGrabber:
elif not opts.range:
if not opts.checkfunc is None:
@@ -700,11 +703,13 @@ index e090e90..daa478d 100644
+ return filename
+
def retryfunc(opts, url, filename):
-+ tm = time.time()
fo = PyCurlFileObject(url, filename, opts)
try:
fo._do_grab()
-+ _TH.update(url, fo._amount_read - fo._reget_length, time.time() - tm, None)
++ if fo._tm_last:
++ dlsz = fo._tm_last[0] - fo._tm_first[0]
++ dltm = fo._tm_last[1] - fo._tm_first[1]
++ _TH.update(url, dlsz, dltm, None)
if not opts.checkfunc is None:
- cb_func, cb_args, cb_kwargs = \
- self._make_callback(opts.checkfunc)
@@ -743,7 +748,7 @@ index e090e90..daa478d 100644
if limit is not None:
limit = limit + 1
-@@ -1000,12 +1172,8 @@ class URLGrabber:
+@@ -1000,12 +1174,8 @@ class URLGrabber:
else: s = fo.read(limit)
if not opts.checkfunc is None:
@@ -758,7 +763,7 @@ index e090e90..daa478d 100644
finally:
fo.close()
return s
-@@ -1020,6 +1188,7 @@ class URLGrabber:
+@@ -1020,6 +1190,7 @@ class URLGrabber:
return s
def _make_callback(self, callback_obj):
@@ -766,7 +771,7 @@ index e090e90..daa478d 100644
if callable(callback_obj):
return callback_obj, (), {}
else:
-@@ -1030,7 +1199,7 @@ class URLGrabber:
+@@ -1030,7 +1201,7 @@ class URLGrabber:
default_grabber = URLGrabber()
@@ -775,13 +780,15 @@ index e090e90..daa478d 100644
def __init__(self, url, filename, opts):
self.fo = None
self._hdr_dump = ''
-@@ -1052,10 +1221,11 @@ class PyCurlFileObject():
+@@ -1052,10 +1223,13 @@ class PyCurlFileObject():
self._reget_length = 0
self._prog_running = False
self._error = (None, None)
- self.size = None
+ self.size = 0
+ self._hdr_ended = False
++ self._tm_first = None
++ self._tm_last = None
self._do_open()
-
@@ -789,7 +796,20 @@ index e090e90..daa478d 100644
def __getattr__(self, name):
"""This effectively allows us to wrap at the instance level.
Any attribute not found in _this_ object will be searched for
-@@ -1085,9 +1255,14 @@ class PyCurlFileObject():
+@@ -1067,6 +1241,12 @@ class PyCurlFileObject():
+
+ def _retrieve(self, buf):
+ try:
++ tm = self._amount_read + len(buf), time.time()
++ if self._tm_first is None:
++ self._tm_first = tm
++ else:
++ self._tm_last = tm
++
+ if not self._prog_running:
+ if self.opts.progress_obj:
+ size = self.size + self._reget_length
+@@ -1085,9 +1265,14 @@ class PyCurlFileObject():
return -1
def _hdr_retrieve(self, buf):
@@ -805,7 +825,7 @@ index e090e90..daa478d 100644
try:
self._hdr_dump += buf
# we have to get the size before we do the progress obj start
-@@ -1104,7 +1279,17 @@ class PyCurlFileObject():
+@@ -1104,7 +1289,17 @@ class PyCurlFileObject():
s = parse150(buf)
if s:
self.size = int(s)
@@ -824,7 +844,7 @@ index e090e90..daa478d 100644
return len(buf)
except KeyboardInterrupt:
return pycurl.READFUNC_ABORT
-@@ -1113,8 +1298,10 @@ class PyCurlFileObject():
+@@ -1113,8 +1308,10 @@ class PyCurlFileObject():
if self._parsed_hdr:
return self._parsed_hdr
statusend = self._hdr_dump.find('\n')
@@ -835,7 +855,7 @@ index e090e90..daa478d 100644
self._parsed_hdr = mimetools.Message(hdrfp)
return self._parsed_hdr
-@@ -1127,6 +1314,9 @@ class PyCurlFileObject():
+@@ -1127,6 +1324,9 @@ class PyCurlFileObject():
if not opts:
opts = self.opts
@@ -845,7 +865,7 @@ index e090e90..daa478d 100644
# defaults we're always going to set
self.curl_obj.setopt(pycurl.NOPROGRESS, False)
-@@ -1136,11 +1326,21 @@ class PyCurlFileObject():
+@@ -1136,11 +1336,21 @@ class PyCurlFileObject():
self.curl_obj.setopt(pycurl.PROGRESSFUNCTION, self._progress_update)
self.curl_obj.setopt(pycurl.FAILONERROR, True)
self.curl_obj.setopt(pycurl.OPT_FILETIME, True)
@@ -867,7 +887,7 @@ index e090e90..daa478d 100644
# maybe to be options later
self.curl_obj.setopt(pycurl.FOLLOWLOCATION, True)
-@@ -1148,9 +1348,11 @@ class PyCurlFileObject():
+@@ -1148,9 +1358,11 @@ class PyCurlFileObject():
# timeouts
timeout = 300
@@ -882,7 +902,7 @@ index e090e90..daa478d 100644
# ssl options
if self.scheme == 'https':
-@@ -1158,13 +1360,16 @@ class PyCurlFileObject():
+@@ -1158,13 +1370,16 @@ class PyCurlFileObject():
self.curl_obj.setopt(pycurl.CAPATH, opts.ssl_ca_cert)
self.curl_obj.setopt(pycurl.CAINFO, opts.ssl_ca_cert)
self.curl_obj.setopt(pycurl.SSL_VERIFYPEER, opts.ssl_verify_peer)
@@ -900,7 +920,7 @@ index e090e90..daa478d 100644
if opts.ssl_cert_type:
self.curl_obj.setopt(pycurl.SSLCERTTYPE, opts.ssl_cert_type)
if opts.ssl_key_pass:
-@@ -1187,28 +1392,26 @@ class PyCurlFileObject():
+@@ -1187,28 +1402,26 @@ class PyCurlFileObject():
if hasattr(opts, 'raw_throttle') and opts.raw_throttle():
self.curl_obj.setopt(pycurl.MAX_RECV_SPEED_LARGE, int(opts.raw_throttle()))
@@ -945,7 +965,7 @@ index e090e90..daa478d 100644
# our url
self.curl_obj.setopt(pycurl.URL, self.url)
-@@ -1228,12 +1431,14 @@ class PyCurlFileObject():
+@@ -1228,12 +1441,14 @@ class PyCurlFileObject():
code = self.http_code
errcode = e.args[0]
@@ -962,7 +982,7 @@ index e090e90..daa478d 100644
# this is probably wrong but ultimately this is what happens
# we have a legit http code and a pycurl 'writer failed' code
-@@ -1244,23 +1449,23 @@ class PyCurlFileObject():
+@@ -1244,23 +1459,23 @@ class PyCurlFileObject():
raise KeyboardInterrupt
elif errcode == 28:
@@ -993,7 +1013,7 @@ index e090e90..daa478d 100644
# this is probably wrong but ultimately this is what happens
# we have a legit http code and a pycurl 'writer failed' code
# which almost always means something aborted it from outside
-@@ -1272,33 +1477,94 @@ class PyCurlFileObject():
+@@ -1272,33 +1487,94 @@ class PyCurlFileObject():
elif errcode == 58:
msg = _("problem with the local client certificate")
err = URLGrabError(14, msg)
@@ -1095,7 +1115,7 @@ index e090e90..daa478d 100644
def _do_open(self):
self.curl_obj = _curl_cache
-@@ -1333,7 +1599,11 @@ class PyCurlFileObject():
+@@ -1333,7 +1609,11 @@ class PyCurlFileObject():
if self.opts.range:
rt = self.opts.range
@@ -1108,7 +1128,7 @@ index e090e90..daa478d 100644
if rt:
header = range_tuple_to_header(rt)
-@@ -1434,21 +1704,46 @@ class PyCurlFileObject():
+@@ -1434,21 +1714,46 @@ class PyCurlFileObject():
#fh, self._temp_name = mkstemp()
#self.fo = open(self._temp_name, 'wb')
@@ -1162,7 +1182,7 @@ index e090e90..daa478d 100644
else:
#self.fo = open(self._temp_name, 'r')
self.fo.seek(0)
-@@ -1526,17 +1821,20 @@ class PyCurlFileObject():
+@@ -1526,17 +1831,20 @@ class PyCurlFileObject():
if self._prog_running:
downloaded += self._reget_length
self.opts.progress_obj.update(downloaded)
@@ -1188,7 +1208,7 @@ index e090e90..daa478d 100644
msg = _("Downloaded more than max size for %s: %s > %s") \
% (self.url, cur, max_size)
-@@ -1544,13 +1842,6 @@ class PyCurlFileObject():
+@@ -1544,13 +1852,6 @@ class PyCurlFileObject():
return True
return False
@@ -1202,7 +1222,7 @@ index e090e90..daa478d 100644
def read(self, amt=None):
self._fill_buffer(amt)
if amt is None:
-@@ -1582,9 +1873,21 @@ class PyCurlFileObject():
+@@ -1582,9 +1883,21 @@ class PyCurlFileObject():
self.opts.progress_obj.end(self._amount_read)
self.fo.close()
@@ -1225,7 +1245,7 @@ index e090e90..daa478d 100644
#####################################################################
# DEPRECATED FUNCTIONS
-@@ -1621,6 +1924,460 @@ def retrygrab(url, filename=None, copy_local=0, close_connection=0,
+@@ -1621,6 +1934,466 @@ def retrygrab(url, filename=None, copy_local=0, close_connection=0,
#####################################################################
@@ -1378,7 +1398,7 @@ index e090e90..daa478d 100644
+ if DEBUG: DEBUG.info('success')
+ else:
+ ug_err = URLGrabError(int(line[4]), line[5])
-+ if DEBUG: DEBUG.info('failure: %s', err)
++ if DEBUG: DEBUG.info('failure: %s', ug_err)
+ _TH.update(opts.url, int(line[2]), float(line[3]), ug_err, opts.async[0])
+ ret.append((opts, size, ug_err))
+ return ret
@@ -1474,10 +1494,19 @@ index e090e90..daa478d 100644
+ for opts, size, ug_err in dl.perform():
+ key, limit = opts.async
+ host_con[key] -= 1
++
++ if ug_err is None:
++ if opts.checkfunc:
++ try: _run_callback(opts.checkfunc, opts)
++ except URLGrabError, ug_err: pass
++
+ if opts.progress_obj:
+ if opts.multi_progress_obj:
-+ opts.multi_progress_obj.re.total += size - opts.size # correct totals
-+ opts._progress.end(size)
++ if ug_err:
++ opts._progress.failure(None)
++ else:
++ opts.multi_progress_obj.re.total += size - opts.size # correct totals
++ opts._progress.end(size)
+ opts.multi_progress_obj.removeMeter(opts._progress)
+ else:
+ opts.progress_obj.start(text=opts.text, now=opts._progress)
@@ -1486,11 +1515,7 @@ index e090e90..daa478d 100644
+ del opts._progress
+
+ if ug_err is None:
-+ if opts.checkfunc:
-+ try: _run_callback(opts.checkfunc, opts)
-+ except URLGrabError, ug_err: pass
-+ if ug_err is None:
-+ continue
++ continue
+
+ retry = opts.retry or 0
+ if opts.failure_callback:
@@ -1558,8 +1583,9 @@ index e090e90..daa478d 100644
+ speed = _TH.estimate(key)
+ speed /= 1 + host_con.get(key, 0)
+
-+ # 2-tuple to select mirror with least failures
-+ speed = -failed.get(key, 0), speed
++ # order by: least failures, private flag, best speed
++ private = mirror.get('kwargs', {}).get('private', False)
++ speed = -failed.get(key, 0), private, speed
+ if best is None or speed > best_speed:
+ best = mirror
+ best_speed = speed
@@ -1831,9 +1857,20 @@ index dad410b..b17be17 100644
def urlopen(self, url, **kwargs):
kw = dict(kwargs)
diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py
-index dd07c6a..ad57dbc 100644
+index dd07c6a..077fd99 100644
--- a/urlgrabber/progress.py
+++ b/urlgrabber/progress.py
+@@ -133,8 +133,8 @@ class BaseMeter:
+ # for a real gui, you probably want to override and put a call
+ # to your mainloop iteration function here
+ if now is None: now = time.time()
+- if (now >= self.last_update_time + self.update_period) or \
+- not self.last_update_time:
++ if (not self.last_update_time or
++ (now >= self.last_update_time + self.update_period)):
+ self.re.update(amount_read, now)
+ self.last_amount_read = amount_read
+ self.last_update_time = now
@@ -211,6 +211,21 @@ def text_meter_total_size(size, downloaded=0):
# 4. + ( 5, total: 32)
#
@@ -1856,7 +1893,38 @@ index dd07c6a..ad57dbc 100644
class TextMeter(BaseMeter):
def __init__(self, fo=sys.stderr):
BaseMeter.__init__(self)
-@@ -259,13 +274,10 @@ class TextMeter(BaseMeter):
+@@ -218,7 +233,6 @@ class TextMeter(BaseMeter):
+
+ def _do_update(self, amount_read, now=None):
+ etime = self.re.elapsed_time()
+- fetime = format_time(etime)
+ fread = format_number(amount_read)
+ #self.size = None
+ if self.text is not None:
+@@ -234,16 +248,20 @@ class TextMeter(BaseMeter):
+
+ # Include text + ui_rate in minimal
+ tl = TerminalLine(8, 8+1+8)
++ if tl._llen > 80:
++ use_hours = True # For big screens, make it more readable.
++ else:
++ use_hours = False
+ ui_size = tl.add(' | %5sB' % fread)
+ if self.size is None:
+- ui_time = tl.add(' %9s' % fetime)
++ ui_time = tl.add(' %9s' % format_time(etime, use_hours))
+ ui_end = tl.add(' ' * 5)
+ ui_rate = tl.add(' %5sB/s' % ave_dl)
+ out = '%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
+ ui_rate, ui_size, ui_time, ui_end)
+ else:
+ rtime = self.re.remaining_time()
+- frtime = format_time(rtime)
++ frtime = format_time(rtime, use_hours)
+ frac = self.re.fraction_read()
+
+ ui_time = tl.add(' %9s' % frtime)
+@@ -259,13 +277,10 @@ class TextMeter(BaseMeter):
ui_rate = tl.add(' %5sB/s' % ave_dl)
# Make text grow a bit before we start growing the bar too
blen = 4 + tl.rest_split(8 + 8 + 4)
@@ -1874,21 +1942,36 @@ index dd07c6a..ad57dbc 100644
self.fo.write(out)
self.fo.flush()
-@@ -284,12 +296,7 @@ class TextMeter(BaseMeter):
+@@ -274,7 +289,6 @@ class TextMeter(BaseMeter):
+ global _text_meter_total_size
+ global _text_meter_sofar_size
+
+- total_time = format_time(self.re.elapsed_time())
+ total_size = format_number(amount_read)
+ if self.text is not None:
+ text = self.text
+@@ -282,14 +296,13 @@ class TextMeter(BaseMeter):
+ text = self.basename
+
tl = TerminalLine(8)
- ui_size = tl.add(' | %5sB' % total_size)
- ui_time = tl.add(' %9s' % total_time)
+- ui_size = tl.add(' | %5sB' % total_size)
+- ui_time = tl.add(' %9s' % total_time)
- not_done = self.size is not None and amount_read != self.size
- if not_done:
- ui_end = tl.add(' ... ')
-- else:
++ if tl._llen > 80:
++ use_hours = True # For big screens, make it more readable.
+ else:
- ui_end = tl.add(' ' * 5)
-
++ use_hours = False
++ ui_size = tl.add(' | %5sB' % total_size)
++ ui_time = tl.add(' %9s' % format_time(self.re.elapsed_time(),use_hours))
+ ui_end, not_done = _term_add_end(tl, self.size, amount_read)
out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text,
ui_size, ui_time, ui_end)
self.fo.write(out)
-@@ -331,12 +338,21 @@ class MultiFileHelper(BaseMeter):
+@@ -331,12 +344,21 @@ class MultiFileHelper(BaseMeter):
def message(self, message):
self.master.message_meter(self, message)
@@ -1912,7 +1995,7 @@ index dd07c6a..ad57dbc 100644
self.update_period = 0.3 # seconds
self.numfiles = None
-@@ -369,6 +385,7 @@ class MultiFileMeter:
+@@ -369,6 +391,7 @@ class MultiFileMeter:
def end(self, now=None):
if now is None: now = time.time()
@@ -1920,7 +2003,18 @@ index dd07c6a..ad57dbc 100644
self._do_end(now)
def _do_end(self, now):
-@@ -466,11 +483,21 @@ class MultiFileMeter:
+@@ -407,8 +430,8 @@ class MultiFileMeter:
+ def update_meter(self, meter, now):
+ if not meter in self.meters:
+ raise ValueError('attempt to use orphaned meter')
+- if (now >= self.last_update_time + self.update_period) or \
+- not self.last_update_time:
++ if (not self.last_update_time or
++ (now >= self.last_update_time + self.update_period)):
+ self.re.update(self._amount_read(), now)
+ self.last_update_time = now
+ self._do_update_meter(meter, now)
+@@ -466,34 +489,87 @@ class MultiFileMeter:
class TextMultiFileMeter(MultiFileMeter):
@@ -1933,10 +2027,15 @@ index dd07c6a..ad57dbc 100644
# files: ###/### ###% data: ######/###### ###% time: ##:##:##/##:##:##
+# New output, like TextMeter output...
++# update: No size (minimal: 17 chars)
++# -----------------------------------
++# (<#file>/<#tot files>): <text> <rate> | <current size> <elapsed>
++# 8-48 1 8 3 6 1 7-9 5
++#
+# update: Size, All files
+# -----------------------
+# (<#file>/<#tot files>): <text> <pc> <bar> <rate> | <size> <eta time> ETA
-+# 8-22 1 3-4 1 6-12 1 8 3 6 1 9 1 3 1
++# 8-22 1 3-4 1 6-12 1 8 3 6 1 7-9 1 3 1
+# end
+# ---
+# <text> | <file size> <file elapsed time>
@@ -1944,24 +2043,31 @@ index dd07c6a..ad57dbc 100644
def _do_update_meter(self, meter, now):
self._lock.acquire()
try:
-@@ -480,7 +507,7 @@ class TextMultiFileMeter(MultiFileMeter):
+- format = "files: %3i/%-3i %3i%% data: %6.6s/%-6.6s %3i%% " \
+- "time: %8.8s/%8.8s"
+ df = self.finished_files
tf = self.numfiles or 1
- pf = 100 * float(df)/tf + 0.49
+- pf = 100 * float(df)/tf + 0.49
++ # Don't use "percent of files complete" ...
++ # pf = 100 * float(df)/tf + 0.49
dd = self.re.last_amount_read
- td = self.total_size
+ td = self.re.total
pd = 100 * (self.re.fraction_read() or 0) + 0.49
dt = self.re.elapsed_time()
rt = self.re.remaining_time()
-@@ -491,9 +518,41 @@ class TextMultiFileMeter(MultiFileMeter):
- ftd = format_number(td) + 'B'
- fdt = format_time(dt, 1)
- ftt = format_time(tt, 1)
+- if rt is None: tt = None
+- else: tt = dt + rt
+
+- fdd = format_number(dd) + 'B'
+- ftd = format_number(td) + 'B'
+- fdt = format_time(dt, 1)
+- ftt = format_time(tt, 1)
-
- out = '%-79.79s' % (format % (df, tf, pf, fdd, ftd, pd, fdt, ftt))
- self.fo.write('\r' + out)
-+
+ frac = self.re.fraction_read() or 0
++ pf = 100 * frac
+ ave_dl = format_number(self.re.average_rate())
+
+ # cycle through active meters
@@ -1977,28 +2083,41 @@ index dd07c6a..ad57dbc 100644
+
+ # Include text + ui_rate in minimal
+ tl = TerminalLine(8, 8+1+8)
++ if tl._llen > 80:
++ use_hours = True # For big screens, make it more readable.
++ time_len = 9
++ else:
++ use_hours = False
++ time_len = 7
+
+ ui_size = tl.add(' | %5sB' % format_number(dd))
+
-+ ui_time = tl.add(' %9s' % format_time(rt))
-+ ui_end = tl.add(' ETA ')
-+
-+ ui_sofar_pc = tl.add(' %i%%' % pf,
-+ full_len=len(" (100%)"))
-+ ui_rate = tl.add(' %5sB/s' % ave_dl)
-+
-+ # Make text grow a bit before we start growing the bar too
-+ blen = 4 + tl.rest_split(8 + 8 + 4)
-+ ui_bar = _term_add_bar(tl, blen, frac)
-+ out = '\r%-*.*s%s%s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
-+ ui_sofar_pc, ui_bar,
-+ ui_rate, ui_size, ui_time,
-+ ui_end)
++ if not self.re.total:
++ ui_time = tl.add(' %*s' % (time_len,format_time(dt, use_hours)))
++ ui_end = tl.add(' ' * 5)
++ ui_rate = tl.add(' %5sB/s' % ave_dl)
++ out = '\r%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
++ ui_rate, ui_size, ui_time, ui_end)
++ else:
++ ui_time = tl.add(' %*s' % (time_len,format_time(rt, use_hours)))
++ ui_end = tl.add(' ETA ')
++
++ ui_sofar_pc = tl.add(' %i%%' % pf,
++ full_len=len(" (100%)"))
++ ui_rate = tl.add(' %5sB/s' % ave_dl)
++
++ # Make text grow a bit before we start growing the bar too
++ blen = 4 + tl.rest_split(8 + 8 + 4)
++ ui_bar = _term_add_bar(tl, blen, frac)
++ out = '\r%-*.*s%s%s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
++ ui_sofar_pc, ui_bar,
++ ui_rate, ui_size, ui_time,
++ ui_end)
+ self.fo.write(out)
self.fo.flush()
finally:
self._lock.release()
-@@ -502,18 +561,30 @@ class TextMultiFileMeter(MultiFileMeter):
+@@ -502,24 +578,40 @@ class TextMultiFileMeter(MultiFileMeter):
self._lock.acquire()
try:
format = "%-30.30s %6.6s %8.8s %9.9s"
@@ -2007,7 +2126,7 @@ index dd07c6a..ad57dbc 100644
size = meter.last_amount_read
fsize = format_number(size) + 'B'
et = meter.re.elapsed_time()
- fet = format_time(et, 1)
+- fet = format_time(et, 1)
- frate = format_number(size / et) + 'B/s'
-
- out = '%-79.79s' % (format % (fn, fsize, fet, frate))
@@ -2016,15 +2135,20 @@ index dd07c6a..ad57dbc 100644
+ df = self.finished_files
+ tf = self.numfiles or 1
+
-+ total_time = format_time(et)
+ total_size = format_number(size)
+ text = meter.text or meter.basename
+ if tf > 1:
+ text = '(%u/%u): %s' % (df, tf, text)
+
+ tl = TerminalLine(8)
++ if tl._llen > 80:
++ use_hours = True # For big screens, make it more readable.
++ time_len = 9
++ else:
++ use_hours = False
++ time_len = 7
+ ui_size = tl.add(' | %5sB' % total_size)
-+ ui_time = tl.add(' %9s' % total_time)
++ ui_time = tl.add(' %*s' % (time_len, format_time(et, use_hours)))
+ ui_end, not_done = _term_add_end(tl, meter.size, size)
+ out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text,
+ ui_size, ui_time, ui_end)
@@ -2035,7 +2159,14 @@ index dd07c6a..ad57dbc 100644
def _do_failure_meter(self, meter, message, now):
self._lock.acquire()
-@@ -536,15 +607,6 @@ class TextMultiFileMeter(MultiFileMeter):
+ try:
+ format = "%-30.30s %6.6s %s"
+- fn = meter.basename
++ fn = meter.text or meter.basename
+ if type(message) in (type(''), type(u'')):
+ message = message.splitlines()
+ if not message: message = ['']
+@@ -536,15 +628,6 @@ class TextMultiFileMeter(MultiFileMeter):
pass
finally:
self._lock.release()
@@ -2051,7 +2182,7 @@ index dd07c6a..ad57dbc 100644
######################################################################
# support classes and functions
-@@ -658,6 +720,8 @@ def format_time(seconds, use_hours=0):
+@@ -658,6 +741,8 @@ def format_time(seconds, use_hours=0):
if seconds is None or seconds < 0:
if use_hours: return '--:--:--'
else: return '--:--'
@@ -2060,3 +2191,81 @@ index dd07c6a..ad57dbc 100644
else:
seconds = int(seconds)
minutes = seconds / 60
+@@ -722,9 +807,77 @@ def _tst(fn, cur, tot, beg, size, *args):
+ time.sleep(delay)
+ tm.end(size)
+
++def _mtst(datas, *args):
++ print '-' * 79
++ tm = TextMultiFileMeter(threaded=False)
++
++ dl_sizes = {}
++
++ num = 0
++ total_size = 0
++ dl_total_size = 0
++ for data in datas:
++ dl_size = None
++ if len(data) == 2:
++ fn, size = data
++ dl_size = size
++ if len(data) == 3:
++ fn, size, dl_size = data
++ nm = tm.newMeter()
++ nm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size,
++ text=fn)
++ num += 1
++ assert dl_size is not None
++ dl_total_size += dl_size
++ dl_sizes[nm] = dl_size
++ if size is None or total_size is None:
++ total_size = None
++ else:
++ total_size += size
++ tm.start(num, total_size)
++
++ num = 0
++ off = 0
++ for (inc, delay) in args:
++ off += 1
++ while num < ((dl_total_size * off) / len(args)):
++ num += inc
++ for nm in tm.meters[:]:
++ if dl_sizes[nm] <= num:
++ nm.end(dl_sizes[nm])
++ tm.removeMeter(nm)
++ else:
++ nm.update(num)
++ time.sleep(delay)
++ assert not tm.meters
++
+ if __name__ == "__main__":
+ # (1/2): subversion-1.4.4-7.x86_64.rpm 2.4 MB / 85 kB/s 00:28
+ # (2/2): mercurial-0.9.5-6.fc8.x86_64.rpm 924 kB / 106 kB/s 00:08
++ if len(sys.argv) >= 2 and sys.argv[1] == 'multi':
++ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
++ ("s-1.0.1-1.fc8.i386.rpm", 5000),
++ ("m-1.0.1-2.fc8.i386.rpm", 10000)),
++ (100, 0.33), (500, 0.25), (1000, 0.1))
++
++ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
++ ("s-1.0.1-1.fc8.i386.rpm", 5000),
++ ("m-1.0.1-2.fc8.i386.rpm", None, 10000)),
++ (100, 0.33), (500, 0.25), (1000, 0.1))
++
++ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
++ ("s-1.0.1-1.fc8.i386.rpm", 2500000),
++ ("m-1.0.1-2.fc8.i386.rpm", 10000)),
++ (10, 0.2), (50, 0.1), (1000, 0.1))
++
++ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
++ ("s-1.0.1-1.fc8.i386.rpm", None, 2500000),
++ ("m-1.0.1-2.fc8.i386.rpm", None, 10000)),
++ (10, 0.2), (50, 0.1), (1000, 0.1))
++ # (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
++ # (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
++ sys.exit(0)
++
+ if len(sys.argv) >= 2 and sys.argv[1] == 'total':
+ text_meter_total_size(1000 + 10000 + 10000 + 1000000 + 1000000 +
+ 1000000 + 10000 + 10000 + 10000 + 1000000)
More information about the scm-commits
mailing list