[PATCH] DNFPayload: display the download progress on the hub.

[Ales Kozumplik]

From: Ales Kozumplik

---
 anaconda.spec.in                   |  2 +-
 pyanaconda/packaging/dnfpayload.py | 62 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/anaconda.spec.in b/anaconda.spec.in
index 7215e32..b648831 100755
--- a/anaconda.spec.in
+++ b/anaconda.spec.in
@@ -23,7 +23,7 @@ Source0: %{name}-%{version}.tar.bz2
 %define intltoolver 0.31.2-3
 %define pykickstartver 1.99.46
 %define yumver 3.4.3-91
-%define dnfver 0.4.14
+%define dnfver 0.4.15
 %define partedver 1.8.1
 %define pypartedver 2.5-2
 %define pythonpyblockver 0.45
diff --git a/pyanaconda/packaging/dnfpayload.py b/pyanaconda/packaging/dnfpayload.py
index ecf7bb1..b2382de 100644
--- a/pyanaconda/packaging/dnfpayload.py
+++ b/pyanaconda/packaging/dnfpayload.py
@@ -25,6 +25,7 @@ from pyanaconda.flags import flags
 from pyanaconda.i18n import _
 from pyanaconda.progress import progressQ

+import collections
 import itertools
 import logging
 import multiprocessing
@@ -41,7 +42,7 @@ try:
     import dnf
     import dnf.exceptions
     import dnf.repo
-    import dnf.output
+    import dnf.callback
     import rpm
 except ImportError as e:
     log.error("dnfpayload: component import failed: %s", e)
@@ -102,7 +103,7 @@ def _pick_mpoint(df, requested):
     return sorted(sufficients.iteritems(), key=operator.itemgetter(1),
                   reverse=True)[0][0]

-class PayloadRPMDisplay(dnf.output.LoggingTransactionDisplay):
+class PayloadRPMDisplay(dnf.callback.LoggingTransactionDisplay):
     def __init__(self, queue):
         super(PayloadRPMDisplay, self).__init__()
         self._queue = queue
@@ -123,6 +124,53 @@ class PayloadRPMDisplay(dnf.output.LoggingTransactionDisplay):
         elif action == self.TRANS_POST:
             self._queue.put(('post', None))

+class DownloadProgress(dnf.callback.DownloadProgress):
+    def __init__(self):
+        self.downloads = collections.defaultdict(int)
+        self.last_time = time.time()
+        self.total_files = 0
+        self.total_size = Size(0)
+
+    def _paced(fn):
+        """Execute `fn` no more often then every 2 seconds."""
+        def paced_fn(self, *args):
+            now = time.time()
+            if now - self.last_time < 2:
+                return
+            self.last_time = now
+            return fn(self, *args)
+        return paced_fn
+
+    @_paced
+    def _update(self):
+        msg = _('Downloading %(total_files)s RPMs, '
+                '%(downloaded)s / %(total_size)s (%(percent)d%%) done.')
+        downloaded = Size(sum(self.downloads.values()))
+        vals = {
+            'downloaded'  : downloaded,
+            'percent'     : int(100 * downloaded/self.total_size),
+            'total_files' : self.total_files,
+            'total_size'  : self.total_size
+        }
+        progressQ.send_message(msg % vals)
+
+    def end(self, payload, status, err_msg):
+        nevra = str(payload)
+        if status is dnf.callback.STATUS_OK:
+            self.downloads[nevra] = payload.download_size
+            self._update()
+            return
+        log.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
+
+    def progress(self, payload, done):
+        nevra = str(payload)
+        self.downloads[nevra] = done
+        self._update()
+
+    def start(self, total_files, total_size):
+        self.total_files = total_files
+        self.total_size = Size(total_size)
+
 def do_transaction(base, queue):
     try:
         display = PayloadRPMDisplay(queue)
@@ -419,7 +467,15 @@ class DNFPayload(packaging.PackagePayload):
         pkgs_to_download = self._base.transaction.install_set
         log.info('Downloading pacakges.')
         progressQ.send_message(_('Downloading packages'))
-        self._base.download_packages(pkgs_to_download)
+        progress = DownloadProgress()
+        try:
+            self._base.download_packages(pkgs_to_download, progress)
+        except dnf.exception.DownloadError as e:
+            msg = 'Failed to download the following packages: %s' % str(e)
+            exc = packaging.PayloadInstallError(msg)
+            if errors.errorHandler.cb(exc) == errors.ERROR_RAISE:
+                _failure_limbo()
+
         log.info('Downloading packages finished.')

pre_msg = _("Preparing transaction from installation source")

[David Shea]

On 02/24/2014 09:55 AM, Ales Kozumplik wrote:
> From: Ales Kozumplik
>
> anaconda.spec.in                   |  2 +-
>   pyanaconda/packaging/dnfpayload.py | 62 ++++++++++++++++++++++++++++++++++++--
>   2 files changed, 60 insertions(+), 4 deletions(-)
>
> diff --git a/anaconda.spec.in b/anaconda.spec.in
> index 7215e32..b648831 100755
> --- a/anaconda.spec.in
> +++ b/anaconda.spec.in
> @@ -23,7 +23,7 @@ Source0: %{name}-%{version}.tar.bz2
>   %define intltoolver 0.31.2-3
>   %define pykickstartver 1.99.46
>   %define yumver 3.4.3-91
> -%define dnfver 0.4.14
> +%define dnfver 0.4.15
>   %define partedver 1.8.1
>   %define pypartedver 2.5-2
>   %define pythonpyblockver 0.45
> diff --git a/pyanaconda/packaging/dnfpayload.py b/pyanaconda/packaging/dnfpayload.py
> index ecf7bb1..b2382de 100644
> --- a/pyanaconda/packaging/dnfpayload.py
> +++ b/pyanaconda/packaging/dnfpayload.py
> @@ -25,6 +25,7 @@ from pyanaconda.flags import flags
>   from pyanaconda.i18n import _
>   from pyanaconda.progress import progressQ
>
> +import collections
>   import itertools
>   import logging
>   import multiprocessing
> @@ -41,7 +42,7 @@ try:
>       import dnf
>       import dnf.exceptions
>       import dnf.repo
> import dnf.output
> import dnf.callback
>  import rpm
> except ImportError as e:
>  log.error("dnfpayload: component import failed: %s", e)
>
> @@ -102,7 +103,7 @@ def _pick_mpoint(df, requested):
>       return sorted(sufficients.iteritems(), key=operator.itemgetter(1),
>                     reverse=True)[0][0]
>
> -class PayloadRPMDisplay(dnf.output.LoggingTransactionDisplay):
> +class PayloadRPMDisplay(dnf.callback.LoggingTransactionDisplay):
>       def __init__(self, queue):
>           super(PayloadRPMDisplay, self).__init__()
>           self._queue = queue
> @@ -123,6 +124,53 @@ class PayloadRPMDisplay(dnf.output.LoggingTransactionDisplay):
>           elif action == self.TRANS_POST:
>               self._queue.put(('post', None))
>
> +class DownloadProgress(dnf.callback.DownloadProgress):
> def __init__(self):
>    self.downloads = collections.defaultdict(int)
>    self.last_time = time.time()
>    self.total_files = 0
>    self.total_size = Size(0)
> def _paced(fn):
>    """Execute `fn` no more often then every 2 seconds."""
>    def paced_fn(self, *args):
>        now = time.time()
>        if now - self.last_time < 2:
>            return
>        self.last_time = now
>        return fn(self, *args)
>    return paced_fn
> @_paced
> def _update(self):
>    msg = _('Downloading %(total_files)s RPMs, '
>            '%(downloaded)s / %(total_size)s (%(percent)d%%) done.')
>    downloaded = Size(sum(self.downloads.values()))
>    vals = {
>        'downloaded'  : downloaded,
>        'percent'     : int(100 * downloaded/self.total_size),
>        'total_files' : self.total_files,
>        'total_size'  : self.total_size
>    }
>    progressQ.send_message(msg % vals)
> def end(self, payload, status, err_msg):
>    nevra = str(payload)
>    if status is dnf.callback.STATUS_OK:
>        self.downloads[nevra] = payload.download_size
>        self._update()
>        return
>    log.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
> def progress(self, payload, done):
>    nevra = str(payload)
>    self.downloads[nevra] = done
>    self._update()
> def start(self, total_files, total_size):
>    self.total_files = total_files
>    self.total_size = Size(total_size)
> def do_transaction(base, queue):
>     try:
>         display = PayloadRPMDisplay(queue)
>
> @@ -419,7 +467,15 @@ class DNFPayload(packaging.PackagePayload):
>           pkgs_to_download = self._base.transaction.install_set
>           log.info('Downloading pacakges.')
>           progressQ.send_message(_('Downloading packages'))
>    self._base.download_packages(pkgs_to_download)
>    progress = DownloadProgress()
>    try:
>        self._base.download_packages(pkgs_to_download, progress)
>    except dnf.exception.DownloadError as e:
>        msg = 'Failed to download the following packages: %s' % str(e)
>        exc = packaging.PayloadInstallError(msg)
>        if errors.errorHandler.cb(exc) == errors.ERROR_RAISE:
>            _failure_limbo()
>     log.info('Downloading packages finished.')
>
>     pre_msg = _("Preparing transaction from installation source")

pylint reports the following issues:
************* Module pyanaconda/packaging/dnfpayload.py
E1101:128,23: DownloadProgress: Module 'dnf.callback' has no 
'DownloadProgress' member
E0213:135,4: DownloadProgress._paced: Method should have "self" as first 
argument
E1102:142,19: DownloadProgress._paced.paced_fn: fn is not callable
E1101:475,15: DNFPayload.install: Module 'dnf' has no 'exception' member

[Ales Kozumplik]

On 02/24/2014 04:14 PM, David Shea wrote:
> pylint reports the following issues:
> ************* Module pyanaconda/packaging/dnfpayload.py

(rehashed)
> E1101:128,23: DownloadProgress: Module 'dnf.callback' has no
> 'DownloadProgress' member
> E1101:475,15: DNFPayload.install: Module 'dnf' has no 'exception' member

These two will go away with the (ongoing) dnf-0.4.15 build.
> E0213:135,4: DownloadProgress._paced: Method should have "self" as first
> argument
> E1102:142,19: DownloadProgress._paced.paced_fn: fn is not callable

I'll turn the function into staticmethod or something.

Ales

[Ales Kozumplik]

From: Ales Kozumplik

---
 anaconda.spec.in                   |  2 +-
 pyanaconda/packaging/dnfpayload.py | 65 +++++++++++++++++++++++++++++++++++---
 2 files changed, 62 insertions(+), 5 deletions(-)

diff --git a/anaconda.spec.in b/anaconda.spec.in
index 7215e32..b648831 100755
--- a/anaconda.spec.in
+++ b/anaconda.spec.in
@@ -23,7 +23,7 @@ Source0: %{name}-%{version}.tar.bz2
 %define intltoolver 0.31.2-3
 %define pykickstartver 1.99.46
 %define yumver 3.4.3-91
-%define dnfver 0.4.14
+%define dnfver 0.4.15
 %define partedver 1.8.1
 %define pypartedver 2.5-2
 %define pythonpyblockver 0.45
diff --git a/pyanaconda/packaging/dnfpayload.py b/pyanaconda/packaging/dnfpayload.py
index ecf7bb1..f11d45c 100644
--- a/pyanaconda/packaging/dnfpayload.py
+++ b/pyanaconda/packaging/dnfpayload.py
@@ -25,9 +25,11 @@ from pyanaconda.flags import flags
 from pyanaconda.i18n import _
 from pyanaconda.progress import progressQ

+import collections
 import itertools
 import logging
 import multiprocessing
+import operator
 import pyanaconda.constants as constants
 import pyanaconda.errors as errors
 import pyanaconda.iutil
@@ -41,7 +43,7 @@ try:
     import dnf
     import dnf.exceptions
     import dnf.repo
-    import dnf.output
+    import dnf.callback
     import rpm
 except ImportError as e:
     log.error("dnfpayload: component import failed: %s", e)
@@ -83,6 +85,16 @@ def _df_map():
         structured[key] = Size(bytes=int(val)*1024)
     return structured

+def _paced(fn):
+    """Execute `fn` no more often then every 2 seconds."""
+    def paced_fn(self, *args):
+        now = time.time()
+        if now - self.last_time < 2:
+            return
+        self.last_time = now
+        return fn(self, *args)
+    return paced_fn
+
 def _pick_mpoint(df, requested):
     def reasonable_mpoint(mpoint):
         return mpoint in DOWNLOAD_MPOINTS
@@ -91,7 +103,7 @@ def _pick_mpoint(df, requested):
     requested = requested + Size(en_spec="150 MB")
     sufficients = {key : val for (key,val) in df.items() if val > requested
                    and reasonable_mpoint(key)}
-    log.info('Sufficient mountpoints found: %s' % sufficients)
+    log.info('Sufficient mountpoints found: %s', sufficients)

# prefer /tmp:
     if '/tmp' in sufficients:
@@ -102,7 +114,7 @@ def _pick_mpoint(df, requested):
     return sorted(sufficients.iteritems(), key=operator.itemgetter(1),
                   reverse=True)[0][0]

-class PayloadRPMDisplay(dnf.output.LoggingTransactionDisplay):
+class PayloadRPMDisplay(dnf.callback.LoggingTransactionDisplay):
     def __init__(self, queue):
         super(PayloadRPMDisplay, self).__init__()
         self._queue = queue
@@ -123,6 +135,43 @@ class PayloadRPMDisplay(dnf.output.LoggingTransactionDisplay):
         elif action == self.TRANS_POST:
             self._queue.put(('post', None))

+class DownloadProgress(dnf.callback.DownloadProgress):
+    def __init__(self):
+        self.downloads = collections.defaultdict(int)
+        self.last_time = time.time()
+        self.total_files = 0
+        self.total_size = Size(0)
+
+    @_paced
+    def _update(self):
+        msg = _('Downloading %(total_files)s RPMs, '
+                '%(downloaded)s / %(total_size)s (%(percent)d%%) done.')
+        downloaded = Size(sum(self.downloads.values()))
+        vals = {
+            'downloaded'  : downloaded,
+            'percent'     : int(100 * downloaded/self.total_size),
+            'total_files' : self.total_files,
+            'total_size'  : self.total_size
+        }
+        progressQ.send_message(msg % vals)
+
+    def end(self, payload, status, err_msg):
+        nevra = str(payload)
+        if status is dnf.callback.STATUS_OK:
+            self.downloads[nevra] = payload.download_size
+            self._update()
+            return
+        log.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
+
+    def progress(self, payload, done):
+        nevra = str(payload)
+        self.downloads[nevra] = done
+        self._update()
+
+    def start(self, total_files, total_size):
+        self.total_files = total_files
+        self.total_size = Size(total_size)
+
 def do_transaction(base, queue):
     try:
         display = PayloadRPMDisplay(queue)
@@ -419,7 +468,15 @@ class DNFPayload(packaging.PackagePayload):
         pkgs_to_download = self._base.transaction.install_set
         log.info('Downloading pacakges.')
         progressQ.send_message(_('Downloading packages'))
-        self._base.download_packages(pkgs_to_download)
+        progress = DownloadProgress()
+        try:
+            self._base.download_packages(pkgs_to_download, progress)
+        except dnf.exceptions.DownloadError as e:
+            msg = 'Failed to download the following packages: %s' % str(e)
+            exc = packaging.PayloadInstallError(msg)
+            if errors.errorHandler.cb(exc) == errors.ERROR_RAISE:
+                _failure_limbo()
+
         log.info('Downloading packages finished.')

pre_msg = _("Preparing transaction from installation source")

[Vratislav Podzimek]

On Tue, 2014-02-25 at 08:55 +0100, Ales Kozumplik wrote:
> From: Ales Kozumplik
>
> anaconda.spec.in                   |  2 +-
>  pyanaconda/packaging/dnfpayload.py | 65 +++++++++++++++++++++++++++++++++++---
>  2 files changed, 62 insertions(+), 5 deletions(-)
>
> diff --git a/anaconda.spec.in b/anaconda.spec.in
> index 7215e32..b648831 100755
> --- a/anaconda.spec.in
> +++ b/anaconda.spec.in
> @@ -23,7 +23,7 @@ Source0: %{name}-%{version}.tar.bz2
>  %define intltoolver 0.31.2-3
>  %define pykickstartver 1.99.46
>  %define yumver 3.4.3-91
> -%define dnfver 0.4.14
> +%define dnfver 0.4.15
>  %define partedver 1.8.1
>  %define pypartedver 2.5-2
>  %define pythonpyblockver 0.45
> diff --git a/pyanaconda/packaging/dnfpayload.py b/pyanaconda/packaging/dnfpayload.py
> index ecf7bb1..f11d45c 100644
> --- a/pyanaconda/packaging/dnfpayload.py
> +++ b/pyanaconda/packaging/dnfpayload.py
> @@ -25,9 +25,11 @@ from pyanaconda.flags import flags
>  from pyanaconda.i18n import _
>  from pyanaconda.progress import progressQ
>
> +import collections
>  import itertools
>  import logging
>  import multiprocessing
> +import operator
>  import pyanaconda.constants as constants
>  import pyanaconda.errors as errors
>  import pyanaconda.iutil
> @@ -41,7 +43,7 @@ try:
>      import dnf
>      import dnf.exceptions
>      import dnf.repo
> import dnf.output
> import dnf.callback
> import rpm
>
> except ImportError as e:
>      log.error("dnfpayload: component import failed: %s", e)
> @@ -83,6 +85,16 @@ def _df_map():
>          structured[key] = Size(bytes=int(val)*1024)
>      return structured
>
> +def _paced(fn):
> """Execute `fn` no more often then every 2 seconds."""
> def paced_fn(self, *args):
>    now = time.time()
>    if now - self.last_time < 2:
>        return
>    self.last_time = now
>    return fn(self, *args)
> return paced_fn
>
> def _pick_mpoint(df, requested):
>      def reasonable_mpoint(mpoint):
>          return mpoint in DOWNLOAD_MPOINTS
> @@ -91,7 +103,7 @@ def _pick_mpoint(df, requested):
>      requested = requested + Size(en_spec="150 MB")
>      sufficients = {key : val for (key,val) in df.items() if val > requested
>                     and reasonable_mpoint(key)}
> log.info('Sufficient mountpoints found: %s' % sufficients)
>
> log.info('Sufficient mountpoints found: %s', sufficients)
>
> # prefer /tmp:
> if '/tmp' in sufficients:
>
> @@ -102,7 +114,7 @@ def _pick_mpoint(df, requested):
>      return sorted(sufficients.iteritems(), key=operator.itemgetter(1),
>                    reverse=True)[0][0]
>
> -class PayloadRPMDisplay(dnf.output.LoggingTransactionDisplay):
> +class PayloadRPMDisplay(dnf.callback.LoggingTransactionDisplay):
>      def __init__(self, queue):
>          super(PayloadRPMDisplay, self).__init__()
>          self._queue = queue
> @@ -123,6 +135,43 @@ class PayloadRPMDisplay(dnf.output.LoggingTransactionDisplay):
>          elif action == self.TRANS_POST:
>              self._queue.put(('post', None))
>
> +class DownloadProgress(dnf.callback.DownloadProgress):
> def __init__(self):
>    self.downloads = collections.defaultdict(int)
>    self.last_time = time.time()
>    self.total_files = 0
>    self.total_size = Size(0)
> @_paced
> def _update(self):
>    msg = _('Downloading %(total_files)s RPMs, '
>            '%(downloaded)s / %(total_size)s (%(percent)d%%) done.')
>    downloaded = Size(sum(self.downloads.values()))
>    vals = {
>        'downloaded'  : downloaded,
>        'percent'     : int(100 * downloaded/self.total_size),
>        'total_files' : self.total_files,
>        'total_size'  : self.total_size
>    }
>    progressQ.send_message(msg % vals)
> def end(self, payload, status, err_msg):
>    nevra = str(payload)
>    if status is dnf.callback.STATUS_OK:
>        self.downloads[nevra] = payload.download_size
>        self._update()
>        return
>    log.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
> def progress(self, payload, done):
>    nevra = str(payload)
>    self.downloads[nevra] = done
>    self._update()
> def start(self, total_files, total_size):
>    self.total_files = total_files
>    self.total_size = Size(total_size)
>
> def do_transaction(base, queue):
>      try:
>          display = PayloadRPMDisplay(queue)
> @@ -419,7 +468,15 @@ class DNFPayload(packaging.PackagePayload):
>          pkgs_to_download = self._base.transaction.install_set
>          log.info('Downloading pacakges.')
>          progressQ.send_message(_('Downloading packages'))
>    self._base.download_packages(pkgs_to_download)
>    progress = DownloadProgress()
>    try:
>        self._base.download_packages(pkgs_to_download, progress)
>    except dnf.exceptions.DownloadError as e:
>        msg = 'Failed to download the following packages: %s' % str(e)
>        exc = packaging.PayloadInstallError(msg)
>        if errors.errorHandler.cb(exc) == errors.ERROR_RAISE:
>            _failure_limbo()
>    log.info('Downloading packages finished.')
>
>    pre_msg = _("Preparing transaction from installation source")

Looks good to me.

[Vratislav Podzimek]

On Tue, 2014-02-25 at 08:55 +0100, Ales Kozumplik wrote:
> From: Ales Kozumplik
>
> anaconda.spec.in                   |  2 +-
>  pyanaconda/packaging/dnfpayload.py | 65 +++++++++++++++++++++++++++++++++++---
>  2 files changed, 62 insertions(+), 5 deletions(-)
>
> diff --git a/anaconda.spec.in b/anaconda.spec.in
> index 7215e32..b648831 100755
> --- a/anaconda.spec.in
> +++ b/anaconda.spec.in
> @@ -23,7 +23,7 @@ Source0: %{name}-%{version}.tar.bz2
>  %define intltoolver 0.31.2-3
>  %define pykickstartver 1.99.46
>  %define yumver 3.4.3-91
> -%define dnfver 0.4.14
> +%define dnfver 0.4.15
>  %define partedver 1.8.1
>  %define pypartedver 2.5-2
>  %define pythonpyblockver 0.45
> diff --git a/pyanaconda/packaging/dnfpayload.py b/pyanaconda/packaging/dnfpayload.py
> index ecf7bb1..f11d45c 100644
> --- a/pyanaconda/packaging/dnfpayload.py
> +++ b/pyanaconda/packaging/dnfpayload.py
> @@ -25,9 +25,11 @@ from pyanaconda.flags import flags
>  from pyanaconda.i18n import _
>  from pyanaconda.progress import progressQ
>
> +import collections
>  import itertools
>  import logging
>  import multiprocessing
> +import operator
>  import pyanaconda.constants as constants
>  import pyanaconda.errors as errors
>  import pyanaconda.iutil
> @@ -41,7 +43,7 @@ try:
>      import dnf
>      import dnf.exceptions
>      import dnf.repo
> import dnf.output
> import dnf.callback
> import rpm
>
> except ImportError as e:
>      log.error("dnfpayload: component import failed: %s", e)
> @@ -83,6 +85,16 @@ def _df_map():
>          structured[key] = Size(bytes=int(val)*1024)
>      return structured
>
> +def _paced(fn):
> """Execute `fn` no more often then every 2 seconds."""

^than