From: "Brian C. Lane" bcl@redhat.com
Add a function to read the output from an executable and act on it in real time. It uses a Thread and a Queue to process the command's stdout without blocking and yield's the lines back to the caller so that it can be used like this:
for line in execReadlines("command", [args]): print line --- pyanaconda/iutil.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 4 deletions(-)
diff --git a/pyanaconda/iutil.py b/pyanaconda/iutil.py index 1f75ad9..d2a02e0 100644 --- a/pyanaconda/iutil.py +++ b/pyanaconda/iutil.py @@ -21,15 +21,15 @@ #
import glob -import os, string, stat, sys -import shutil -import signal +import os +import stat import os.path import errno import subprocess -import threading import re import unicodedata +from threading import Thread +from Queue import Queue, Empty
from flags import flags from constants import * @@ -134,6 +134,68 @@ def execWithCapture(command, argv, stdin=None, stderr=None, root='/', argv = [command] + argv return _run_program(argv, stdin=stdin, root=root)[1]
+def execReadlines(command, argv, stdin=None, root='/', env_prune=[]): + """ Execute an external command and return the line output of the command + in real-time. + + @param command The command to run + @param argv The argument list + @param stdin The file object to read stdin from. + @param stdout Optional file object to redirect stdout and stderr to. + @param stderr not used + @param root The directory to chroot to before running command. + @param env_prune environment variable to remove before execution + + Output from the file is not logged to program.log + This returns a generator with the lines from the command until it has finished + """ + if env_prune is None: + env_prune = [] + + # Return the lines from stdout via a Queue + def queue_lines(out, queue): + for line in iter(out.readline, b''): + queue.put(line.strip()) + out.close() + + def chroot(): + if root and root != '/': + os.chroot(root) + + argv = [command] + argv + with program_log_lock: + program_log.info("Running... %s" % " ".join(argv)) + + env = augmentEnv() + for var in env_prune: + env.pop(var, None) + try: + proc = subprocess.Popen(argv, + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + preexec_fn=chroot, cwd=root, env=env) + except OSError as e: + program_log.error("Error running %s: %s" % (argv[0], e.strerror)) + raise + + q = Queue() + t = Thread(target=queue_lines, args=(proc.stdout, q)) + t.daemon = True # thread dies with the program + t.start() + + while True: + try: + line = q.get(timeout=.1) + yield line + q.task_done() + except Empty: + if proc.poll() is not None: + break + q.join() + + ## Run a shell. def execConsole(): try:
From: "Brian C. Lane" bcl@redhat.com
Add a helper for installing packages via yum. This script reads a yum transaction file and a yum config file and executes the installation. It communicates with the called via stdout and a few string prefixes. This allows us to run the install in a totally different process. --- scripts/Makefile.am | 2 +- scripts/anaconda-yum | 340 +++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/makeupdates | 2 + 3 files changed, 343 insertions(+), 1 deletion(-) create mode 100755 scripts/anaconda-yum
diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 3813ee6..8bd6626 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -22,7 +22,7 @@ dist_scripts_SCRIPTS = upd-updates run-anaconda dist_scripts_DATA = pyrc.py dist_noinst_SCRIPTS = upd-kernel makeupdates
-dist_bin_SCRIPTS = analog anaconda-cleanup instperf +dist_bin_SCRIPTS = analog anaconda-cleanup instperf anaconda-yum
stage2scriptsdir = $(datadir)/$(PACKAGE_NAME) dist_stage2scripts_SCRIPTS = restart-anaconda diff --git a/scripts/anaconda-yum b/scripts/anaconda-yum new file mode 100755 index 0000000..f3b2aea --- /dev/null +++ b/scripts/anaconda-yum @@ -0,0 +1,340 @@ +#!/usr/bin/python +# +# Copyright (C) 2013 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# +# Red Hat Author(s): Brian C. Lane bcl@redhat.com +# +import logging +import os +import sys +import argparse +import rpm +import rpmUtils +import yum +from urlgrabber.grabber import URLGrabError + +YUM_PLUGINS = ["blacklist", "whiteout", "fastestmirror", "langpacks"] + +def setup_parser(): + """ Setup argparse with supported arguments + + :rtype: ArgumentParser + """ + parser = argparse.ArgumentParser(description="anaconda-yum") + parser.add_argument("-a", "--arch", required=True, help="Arch to install") + parser.add_argument("-c", "--config", help="Path to yum config file", default="/tmp/anaconda-yum.conf") + parser.add_argument("-t", "--tsfile", required=True, help="Path to yum transaction file") + parser.add_argument("-l", "--rpmlog", help="Path to rpm script logfile", default="/tmp/rpm-script.log") + parser.add_argument("-r", "--release", required=True, help="Release version") + parser.add_argument("-i", "--installroot", help="Path to top directory of installroot", default="/mnt/sysimage") + parser.add_argument("-T", "--test", action="store_true", help="Test transaction, don't actually install") + parser.add_argument("-d", "--debug", action="store_true", help="Extra debugging output") + + return parser + + +def run_yum_transaction(release, arch, yum_conf, install_root, ts_file, script_log, + testing=False, debug=False): + """ Execute a yum transaction loaded from a transaction file + + :param release: The release version to use + :type release: string + :param arch: The arch to install + :type arch: string + :param yum_conf: Path to yum config file to use + :type yum_conf: string + :param install_root: Path to install root + :type install_root: string + :param ts_file: Path to yum transaction file to load and execute + :type ts_file: string + :param script_log: Path to file to store rpm script logs in + :type script_log: string + :param testing: True sets RPMTRANS_FLAG_TEST (default is false) + :type testing: bool + :returns: Nothing + + This is used to run the yum transaction in a separate process, preventing + problems with threads and rpm chrooting during the install. + """ + from yum.Errors import PackageSackError, RepoError, YumBaseError, YumRPMTransError + + # remove some environmental variables that can cause problems with package scripts + env_remove = ('DISPLAY', 'DBUS_SESSION_BUS_ADDRESS') + [os.environ.pop(k) for k in env_remove if k in os.environ] + + try: + # Setup the basics, point to the config file and install_root + yb = yum.YumBase() + yb.use_txmbr_in_callback = True + + # Set some configuration parameters that don't get set through a config + # file. yum will know what to do with these. + # Enable all types of yum plugins. We're somewhat careful about what + # plugins we put in the environment. + yb.preconf.plugin_types = yum.plugins.ALL_TYPES + yb.preconf.enabled_plugins = YUM_PLUGINS + yb.preconf.fn = yum_conf + yb.preconf.root = install_root + yb.preconf.releasever = release + + if debug: + yb.preconf.debuglevel = 10 + yb.preconf.errorlevel = 10 + yb.preconf.rpmverbosity = "debug" + + # Setup yum cache dir outside the installroot + if yb.conf.cachedir.startswith(yb.conf.installroot): + root = yb.conf.installroot + yb.conf.cachedir = yb.conf.cachedir[len(root):] + + # Load the transaction file and execute it + yb.load_ts(ts_file) + yb.initActionTs() + + if rpmUtils and rpmUtils.arch.isMultiLibArch(): + yb.ts.ts.setColor(3) + + print("DEBUG: populate transaction set") + try: + # uses dsCallback.transactionPopulation + yb.populateTs(keepold=0) + except RepoError as e: + print("ERROR: error populating transaction: %s" % e) + print("QUIT:") + return + + print("DEBUG: check transaction set") + yb.ts.check() + print("DEBUG: order transaction set") + yb.ts.order() + yb.ts.clean() + + # Write scriptlet output to a file to be logged later + logfile = open(script_log, "w") + yb.ts.ts.scriptFd = logfile.fileno() + rpm.setLogFile(logfile) + + # create the install callback + rpmcb = RPMCallback(yb, arch, logfile, debug) + + if testing: + yb.ts.setFlags(rpm.RPMTRANS_FLAG_TEST) + + print("INFO: running transaction") + try: + yb.runTransaction(cb=rpmcb) + except PackageSackError as e: + print("ERROR: PackageSackError: %s" % e) + except YumRPMTransError as e: + print("ERROR: YumRPMTransError: %s" % e) + except YumBaseError as e: + print("ERROR: YumBaseError: %s" % e) + for error in e.errors: + print("ERROR: %s" % error) + else: + print("INFO: transaction complete") + finally: + yb.ts.close() + logfile.close() + except Exception as e: + print("ERROR: transaction error: %s" % e) + finally: + print("QUIT:") + + +class RPMCallback(object): + """ Custom RPMTransaction Callback class. You need one of these to actually + make a transaction work. + If you subclass it, make sure you preserve the behavior of + inst_open_file and inst_close_file, or nothing will actually happen. + """ + callback_map = dict((rpm.__dict__[k], k[12:].lower()) + for k in rpm.__dict__ + if k.startswith('RPMCALLBACK_')) + + def callback(self, what, amount, total, key, data): + """ Handle calling appropriate method, if it exists. + """ + if what not in self.callback_map: + print("DEBUG: Ignoring unknown callback number %i", what) + return + name = self.callback_map[what] + func = getattr(self, name, None) + if callable(func): + return func(amount, total, key, data) + + def __init__(self, yb, arch, log, debug=False): + """ :param yb: YumBase object + :type yb: YumBase + :param log: script logfile + :type log: string + :param statusQ: status communication back to other process + :type statusQ: Queue + """ + self.yb = yb # yum.YumBase + self.base_arch = arch + self.install_log = log # logfile for yum script logs + self.debug = debug + + self.package_file = None # file instance (package file management) + self.total_actions = 0 + self.completed_actions = None # will be set to 0 when starting tx + + def _get_txmbr(self, key): + """ Return a (name, TransactionMember) tuple from cb key. """ + if hasattr(key, "po"): + # New-style callback, key is a TransactionMember + txmbr = key + name = key.name + else: + # cleanup/remove error + name = key + txmbr = None + + return (name, txmbr) + + def trans_start(self, amount, total, key, data): + """ Start of the install transaction + + Reset the actions counter and save the total to be completed. + """ + if amount == 6: + print("PROGRESS_PREP:") + self.total_actions = total + self.completed_actions = 0 + + def inst_open_file(self, amount, total, key, data): + """ Open a file for installation + + :returns: open file descriptor + :rtype: int + """ + txmbr = self._get_txmbr(key)[1] + if self.debug: + print("DEBUG: txmbr = %s" % txmbr) + + # If self.completed_actions is still None, that means this package + # is being opened to retrieve a %pretrans script. Don't log that + # we're installing the package unless trans_start() has been called. + if self.completed_actions is not None: + self.completed_actions += 1 + msg_format = "%s (%d/%d)" + progress_package = txmbr.name + if txmbr.arch not in ["noarch", self.base_arch]: + progress_package = "%s.%s" % (txmbr.name, txmbr.arch) + + progress_msg = msg_format % (progress_package, + self.completed_actions, + self.total_actions) + log_msg = msg_format % (txmbr.po, + self.completed_actions, + self.total_actions) + self.install_log.write(log_msg+"\n") + print("PROGRESS_INSTALL: %s" % progress_msg) + + try: + repo = self.yb.repos.getRepo(txmbr.po.repoid) + except Exception as e: + print("ERROR: getRepo failed: %s" % e) + raise Exception("rpmcallback getRepo failed") + + self.package_file = None + while self.package_file is None: + try: + # checkfunc gets passed to yum's use of URLGrabber which + # then calls it with the file being fetched. verifyPkg + # makes sure the checksum matches the one in the metadata. + # + # From the URLGrab documents: + # checkfunc=(function, ('arg1', 2), {'kwarg': 3}) + # results in a callback like: + # function(obj, 'arg1', 2, kwarg=3) + # obj.filename = '/tmp/stuff' + # obj.url = 'http://foo.com/stuff' + checkfunc = (self.yb.verifyPkg, (txmbr.po, 1), {}) + if self.debug: + print("DEBUG: getPackage %s" % txmbr.name) + package_path = repo.getPackage(txmbr.po, checkfunc=checkfunc) + except URLGrabError as e: + print("ERROR: URLGrabError: %s" % e) + raise Exception("rpmcallback failed") + except (yum.Errors.NoMoreMirrorsRepoError, IOError) as e: + if os.path.exists(txmbr.po.localPkg()): + os.unlink(txmbr.po.localPkg()) + print("DEBUG: retrying download of %s" % txmbr.po) + continue + print("ERROR: getPackage error: %s" % e) + raise Exception("getPackage failed") + except yum.Errors.RepoError as e: + print("DEBUG: RepoError: %s" % e) + continue + + self.package_file = open(package_path) + + if self.debug: + print("DEBUG: opening package %s" % self.package_file.name) + return self.package_file.fileno() + + def inst_close_file(self, amount, total, key, data): + """ close and remove the file + + Update the count of installed packages + """ + package_path = self.package_file.name + self.package_file.close() + self.package_file = None + + if package_path.startswith(self.yb.conf.cachedir): + try: + os.unlink(package_path) + except OSError as e: + print("WARN: unable to remove file %s" % e.strerror) + + # rpm doesn't tell us when it's started post-trans stuff which can + # take a very long time. So when it closes the last package, just + # display the message. + if self.completed_actions == self.total_actions: + print("PROGRESS_POST:") + + def cpio_error(self, amount, total, key, data): + name = self._get_txmbr(key)[0] + print("ERROR: cpio error with package %s" % name) + raise Exception("cpio error") + + def unpack_error(self, amount, total, key, data): + name = self._get_txmbr(key)[0] + print("ERROR: unpack error with package %s" % name) + raise Exception("unpack error") + + def script_error(self, amount, total, key, data): + name = self._get_txmbr(key)[0] + # Script errors store whether or not they're fatal in "total". + if total: + print("ERROR: script error with package %s" % name) + raise Exception("script error") + + +if __name__ == "__main__": + parser = setup_parser() + args = parser.parse_args() + + # force output to be flushed + sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) + + run_yum_transaction(args.release, args.arch, args.config, args.installroot, + args.tsfile, args.rpmlog, args.test, args.debug) + diff --git a/scripts/makeupdates b/scripts/makeupdates index 1ffb166..a0d8366 100755 --- a/scripts/makeupdates +++ b/scripts/makeupdates @@ -171,6 +171,8 @@ def copyUpdatedFiles(tag, updates, cwd): install_to_dir(file, "usr/share/anaconda/ui") elif file.startswith("data/post-scripts/"): install_to_dir(file, "usr/share/anaconda/post-scripts") + elif file.endswith("anaconda-yum"): + install_to_dir(file, "usr/bin") elif file.find('/') != -1: fields = file.split('/') subdir = fields[0]
From: "Brian C. Lane" bcl@redhat.com
This moves the transaction into a helper script and uses the yum transaction save/load feature to transfer the data to it. This should prevent yum thread problems and rpm chroot from interfering with the main anaconda process. --- pyanaconda/packaging/yumpayload.py | 294 +++++++++---------------------------- 1 file changed, 72 insertions(+), 222 deletions(-)
diff --git a/pyanaconda/packaging/yumpayload.py b/pyanaconda/packaging/yumpayload.py index 4fc6177..502a38c 100644 --- a/pyanaconda/packaging/yumpayload.py +++ b/pyanaconda/packaging/yumpayload.py @@ -41,7 +41,7 @@ import os import shutil import sys import time -import tempfile +from pyanaconda.iutil import execReadlines
from . import *
@@ -92,6 +92,7 @@ import itertools
from pykickstart.constants import KS_MISSING_IGNORE
+YUM_PLUGINS = ["blacklist", "whiteout", "fastestmirror", "langpacks"] default_repos = [productName.lower(), "rawhide"]
import inspect @@ -217,21 +218,12 @@ class YumPayload(PackagePayload): # Enable all types of yum plugins. We're somewhat careful about what # plugins we put in the environment. self._yum.preconf.plugin_types = yum.plugins.ALL_TYPES - self._yum.preconf.enabled_plugins = ["blacklist", "whiteout", "fastestmirror", - "langpacks"] + self._yum.preconf.enabled_plugins = YUM_PLUGINS self._yum.preconf.fn = "/tmp/anaconda-yum.conf" self._yum.preconf.root = root # set this now to the best default we've got ; we'll update it if/when # we get a base repo set up self._yum.preconf.releasever = self._getReleaseVersion(None) - # Set the yum verbosity to 6, and update yum's internal logger - # objects to the debug level. This is a bit of a hack requiring - # internal knowledge of yum, that will hopefully go away in the - # future with API improvements. - self._yum.preconf.debuglevel = 6 - self._yum.preconf.errorlevel = 6 - self._yum.logger.setLevel(logging.DEBUG) - self._yum.verbose_logger.setLevel(logging.DEBUG)
self.txID = None
@@ -246,6 +238,8 @@ metadata_expire=never pluginpath=/usr/lib/yum-plugins,/tmp/updates/yum-plugins pluginconfpath=/etc/yum/pluginconf.d,/tmp/updates/pluginconf.d plugins=1 +debuglevel=3 +errorlevel=6 reposdir=%s """ % (_yum_cache_dir, self._repos_dir)
@@ -1460,82 +1454,81 @@ reposdir=%s return retval
def install(self): - """ Install the payload. """ - from yum.Errors import PackageSackError, RepoError, YumBaseError, YumRPMTransError + """ Install the payload.
- log.info("preparing transaction") - log.debug("initialize transaction set") + This writes out the yum transaction and then uses a Process thread + to execute it in a totally separate process. + + It monitors the status of the install and logs debug info, updates + the progress meter and cleans up when it is done. + """ + progress_map = { + "PROGRESS_PREP" : _("Preparing transaction from installation source"), + "PROGRESS_INSTALL" : _("Installing"), + "PROGRESS_POST" : _("Performing post-installation setup tasks") + } + + ts_file = ROOT_PATH+"/anaconda-yum.yumtx" with _yum_lock: - self._yum.initActionTs() + # Save the transaction, this will be loaded and executed by the new + # process. + self._yum.save_ts(ts_file)
- if rpmUtils and rpmUtils.arch.isMultiLibArch(): - self._yum.ts.ts.setColor(3) + # Try and clean up yum before the fork + self.release() + self.deleteYumTS() + self._yum.close()
- log.debug("populate transaction set") - try: - # uses dsCallback.transactionPopulation - self._yum.populateTs(keepold=0) - except RepoError as e: - log.error("error populating transaction: %s" % e) - exn = PayloadInstallError(str(e)) - if errorHandler.cb(exn) == ERROR_RAISE: - raise exn - - log.debug("check transaction set") - self._yum.ts.check() - log.debug("order transaction set") - self._yum.ts.order() - self._yum.ts.clean() - - # Write scriptlet output to a file to be logged later - script_log = tempfile.NamedTemporaryFile(delete=False) - self._yum.ts.ts.scriptFd = script_log.fileno() - rpm.setLogFile(script_log) - - # create the install callback - rpmcb = RPMCallback(self._yum, script_log, - upgrade=self.data.upgrade.upgrade) - - if flags.testing: - self._yum.ts.setFlags(rpm.RPMTRANS_FLAG_TEST) - - log.info("running transaction") - progress.send_step() - try: - self._yum.runTransaction(cb=rpmcb) - except PackageSackError as e: - log.error("error [1] running transaction: %s" % e) - exn = PayloadInstallError(str(e)) - if errorHandler.cb(exn) == ERROR_RAISE: - raise exn - except YumRPMTransError as e: - log.error("error [2] running transaction: %s" % e) - exn = PayloadInstallError(self._transactionErrors(e.errors)) - if errorHandler.cb(exn) == ERROR_RAISE: - progress.send_quit(1) - sys.exit(1) - except YumBaseError as e: - log.error("error [3] running transaction: %s" % e) - for error in e.errors: - log.error("%s" % error[0]) - exn = PayloadInstallError(str(e)) - if errorHandler.cb(exn) == ERROR_RAISE: - raise exn - else: - log.info("transaction complete") - progress.send_step() - finally: - self._yum.ts.close() - iutil.resetRpmDb() - script_log.close() - - # log the contents of the scriptlet logfile + script_log = "/tmp/rpm-script.log" + release = self._getReleaseVersion(None) + + args = ["--config", "/tmp/anaconda-yum.conf", + "--tsfile", ts_file, + "--rpmlog", script_log, + "--installroot", ROOT_PATH, + "--release", release, + "--arch", blivet.arch.getArch()] + + log.info("Running anaconda-yum to install packages") + # Watch output for progress, debug and error information + install_errors = [] + try: + for line in execReadlines("anaconda-yum", args): + if line.startswith("PROGRESS_"): + key, text = line.split(":", 2) + msg = progress_map[key] + text + progress.send_message(msg) + log.debug(msg) + elif line.startswith("DEBUG:"): + log.debug(line[6:]) + elif line.startswith("INFO:"): + log.info(line[5:]) + elif line.startswith("WARN:"): + log.warn(line[5:]) + elif line.startswith("ERROR:"): + log.error(line[6:]) + install_errors.append(line[6:]) + else: + log.debug(line) + except IOError as e: + log.error("Error running anaconda-yum: %s" % e) + exn = PayloadInstallError(str(e)) + if errorHandler.cb(exn) == ERROR_RAISE: + raise exn + finally: + # log the contents of the scriptlet logfile if any + if os.path.exists(script_log): log.info("==== start rpm scriptlet logs ====") - with open(script_log.name) as f: + with open(script_log) as f: for l in f: log.info(l) log.info("==== end rpm scriptlet logs ====") - os.unlink(script_log.name) + os.unlink(script_log) + + if install_errors: + exn = PayloadInstallError("\n".join(install_errors)) + if errorHandler.cb(exn) == ERROR_RAISE: + raise exn
def writeMultiLibConfig(self): if not self.data.packages.multiLib: @@ -1571,8 +1564,6 @@ reposdir=%s def postInstall(self): """ Perform post-installation tasks. """ with _yum_lock: - self._yum.close() - # clean up repo tmpdirs self._yum.cleanPackages() self._yum.cleanHeaders() @@ -1594,145 +1585,4 @@ reposdir=%s
super(YumPayload, self).postInstall()
-class RPMCallback(object): - def __init__(self, yb, log, upgrade=False): - self._yum = yb # yum.YumBase - self.install_log = log # logfile for yum script logs - self.upgrade = upgrade # boolean - - self.package_file = None # file instance (package file management)
- self.total_actions = 0 - self.completed_actions = None # will be set to 0 when starting tx - self.base_arch = blivet.arch.getArch() - - def _get_txmbr(self, key): - """ Return a (name, TransactionMember) tuple from cb key. """ - if hasattr(key, "po"): - # New-style callback, key is a TransactionMember - txmbr = key - name = key.name - else: - # cleanup/remove error - name = key - txmbr = None - - return (name, txmbr) - - def callback(self, event, amount, total, key, userdata): - """ Yum install callback. """ - if event == rpm.RPMCALLBACK_TRANS_START: - if amount == 6: - progress.send_message(_("Preparing transaction from installation source")) - self.total_actions = total - self.completed_actions = 0 - elif event == rpm.RPMCALLBACK_TRANS_PROGRESS: - # amount / total complete - pass - elif event == rpm.RPMCALLBACK_TRANS_STOP: - # we are done - pass - elif event == rpm.RPMCALLBACK_INST_OPEN_FILE: - # update status that we're installing/upgrading %h - # return an open fd to the file - txmbr = self._get_txmbr(key)[1] - - # If self.completed_actions is still None, that means this package - # is being opened to retrieve a %pretrans script. Don't log that - # we're installing the package unless we've been called with a - # TRANS_START event. - if self.completed_actions is not None: - if self.upgrade: - mode = _("Upgrading") - else: - mode = _("Installing") - - self.completed_actions += 1 - msg_format = "%s %s (%d/%d)" - progress_package = txmbr.name - if txmbr.arch not in ["noarch", self.base_arch]: - progress_package = "%s.%s" % (txmbr.name, txmbr.arch) - - progress_msg = msg_format % (mode, progress_package, - self.completed_actions, - self.total_actions) - log_msg = msg_format % (mode, txmbr.po, - self.completed_actions, - self.total_actions) - log.info(log_msg) - self.install_log.write(log_msg+"\n") - self.install_log.flush() - - progress.send_message(progress_msg) - - self.package_file = None - repo = self._yum.repos.getRepo(txmbr.po.repoid) - - while self.package_file is None: - try: - # checkfunc gets passed to yum's use of URLGrabber which - # then calls it with the file being fetched. verifyPkg - # makes sure the checksum matches the one in the metadata. - # - # From the URLGrab documents: - # checkfunc=(function, ('arg1', 2), {'kwarg': 3}) - # results in a callback like: - # function(obj, 'arg1', 2, kwarg=3) - # obj.filename = '/tmp/stuff' - # obj.url = 'http://foo.com/stuff' - checkfunc = (self._yum.verifyPkg, (txmbr.po, 1), {}) - package_path = repo.getPackage(txmbr.po, checkfunc=checkfunc) - except URLGrabError as e: - log.error("URLGrabError: %s" % (e,)) - exn = PayloadInstallError("failed to get package") - if errorHandler.cb(exn, package=txmbr.po) == ERROR_RAISE: - raise exn - except (yum.Errors.NoMoreMirrorsRepoError, IOError) as e: - if os.path.exists(txmbr.po.localPkg()): - os.unlink(txmbr.po.localPkg()) - log.debug("retrying download of %s" % txmbr.po) - continue - log.error("getPackage error: %s" % (e,)) - exn = PayloadInstallError("failed to open package") - if errorHandler.cb(exn, package=txmbr.po) == ERROR_RAISE: - raise exn - except yum.Errors.RepoError: - continue - - self.package_file = open(package_path) - - return self.package_file.fileno() - elif event == rpm.RPMCALLBACK_INST_CLOSE_FILE: - # close and remove the last opened file - # update count of installed/upgraded packages - package_path = self.package_file.name - self.package_file.close() - self.package_file = None - - if package_path.startswith(_yum_cache_dir): - try: - os.unlink(package_path) - except OSError as e: - log.debug("unable to remove file %s" % e.strerror) - - # rpm doesn't tell us when it's started post-trans stuff which can - # take a very long time. So when it closes the last package, just - # display the message. - if self.completed_actions == self.total_actions: - progress.send_message(_("Performing post-installation setup tasks")) - elif event == rpm.RPMCALLBACK_UNINST_START: - # update status that we're cleaning up %key - #progress.set_text(_("Cleaning up %s" % key)) - pass - elif event in (rpm.RPMCALLBACK_CPIO_ERROR, - rpm.RPMCALLBACK_UNPACK_ERROR, - rpm.RPMCALLBACK_SCRIPT_ERROR): - name = self._get_txmbr(key)[0] - - # Script errors store whether or not they're fatal in "total". So, - # we should only error out for fatal script errors or the cpio and - # unpack problems. - if event != rpm.RPMCALLBACK_SCRIPT_ERROR or total: - exn = PayloadInstallError("cpio, unpack, or fatal script error") - if errorHandler.cb(exn, package=name) == ERROR_RAISE: - raise exn
log.info("Running anaconda-yum to install packages")# Watch output for progress, debug and error informationinstall_errors = []try:for line in execReadlines("anaconda-yum", args):if line.startswith("PROGRESS_"):key, text = line.split(":", 2)msg = progress_map[key] + textprogress.send_message(msg)log.debug(msg)elif line.startswith("DEBUG:"):log.debug(line[6:])elif line.startswith("INFO:"):log.info(line[5:])elif line.startswith("WARN:"):log.warn(line[5:])elif line.startswith("ERROR:"):log.error(line[6:])install_errors.append(line[6:])else:log.debug(line)except IOError as e:log.error("Error running anaconda-yum: %s" % e)exn = PayloadInstallError(str(e))if errorHandler.cb(exn) == ERROR_RAISE:raise exnfinally:# log the contents of the scriptlet logfile if anyif os.path.exists(script_log): log.info("==== start rpm scriptlet logs ====")
with open(script_log.name) as f:
with open(script_log) as f: for l in f: log.info(l) log.info("==== end rpm scriptlet logs ====")
os.unlink(script_log.name)
os.unlink(script_log)if install_errors:exn = PayloadInstallError("\n".join(install_errors))if errorHandler.cb(exn) == ERROR_RAISE:raise exndef writeMultiLibConfig(self): if not self.data.packages.multiLib:
This section is most concerning to me, from a translation perspective. I'm not sure how good of a job we are doing right now with displaying correctly translated messages and yum errors but if we're not doing a good job of it, we could easily fix it given everything is in the same process.
Do you have any idea what shape it's in given these patches?
- Chris
On Wed, Apr 03, 2013 at 05:04:58PM -0400, Chris Lumens wrote:
log.info("Running anaconda-yum to install packages")# Watch output for progress, debug and error informationinstall_errors = []try:for line in execReadlines("anaconda-yum", args):if line.startswith("PROGRESS_"):key, text = line.split(":", 2)msg = progress_map[key] + textprogress.send_message(msg)log.debug(msg)elif line.startswith("DEBUG:"):log.debug(line[6:])elif line.startswith("INFO:"):log.info(line[5:])elif line.startswith("WARN:"):log.warn(line[5:])elif line.startswith("ERROR:"):log.error(line[6:])install_errors.append(line[6:])else:log.debug(line)except IOError as e:log.error("Error running anaconda-yum: %s" % e)exn = PayloadInstallError(str(e))if errorHandler.cb(exn) == ERROR_RAISE:raise exnfinally:# log the contents of the scriptlet logfile if anyif os.path.exists(script_log): log.info("==== start rpm scriptlet logs ====")
with open(script_log.name) as f:
with open(script_log) as f: for l in f: log.info(l) log.info("==== end rpm scriptlet logs ====")
os.unlink(script_log.name)
os.unlink(script_log)if install_errors:exn = PayloadInstallError("\n".join(install_errors))if errorHandler.cb(exn) == ERROR_RAISE:raise exndef writeMultiLibConfig(self): if not self.data.packages.multiLib:
This section is most concerning to me, from a translation perspective. I'm not sure how good of a job we are doing right now with displaying correctly translated messages and yum errors but if we're not doing a good job of it, we could easily fix it given everything is in the same process.
Do you have any idea what shape it's in given these patches?
It looks like yum is translated, but I am not sure what's going to happen to those translations when they get passed through the stdout|stdin link to the main process. I wanted to keep our translations out of the anaconda-yum script because of this so that's why I made the PROGRESS_* mapping.
If they do get mangled I'm not sure what we can do about it.
On Wed, 2013-04-03 at 15:14 -0700, Brian C. Lane wrote:
On Wed, Apr 03, 2013 at 05:04:58PM -0400, Chris Lumens wrote:
log.info("Running anaconda-yum to install packages")# Watch output for progress, debug and error informationinstall_errors = []try:for line in execReadlines("anaconda-yum", args):if line.startswith("PROGRESS_"):key, text = line.split(":", 2)msg = progress_map[key] + textprogress.send_message(msg)log.debug(msg)elif line.startswith("DEBUG:"):log.debug(line[6:])elif line.startswith("INFO:"):log.info(line[5:])elif line.startswith("WARN:"):log.warn(line[5:])elif line.startswith("ERROR:"):log.error(line[6:])install_errors.append(line[6:])else:log.debug(line)except IOError as e:log.error("Error running anaconda-yum: %s" % e)exn = PayloadInstallError(str(e))if errorHandler.cb(exn) == ERROR_RAISE:raise exnfinally:# log the contents of the scriptlet logfile if anyif os.path.exists(script_log): log.info("==== start rpm scriptlet logs ====")
with open(script_log.name) as f:
with open(script_log) as f: for l in f: log.info(l) log.info("==== end rpm scriptlet logs ====")
os.unlink(script_log.name)
os.unlink(script_log)if install_errors:exn = PayloadInstallError("\n".join(install_errors))if errorHandler.cb(exn) == ERROR_RAISE:raise exndef writeMultiLibConfig(self): if not self.data.packages.multiLib:
This section is most concerning to me, from a translation perspective. I'm not sure how good of a job we are doing right now with displaying correctly translated messages and yum errors but if we're not doing a good job of it, we could easily fix it given everything is in the same process.
Do you have any idea what shape it's in given these patches?
It looks like yum is translated, but I am not sure what's going to happen to those translations when they get passed through the stdout|stdin link to the main process. I wanted to keep our translations out of the anaconda-yum script because of this so that's why I made the PROGRESS_* mapping.
If they do get mangled I'm not sure what we can do about it.
I believe we could use the 'codecs' module for correct utf-8 handling in that case (we could e.g. wrap the Popen.stdout file object).
It looks like yum is translated, but I am not sure what's going to happen to those translations when they get passed through the stdout|stdin link to the main process. I wanted to keep our translations out of the anaconda-yum script because of this so that's why I made the PROGRESS_* mapping.
If they do get mangled I'm not sure what we can do about it.
I think I'd prefer to take these patches and deal with any translation errors later. At the least, we are fixing real bugs here.
- Chris
On Wed, Apr 03, 2013 at 04:51:49PM -0400, Chris Lumens wrote:
+def execReadlines(command, argv, stdin=None, root='/', env_prune=[]):
You meant env_prune=None here. I'll spare you the standard lecture since you did it correctly later on, so you obviously just spaced out on this one.
Yeah, I must have copied that from someplace else. I'll fix it up.
anaconda-patches@lists.fedorahosted.org
-
Brian C. Lane -
Chris Lumens -
Vratislav Podzimek