From: "Brian C. Lane" bcl@redhat.com
Add error checking, don't fail the whole operation on just one part failing.
Make sure execution happens inside chroot. Some of them were actually running on the host's filesystem.
refactor deps gathering so rpm -qa is only run once.
(note that this patch will be squashed with the previous one and these comments removed). --- src/pylorax/treebuilder.py | 140 +++++++++++++++++++++++++++++---------------- src/sbin/livemedia-creator | 2 +- 2 files changed, 91 insertions(+), 51 deletions(-)
diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py index 0311118..2581a1b 100644 --- a/src/pylorax/treebuilder.py +++ b/src/pylorax/treebuilder.py @@ -28,8 +28,8 @@ from shutil import copytree, copy2 from base import DataHolder from ltmpl import LoraxTemplateRunner import imgutils -from pylorax.executils import runcmd, runcmd_output -from subprocess import Popen, PIPE, STDOUT +from pylorax.executils import runcmd, runcmd_output, execWithCapture +from subprocess import STDOUT, CalledProcessError
templatemap = { 'i386': 'x86.tmpl', @@ -180,92 +180,132 @@ class TreeBuilder(object): self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir) self._runner.defaults = self.vars self.templatedir = templatedir + self._manifest_path = joinpaths(self.vars.outroot, "isolinux")
@property def kernels(self): return findkernels(root=self.vars.inroot)
def collect_manifest(self, cmd, output_file, sort=False): - output = runcmd_output(cmd, root=self.vars.inroot) - with open(os.path.join(self.manifest_path, output_file), "w") as f: - if sort: - output = output.split("\n") - for line in sorted(set(output),key=str.lower): - if line: - f.write("%s\n" % line) - else: - f.write(output) + """ Collect the output of cmd into output_file, optionally sorting it. + """ + try: + output = runcmd_output(cmd, root=self.vars.inroot) + except CalledProcessError as e: + logger.error(e) + else: + with open(joinpaths(self._manifest_path, output_file), "w") as f: + if sort: + output = output.split("\n") + for line in sorted(set(output), key=str.lower): + if line: + f.write("%s\n" % line) + else: + f.write(output)
def create_manifests(self): - self.manifest_path = os.path.join(self.vars.outroot, "isolinux") - env = os.environ.copy() - env['LC_ALL'] = 'C' - os.makedirs(self.manifest_path) + """ Create the manifest files + + This gathers up various pieces of information about the + installed image and writes it to manifest_* files in + the isolinux directory. + """ + os.makedirs(self._manifest_path) # collect installed packages information + logger.info("Collecting information on installed packages") cmd = ["rpm", "-qa", "--qf", "%{name}-%{version}-%{release}.%{arch} (%{SIGPGP:pgpsig})\n"] self.collect_manifest(cmd, "manifest-rpm.txt", True)
# collect source rpms + logger.info("Collecting information on source packages") cmd = ["rpm", "-qa", "--qf", "%{sourcerpm}\n"] self.collect_manifest(cmd, "manifest-srpm.txt", True)
# collect all included licenses rhbz#601927 + logger.info("Collecting information on installed licenses") cmd = ["rpm", "-qa", "--qf", "%{license}\n"] self.collect_manifest(cmd, "manifest-license.txt", True)
# get individial file sizes + logger.info("Collecting information on file sizes") cmd = ["du", "-akx", "--exclude=/var/cache/yum", "/"] self.collect_manifest(cmd, "manifest-file.txt", True)
# get directory sizes + logger.info("Collecting information on directory sizes") cmd = ["du", "-x", "--exclude=/var/cache/yum", "/"] self.collect_manifest(cmd, "manifest-dir.txt", True)
# find orphaned files file_list = [] - cmd = ["rpm", "-qa"] - installed_rpms = runcmd_output(cmd).strip() - - for pkg in installed_rpms.split("\n"): - cmd = ["rpm", "-ql", pkg] - pkg_files = runcmd_output(cmd).strip() - for f in pkg_files.split("\n"): - file_list.append("%s##%s" % (f, pkg)) - - cmd = ["find", "/", "-xdev"] - fs_list = runcmd_output(cmd).strip() - - for f in fs_list.split("\n"): - file_list.append("%s##Not owned by any package." % f) + deps_list = [] + installed_rpms = "" + try: + logger.info("Finding orphaned files and dependency list") + cmd = ["rpm", "-qa"] + installed_rpms = runcmd_output(cmd, root=self.vars.inroot).strip() + except CalledProcessError as e: + logger.error(e) + else: + for pkg in installed_rpms.split("\n"): + try: + cmd = ["rpm", "-ql", pkg] + pkg_files = runcmd_output(cmd, root=self.vars.inroot).strip() + except CalledProcessError as e: + logger.error(e) + else: + for f in pkg_files.split("\n"): + file_list.append((f, pkg)) + + # Get the dependencies + try: + cmd = ["rpm", "-e", "--test", pkg] + dep_files = execWithCapture(cmd[0], cmd[1:], stderr=STDOUT, + root=self.vars.inroot) + except RuntimeError as e: + logger.error(e) + else: + if dep_files: + for dep in dep_files.split("\n"): + deps_list.append(dep) + + deps_file = joinpaths(self._manifest_path, "manifest-deps.txt") + with open(deps_file, "w") as f: + for line in deps_list: + f.write("%s\n" % line) + + try: + cmd = ["find", "/", "-xdev"] + fs_list = runcmd_output(cmd, root=self.vars.inroot).strip() + except CalledProcessError as e: + logger.error(e) + else: + for f in fs_list.split("\n"): + file_list.append((f, "Not owned by any package."))
final_list = {} - - for item in file_list: - f, owner = item.split("##") + for f, owner in file_list: if not f in final_list: final_list[f] = owner
- output_file = os.path.join(self.manifest_path, "manifest-owns.txt") - with open(output_file, "w") as t: - for x in final_list.keys(): - t.write("%s\n%s\n" % (x, final_list[x])) - - # get dependency list - cmd = ["rpm", "-qa"] - output = runcmd_output(cmd, root=self.vars.inroot) - piped_cmd = ["xargs", "-n1", "rpm", "-e", "--test"] - run_cmd = Popen(piped_cmd, stdout=PIPE, stdin=PIPE, stderr=PIPE) - stdout_data = run_cmd.communicate(input=output)[1] - deps_file = os.path.join(self.manifest_path, "manifest-deps.txt") - with open(deps_file, "w") as f: - f.write("%s\n" % stdout_data) + output_file = joinpaths(self._manifest_path, "manifest-owns.txt") + with open(output_file, "w") as f: + for x in final_list: + f.write("%s\n%s\n" % (x, final_list[x])) +
- # bzip large manufest files + # bzip large manifest files + logger.info("Compressing manifest files") for f in ["deps", "owns", "file", "dir" ]: - path = os.path.join(self.manifest_path, "manifest-%s.txt" % f) - cmd = ["bzip2", path] - runcmd(cmd) + path = joinpaths(self._manifest_path, "manifest-%s.txt" % f) + if not os.path.exists(path): + continue + try: + cmd = ["bzip2", path] + runcmd(cmd) + except CalledProcessError as e: + logger.error(e)
def rebuild_initrds(self, add_args=[], backup="", prefix=""): '''Rebuild all the initrds in the tree. If backup is specified, each diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index 61c4a5e..d57655f 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -535,8 +535,8 @@ def make_livecd(opts, mount_dir, work_dir): log.info("dracut args = {0}".format(dracut_args)) tb.rebuild_initrds(add_args=dracut_args) log.info("Building boot.iso") - log.info("Generating Manifests") if opts.create_manifests and opts.make_iso: + log.info("Generating Manifests") tb.create_manifests() tb.build()