[PATCH pylorax] livemedia-creator: add file system manifest creation option for --make-iso builds
This add a fine level of details about the created image including: - installed packages - source rpms used - licensing - dependencies - where disk space is used
Signed-off-by: Joey Boggs jboggs@redhat.com --- src/pylorax/treebuilder.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++ src/sbin/livemedia-creator | 6 ++++ 2 files changed, 90 insertions(+)
diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py index 2e7c43e..0311118 100644 --- a/src/pylorax/treebuilder.py +++ b/src/pylorax/treebuilder.py @@ -29,6 +29,7 @@ from base import DataHolder from ltmpl import LoraxTemplateRunner import imgutils from pylorax.executils import runcmd, runcmd_output +from subprocess import Popen, PIPE, STDOUT
templatemap = { 'i386': 'x86.tmpl', @@ -184,6 +185,88 @@ class TreeBuilder(object): def kernels(self): return findkernels(root=self.vars.inroot)
+ def collect_manifest(self, cmd, output_file, sort=False): + output = runcmd_output(cmd, root=self.vars.inroot) + with open(os.path.join(self.manifest_path, output_file), "w") as f: + if sort: + output = output.split("\n") + for line in sorted(set(output),key=str.lower): + if line: + f.write("%s\n" % line) + else: + f.write(output) + + def create_manifests(self): + self.manifest_path = os.path.join(self.vars.outroot, "isolinux") + env = os.environ.copy() + env['LC_ALL'] = 'C' + os.makedirs(self.manifest_path) + # collect installed packages information + cmd = ["rpm", "-qa", "--qf", + "%{name}-%{version}-%{release}.%{arch} (%{SIGPGP:pgpsig})\n"] + self.collect_manifest(cmd, "manifest-rpm.txt", True) + + # collect source rpms + cmd = ["rpm", "-qa", "--qf", "%{sourcerpm}\n"] + self.collect_manifest(cmd, "manifest-srpm.txt", True) + + # collect all included licenses rhbz#601927 + cmd = ["rpm", "-qa", "--qf", "%{license}\n"] + self.collect_manifest(cmd, "manifest-license.txt", True) + + # get individial file sizes + cmd = ["du", "-akx", "--exclude=/var/cache/yum", "/"] + self.collect_manifest(cmd, "manifest-file.txt", True) + + # get directory sizes + cmd = ["du", "-x", "--exclude=/var/cache/yum", "/"] + self.collect_manifest(cmd, "manifest-dir.txt", True) + + # find orphaned files + file_list = [] + cmd = ["rpm", "-qa"] + installed_rpms = runcmd_output(cmd).strip() + + for pkg in installed_rpms.split("\n"): + cmd = ["rpm", "-ql", pkg] + pkg_files = runcmd_output(cmd).strip() + for f in pkg_files.split("\n"): + file_list.append("%s##%s" % (f, pkg)) + + cmd = ["find", "/", "-xdev"] + fs_list = runcmd_output(cmd).strip() + + for f in fs_list.split("\n"): + file_list.append("%s##Not owned by any package." % f) + + final_list = {} + + for item in file_list: + f, owner = item.split("##") + if not f in final_list: + final_list[f] = owner + + output_file = os.path.join(self.manifest_path, "manifest-owns.txt") + with open(output_file, "w") as t: + for x in final_list.keys(): + t.write("%s\n%s\n" % (x, final_list[x])) + + # get dependency list + cmd = ["rpm", "-qa"] + output = runcmd_output(cmd, root=self.vars.inroot) + piped_cmd = ["xargs", "-n1", "rpm", "-e", "--test"] + run_cmd = Popen(piped_cmd, stdout=PIPE, stdin=PIPE, stderr=PIPE) + stdout_data = run_cmd.communicate(input=output)[1] + deps_file = os.path.join(self.manifest_path, "manifest-deps.txt") + with open(deps_file, "w") as f: + f.write("%s\n" % stdout_data) + + # bzip large manufest files + for f in ["deps", "owns", "file", "dir" ]: + path = os.path.join(self.manifest_path, "manifest-%s.txt" % f) + cmd = ["bzip2", path] + runcmd(cmd) + def rebuild_initrds(self, add_args=[], backup="", prefix=""): '''Rebuild all the initrds in the tree. If backup is specified, each initrd will be renamed with backup as a suffix before rebuilding. @@ -269,6 +352,7 @@ def findkernels(root="/", kdir="boot"): r"(.(?P<flavor>{0}))?)$".format("|".join(flavors))) kernels = [] bootfiles = os.listdir(joinpaths(root, kdir)) + for f in bootfiles: match = kre.match(f) if match: diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index 9239445..61c4a5e 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -535,6 +535,9 @@ def make_livecd(opts, mount_dir, work_dir): log.info("dracut args = {0}".format(dracut_args)) tb.rebuild_initrds(add_args=dracut_args) log.info("Building boot.iso") + log.info("Generating Manifests") + if opts.create_manifests and opts.make_iso: + tb.create_manifests() tb.build()
return work_dir @@ -814,6 +817,9 @@ if __name__ == '__main__': parser.add_argument( "--volid", default=None, help="volume id") parser.add_argument( "--squashfs_args", help="additional squashfs args" ) + parser.add_argument( "--create-manifests", action="store_true", + help="Create file system manifests of image", + default=False )
opts = parser.parse_args()
From: "Brian C. Lane" bcl@redhat.com
Add error checking, don't fail the whole operation on just one part failing.
Make sure execution happens inside chroot. Some of them were actually running on the host's filesystem.
refactor deps gathering so rpm -qa is only run once.
(note that this patch will be squashed with the previous one and these comments removed). --- src/pylorax/treebuilder.py | 140 +++++++++++++++++++++++++++++---------------- src/sbin/livemedia-creator | 2 +- 2 files changed, 91 insertions(+), 51 deletions(-)
diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py index 0311118..2581a1b 100644 --- a/src/pylorax/treebuilder.py +++ b/src/pylorax/treebuilder.py @@ -28,8 +28,8 @@ from shutil import copytree, copy2 from base import DataHolder from ltmpl import LoraxTemplateRunner import imgutils -from pylorax.executils import runcmd, runcmd_output -from subprocess import Popen, PIPE, STDOUT +from pylorax.executils import runcmd, runcmd_output, execWithCapture +from subprocess import STDOUT, CalledProcessError
templatemap = { 'i386': 'x86.tmpl', @@ -180,92 +180,132 @@ class TreeBuilder(object): self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir) self._runner.defaults = self.vars self.templatedir = templatedir + self._manifest_path = joinpaths(self.vars.outroot, "isolinux")
@property def kernels(self): return findkernels(root=self.vars.inroot)
def collect_manifest(self, cmd, output_file, sort=False): - output = runcmd_output(cmd, root=self.vars.inroot) - with open(os.path.join(self.manifest_path, output_file), "w") as f: - if sort: - output = output.split("\n") - for line in sorted(set(output),key=str.lower): - if line: - f.write("%s\n" % line) - else: - f.write(output) + """ Collect the output of cmd into output_file, optionally sorting it. + """ + try: + output = runcmd_output(cmd, root=self.vars.inroot) + except CalledProcessError as e: + logger.error(e) + else: + with open(joinpaths(self._manifest_path, output_file), "w") as f: + if sort: + output = output.split("\n") + for line in sorted(set(output), key=str.lower): + if line: + f.write("%s\n" % line) + else: + f.write(output)
def create_manifests(self): - self.manifest_path = os.path.join(self.vars.outroot, "isolinux") - env = os.environ.copy() - env['LC_ALL'] = 'C' - os.makedirs(self.manifest_path) + """ Create the manifest files + + This gathers up various pieces of information about the + installed image and writes it to manifest_* files in + the isolinux directory. + """ + os.makedirs(self._manifest_path) # collect installed packages information + logger.info("Collecting information on installed packages") cmd = ["rpm", "-qa", "--qf", "%{name}-%{version}-%{release}.%{arch} (%{SIGPGP:pgpsig})\n"] self.collect_manifest(cmd, "manifest-rpm.txt", True)
# collect source rpms + logger.info("Collecting information on source packages") cmd = ["rpm", "-qa", "--qf", "%{sourcerpm}\n"] self.collect_manifest(cmd, "manifest-srpm.txt", True)
# collect all included licenses rhbz#601927 + logger.info("Collecting information on installed licenses") cmd = ["rpm", "-qa", "--qf", "%{license}\n"] self.collect_manifest(cmd, "manifest-license.txt", True)
# get individial file sizes + logger.info("Collecting information on file sizes") cmd = ["du", "-akx", "--exclude=/var/cache/yum", "/"] self.collect_manifest(cmd, "manifest-file.txt", True)
# get directory sizes + logger.info("Collecting information on directory sizes") cmd = ["du", "-x", "--exclude=/var/cache/yum", "/"] self.collect_manifest(cmd, "manifest-dir.txt", True)
# find orphaned files file_list = [] - cmd = ["rpm", "-qa"] - installed_rpms = runcmd_output(cmd).strip() - - for pkg in installed_rpms.split("\n"): - cmd = ["rpm", "-ql", pkg] - pkg_files = runcmd_output(cmd).strip() - for f in pkg_files.split("\n"): - file_list.append("%s##%s" % (f, pkg)) - - cmd = ["find", "/", "-xdev"] - fs_list = runcmd_output(cmd).strip() - - for f in fs_list.split("\n"): - file_list.append("%s##Not owned by any package." % f) + deps_list = [] + installed_rpms = "" + try: + logger.info("Finding orphaned files and dependency list") + cmd = ["rpm", "-qa"] + installed_rpms = runcmd_output(cmd, root=self.vars.inroot).strip() + except CalledProcessError as e: + logger.error(e) + else: + for pkg in installed_rpms.split("\n"): + try: + cmd = ["rpm", "-ql", pkg] + pkg_files = runcmd_output(cmd, root=self.vars.inroot).strip() + except CalledProcessError as e: + logger.error(e) + else: + for f in pkg_files.split("\n"): + file_list.append((f, pkg)) + + # Get the dependencies + try: + cmd = ["rpm", "-e", "--test", pkg] + dep_files = execWithCapture(cmd[0], cmd[1:], stderr=STDOUT, + root=self.vars.inroot) + except RuntimeError as e: + logger.error(e) + else: + if dep_files: + for dep in dep_files.split("\n"): + deps_list.append(dep) + + deps_file = joinpaths(self._manifest_path, "manifest-deps.txt") + with open(deps_file, "w") as f: + for line in deps_list: + f.write("%s\n" % line) + + try: + cmd = ["find", "/", "-xdev"] + fs_list = runcmd_output(cmd, root=self.vars.inroot).strip() + except CalledProcessError as e: + logger.error(e) + else: + for f in fs_list.split("\n"): + file_list.append((f, "Not owned by any package."))
final_list = {} - - for item in file_list: - f, owner = item.split("##") + for f, owner in file_list: if not f in final_list: final_list[f] = owner
- output_file = os.path.join(self.manifest_path, "manifest-owns.txt") - with open(output_file, "w") as t: - for x in final_list.keys(): - t.write("%s\n%s\n" % (x, final_list[x])) - - # get dependency list - cmd = ["rpm", "-qa"] - output = runcmd_output(cmd, root=self.vars.inroot) - piped_cmd = ["xargs", "-n1", "rpm", "-e", "--test"] - run_cmd = Popen(piped_cmd, stdout=PIPE, stdin=PIPE, stderr=PIPE) - stdout_data = run_cmd.communicate(input=output)[1] - deps_file = os.path.join(self.manifest_path, "manifest-deps.txt") - with open(deps_file, "w") as f: - f.write("%s\n" % stdout_data) + output_file = joinpaths(self._manifest_path, "manifest-owns.txt") + with open(output_file, "w") as f: + for x in final_list: + f.write("%s\n%s\n" % (x, final_list[x])) +
- # bzip large manufest files + # bzip large manifest files + logger.info("Compressing manifest files") for f in ["deps", "owns", "file", "dir" ]: - path = os.path.join(self.manifest_path, "manifest-%s.txt" % f) - cmd = ["bzip2", path] - runcmd(cmd) + path = joinpaths(self._manifest_path, "manifest-%s.txt" % f) + if not os.path.exists(path): + continue + try: + cmd = ["bzip2", path] + runcmd(cmd) + except CalledProcessError as e: + logger.error(e)
def rebuild_initrds(self, add_args=[], backup="", prefix=""): '''Rebuild all the initrds in the tree. If backup is specified, each diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index 61c4a5e..d57655f 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -535,8 +535,8 @@ def make_livecd(opts, mount_dir, work_dir): log.info("dracut args = {0}".format(dracut_args)) tb.rebuild_initrds(add_args=dracut_args) log.info("Building boot.iso") - log.info("Generating Manifests") if opts.create_manifests and opts.make_iso: + log.info("Generating Manifests") tb.create_manifests() tb.build()
From: "Brian C. Lane" bcl@redhat.com
Setting stderr would write to the passed file, etc. but would not allow combining the stdout and stderr output in the return value. This changes the behavior so that setting stderr=subprocess.STDOUT will result in the stderr output being returned by the method. Note that it may be mixed with stdout, depending on how the called process flushes its buffers. --- src/pylorax/executils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/src/pylorax/executils.py b/src/pylorax/executils.py index 6da74c8..2f0a042 100644 --- a/src/pylorax/executils.py +++ b/src/pylorax/executils.py @@ -192,6 +192,9 @@ def execWithCapture(command, argv, stdin = None, stderr = None, root=None, @param cwd working directory to pass to Popen @param raise_err raise CalledProcessError when the returncode is not 0 @return The output of command from stdout. + + Setting stderr to subprocess.STDOUT will combine stderr and stdout + in the return value. """ def chroot(): os.chroot(root) @@ -215,9 +218,13 @@ def execWithCapture(command, argv, stdin = None, stderr = None, root=None, elif stdin is None or not isinstance(stdin, file): stdin = sys.stdin.fileno()
+ stderr_pipe = subprocess.PIPE if isinstance(stderr, str): stderr = os.open(stderr, os.O_RDWR|os.O_CREAT) stderrclose = lambda : os.close(stderr) + elif stderr is subprocess.STDOUT: + stderr_pipe = subprocess.STDOUT + stderr = sys.stderr.fileno() elif isinstance(stderr, int): pass elif stderr is None or not isinstance(stderr, file): @@ -238,7 +245,7 @@ def execWithCapture(command, argv, stdin = None, stderr = None, root=None, try: proc = subprocess.Popen([command] + argv, stdin=stdin, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stderr=stderr_pipe, preexec_fn=preexec_fn, cwd=cwd, env=env)
anaconda-patches@lists.fedorahosted.org
-
Brian C. Lane -
Joey Boggs