New: [rhinstaller/lorax/pulls/33 master] verify that everything in /usr/bin and /usr/sbin is still runnable after cleanup
spice-vdagent, part of the thing that makes copy/paste work between a host and a virt guest, is missing libraries in rawhide. It isn't as noticeable when spice-vdagent breaks as it is, say, when dhclient breaks, so it's probably been broken for a little while. I would like changes like this to be less of a surprise.
I don't have any strong opinions about whether verify should be on or off by default as long as the jenkins job runs with it on.
From: David Shea dshea@redhat.com
After the cleanup step, check that everything in /usr/bin and /usr/sbin can still run. Currently, this just checks that ELF files have everything they need to link, and scripts have an interpreter.
Verifying is on by default but can be skipped with --noverify --- src/pylorax/__init__.py | 10 ++++++++- src/pylorax/treebuilder.py | 52 +++++++++++++++++++++++++++++++++++++++++++++- src/sbin/lorax | 4 +++- 3 files changed, 63 insertions(+), 3 deletions(-)
diff --git a/src/pylorax/__init__.py b/src/pylorax/__init__.py index b1178d0..cbe66c3 100644 --- a/src/pylorax/__init__.py +++ b/src/pylorax/__init__.py @@ -156,7 +156,8 @@ def run(self, dbo, product, version, release, variant="", bugurl="", add_templates=None, add_template_vars=None, add_arch_templates=None, - add_arch_template_vars=None): + add_arch_template_vars=None, + verify=True):
assert self._configured
@@ -289,6 +290,13 @@ def run(self, dbo, product, version, release, variant="", bugurl="", logger.info("cleaning unneeded files") rb.cleanup()
+ if verify: + logger.info("verifying the installroot") + if not rb.verify(): + sys.exit(1) + else: + logger.info("Skipping verify") + if self.debug: rb.writepkgsizes(joinpaths(logdir, "final-pkgsizes.txt"))
diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py index bea9bcb..5d4f8b7 100644 --- a/src/pylorax/treebuilder.py +++ b/src/pylorax/treebuilder.py @@ -23,12 +23,14 @@ import os, re from os.path import basename from shutil import copytree, copy2 +from pathlib import Path +import itertools
from pylorax.sysutils import joinpaths, remove from pylorax.base import DataHolder from pylorax.ltmpl import LoraxTemplateRunner import pylorax.imgutils as imgutils -from pylorax.executils import runcmd, runcmd_output +from pylorax.executils import runcmd, runcmd_output, execWithCapture
templatemap = { 'i386': 'x86.tmpl', @@ -143,6 +145,54 @@ def cleanup(self): '''Remove unneeded packages and files with runtime-cleanup.tmpl''' self._runner.run("runtime-cleanup.tmpl")
+ def verify(self): + '''Ensure that contents of the installroot can run''' + status = True + + ELF_MAGIC = b'\x7fELF' + + # Iterate over all files in /usr/bin and /usr/sbin + # For ELF files, gather them into a list and we'll check them all at + # the end. For files with a #!, check them as we go + elf_files = [] + usr_bin = Path(self.vars.root + '/usr/bin') + usr_sbin = Path(self.vars.root + '/usr/sbin') + for path in (str(x) for x in itertools.chain(usr_bin.iterdir(), usr_sbin.iterdir()) \ + if x.is_file()): + with open(path, "rb") as f: + magic = f.read(4) + if magic == ELF_MAGIC: + # Save the path, minus the chroot prefix + elf_files.append(path[len(self.vars.root):]) + elif magic[:2] == b'#!': + # Reopen the file as text and read the first line. + # Open as latin-1 so that stray 8-bit characters don't make + # things blow up. We only really care about ASCII parts. + with open(path, "rt", encoding="latin-1") as f_text: + # Remove the #!, split on space, and take the first part + shabang = f_text.readline()[2:].split()[0] + + # Does the path exist? + if not os.path.exists(self.vars.root + shabang): + logger.error('%s, needed by %s, does not exist', shabang, path) + status = False + + # Now, run ldd on all the ELF files + # Just run ldd once on everything so it isn't logged a million times. + # At least one thing in the list isn't going to be a dynamic executable, + # so use execWithCapture to ignore the exit code. + filename = '' + for line in execWithCapture('ldd', elf_files, root=self.vars.root, + log_output=False, filter_stderr=True).split('\n'): + if line and not line[0].isspace(): + # New filename header, strip the : at the end and save + filename = line[:-1] + elif 'not found' in line: + logger.error('%s, needed by %s, not found', line.split()[0], filename) + status = False + + return status + def writepkgsizes(self, pkgsizefile): '''debugging data: write a big list of pkg sizes''' fobj = open(pkgsizefile, "w") diff --git a/src/sbin/lorax b/src/sbin/lorax index 93e9ed7..f92aeb9 100755 --- a/src/sbin/lorax +++ b/src/sbin/lorax @@ -114,6 +114,8 @@ def main(args): optional.add_argument("--add-arch-template-var", dest="add_arch_template_vars", action="append", help="Set variable for architecture-specific image", default=[]) + optional.add_argument("--noverify", action="store_false", default=True, dest="verify", + help="Do not verify the install root")
# add the show version option parser.add_argument("-V", help="show program's version number and exit", @@ -200,7 +202,7 @@ def main(args): add_template_vars=parsed_add_template_vars, add_arch_templates=opts.add_arch_templates, add_arch_template_vars=parsed_add_arch_template_vars, - remove_temp=True) + remove_temp=True, verify=opts.verify)
def get_dnf_base_object(installroot, repositories, mirrorlists=None,
From: David Shea dshea@redhat.com
systemd-nspawn is some kind of container thing, and cairo-sphinx, as far as I can tell (go ahead, try to google it) is a cairo debugging tool. Neither of these are particularly useful on installer media and both use libraries that are removed during cleanup. --- share/runtime-cleanup.tmpl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/share/runtime-cleanup.tmpl b/share/runtime-cleanup.tmpl index 1dc720c..4a7ea5d 100644 --- a/share/runtime-cleanup.tmpl +++ b/share/runtime-cleanup.tmpl @@ -48,6 +48,10 @@ remove /etc/logrotate.d ## anaconda needs this to do media check removefrom isomd5sum --allbut /usr/bin/checkisomd5
+## systemd-nspawn isn't very useful and doesn't link anyway without iptables, +## and there's no need for a bunch of zsh files without zsh +removefrom systemd /usr/bin/systemd-nspawn /usr/share/zsh + ## various other things we remove to save space removepkg avahi-autoipd coreutils-libs dash db4-utils diffutils file removepkg genisoimage info iptables @@ -130,7 +134,7 @@ removefrom bind-utils /usr/bin/dig /usr/bin/host /usr/bin/nsupdate removefrom bitmap-fangsongti-fonts /usr/share/fonts/* removefrom ca-certificates /etc/pki/java/* removefrom ca-certificates /etc/pki/tls/certs/ca-bundle.trust.crt /etc/ssl/* -removefrom cairo /usr/${libdir}/libcairo-script* +removefrom cairo /usr/${libdir}/libcairo-script* /usr/bin/cairo-sphinx removefrom coreutils /etc/* /usr/bin/link /usr/bin/nice /usr/bin/stty /usr/bin/su /usr/bin/unlink removefrom coreutils /usr/sbin/runuser /usr/bin/[ /usr/bin/base64 /usr/bin/chcon removefrom coreutils /usr/bin/cksum /usr/bin/comm /usr/bin/csplit
From: David Shea dshea@redhat.com
spice-vdagent, half of the thing that lets copy/paste work across a spice connection, added sound support, I guess, so now we need alsa-libs. --- share/runtime-cleanup.tmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/share/runtime-cleanup.tmpl b/share/runtime-cleanup.tmpl index 4a7ea5d..8100d16 100644 --- a/share/runtime-cleanup.tmpl +++ b/share/runtime-cleanup.tmpl @@ -10,7 +10,8 @@ remove usr/share/i18n removepkg perl* %endif ## no sound support, thanks -removepkg alsa* flac gstreamer-tools libsndfile pulseaudio* sound-theme-freedesktop +## ...except alsa-libs, which are needed by spice-vdagent +removepkg alsa-*firmware* flac gstreamer-tools libsndfile pulseaudio* sound-theme-freedesktop removepkg midisport-firmware ## no fancy video, either removepkg libcrystalhd crystalhd-firmware ivtv-firmware cx18-firmware
From: David Shea dshea@redhat.com
We really shouldn't need to run ldconfig on boot from read-only media, unless someone messed up %post in a package. And the verify step will catch that.
This reverts commit 3981ff5b795401f26a7c76b20b6d1141ea5e8bfa. --- share/runtime-postinstall.tmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/share/runtime-postinstall.tmpl b/share/runtime-postinstall.tmpl index 83373bf..336bc3c 100644 --- a/share/runtime-postinstall.tmpl +++ b/share/runtime-postinstall.tmpl @@ -41,7 +41,8 @@ systemctl disable systemd-readahead-collect.service \ systemctl mask fedora-configure.service fedora-loadmodules.service \ fedora-autorelabel.service fedora-autorelabel-mark.service \ fedora-wait-storage.service media.mount \ - systemd-tmpfiles-clean.service systemd-tmpfiles-clean.timer + systemd-tmpfiles-clean.service systemd-tmpfiles-clean.timer \ + ldconfig.service
## Make logind activate anaconda-shell@.service on switch to empty VT symlink anaconda-shell@.service lib/systemd/system/autovt@.service
Nice! Looks good to me.
Added label: ACK.
Very nice! Feel free to push when you want.
Pushed.
Closed.
anaconda-patches@lists.fedorahosted.org
-
dashea