New subject: [lorax] Split compression into separate methods; use xz to compress initrd

14 Feb 2011

xz compression makes the initrd 33% smaller (136M -> 90M). The extra
memory overhead at decompression time is negligible: testing showed
that any system with enough RAM to use the gzip-compressed initrd was
also able to load the xz-compressed initrd with no trouble.
Note that '--check=crc32' is needed because the kernel doesn't know
how to perform the default xz integrity check (crc64).
---
 src/pylorax/constants.py   |    2 ++
 src/pylorax/installtree.py |   24 ++++++++++++------------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/pylorax/constants.py b/src/pylorax/constants.py
index 547a938..4967450 100644
--- a/src/pylorax/constants.py
+++ b/src/pylorax/constants.py
@@ -41,6 +41,7 @@ class LoraxRequiredCommands(dict):
         self["DMSETUP"] = "dmsetup"
         self["FIND"] = "find"
         self["GCONFTOOL"] = "gconftool-2"
+        self["GZIP"] = "gzip"
         self["IMPLANTISOMD5"] = "implantisomd5"
         self["ISOHYBRID"] = "isohybrid"
         self["LDCONFIG"] = "ldconfig"
@@ -52,6 +53,7 @@ class LoraxRequiredCommands(dict):
         self["MOUNT"] = "mount"
         self["PARTED"] = "parted"
         self["UMOUNT"] = "umount"
+        self["XZ"] = "xz"
def __getattr__(self, attr):
         return self[attr]
diff --git a/src/pylorax/installtree.py b/src/pylorax/installtree.py
index 9883bba..e1a0bdb 100644
--- a/src/pylorax/installtree.py
+++ b/src/pylorax/installtree.py
@@ -25,7 +25,6 @@ logger = logging.getLogger("pylorax.installtree")
 import sys
 import os
 import shutil
-import gzip
 import re
 import glob
 import time
@@ -49,6 +48,14 @@ class LoraxInstallTree(BaseLoraxClass):
self.lcmds = constants.LoraxRequiredCommands()
+    def compress_initrd_pipe(self, stdin, stdout):
+        return subprocess.Popen([self.lcmds.XZ, '--check=crc32', '-9', '-c'],
+                                 stdin=stdin, stdout=stdout)
+
+    def compress_module(self, filename):
+        return subprocess.call([self.lcmds.GZIP, '-9', '-f',
+                                filename])
+
     def remove_locales(self):
         chroot = lambda: os.chroot(self.root)
@@ -310,14 +317,7 @@ class LoraxInstallTree(BaseLoraxClass):
         for root, _, fnames in os.walk(moddir):
             for fname in filter(lambda f: f.endswith(".ko"), fnames):
                 path = os.path.join(root, fname)
-                with open(path, "rb") as fobj:
-                    data = fobj.read()
-
-                gzipped = gzip.open("{0}.gz".format(path), "wb")
-                gzipped.write(data)
-                gzipped.close()
-
-                os.unlink(path)
+                self.compress_module(path)
def run_depmod(self, kernel):
         systemmap = "System.map-{0.version}".format(kernel)
@@ -521,9 +521,9 @@ class LoraxInstallTree(BaseLoraxClass):
                                 stdin=find.stdout, stdout=subprocess.PIPE,
                                 preexec_fn=chdir)
-        gzipped = gzip.open(initrd.fpath, "wb")
-        gzipped.write(cpio.stdout.read())
-        gzipped.close()
+        compress = self.compress_initrd_pipe(stdin=cpio.stdout,
+                                             stdout=open(initrd.fpath,"wb"))
+        compress.communicate()
# move modules out of the tree again
         shutil.move(joinpaths(self.root, "modules", kernel.version),
-- 
1.7.4


    