diff --git a/API b/API index 9783c74..ef9c1aa 100644 --- a/API +++ b/API @@ -76,3 +76,7 @@ switching from the LiveImageCreator to another ImageCreator object. build live images which use dm-snapshot, etc. This is what is used by livecd-creator. +* DiskImageCreator: This generates disk images containing multiple + partitions in a loopback file. It installs grub in the MBR and + can be directly booted in any virtual machine + diff --git a/Makefile b/Makefile index 050fe88..d3d4ab7 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ all: install: $(INSTALL_PROGRAM) -D tools/livecd-creator $(DESTDIR)/usr/bin/livecd-creator $(INSTALL_PROGRAM) -D tools/image-creator $(DESTDIR)/usr/bin/image-creator + $(INSTALL_PROGRAM) -D tools/disk-creator $(DESTDIR)/usr/bin/disk-creator $(INSTALL_PROGRAM) -D tools/livecd-iso-to-disk.sh $(DESTDIR)/usr/bin/livecd-iso-to-disk $(INSTALL_PROGRAM) -D tools/livecd-iso-to-pxeboot.sh $(DESTDIR)/usr/bin/livecd-iso-to-pxeboot $(INSTALL_PROGRAM) -D tools/mayflower $(DESTDIR)/usr/lib/livecd-creator/mayflower @@ -34,6 +35,8 @@ install: uninstall: rm -f $(DESTDIR)/usr/bin/livecd-creator rm -rf $(DESTDIR)/usr/lib/livecd-creator + rm -rf $(DESTDIR)/usr/bin/disk-creator + rm -rf $(DESTDIR)/usr/bin/image-creator rm -rf $(DESTDIR)/usr/share/doc/livecd-tools-$(VERSION) rm -rf $(DESTDIR)/usr/share/livecd-tools diff --git a/imgcreate/__init__.py b/imgcreate/__init__.py index e535014..44f3ec5 100644 --- a/imgcreate/__init__.py +++ b/imgcreate/__init__.py @@ -21,6 +21,7 @@ from imgcreate.creator import * from imgcreate.yuminst import * from imgcreate.kickstart import * from imgcreate.fs import * +from imgcreate.disk import * """A set of classes for building Fedora system images. @@ -28,6 +29,7 @@ The following image creators are available: - ImageCreator - installs to a directory - LoopImageCreator - installs to an ext3 image - LiveImageCreator - installs to a bootable ISO + - DiskImageCreator - installs to a partitioned disk image Also exported are: - CreatorError - all exceptions throw are of this type @@ -60,6 +62,7 @@ __all__ = ( 'ImageCreator', 'LiveImageCreator', 'LoopImageCreator', + 'DiskImageCreator', 'FSLABEL_MAXLEN', 'read_kickstart', 'construct_name' diff --git a/imgcreate/creator.py b/imgcreate/creator.py index c2ed770..fb3b309 100644 --- a/imgcreate/creator.py +++ b/imgcreate/creator.py @@ -207,7 +207,11 @@ class ImageCreator(object): """ s = "/dev/root / %s defaults,noatime 0 0\n" %(self._fstype) - s += "devpts /dev/pts devpts gid=5,mode=620 0 0\n" + s += self._get_fstab_special() + return s + + def _get_fstab_special(self): + s = "devpts /dev/pts devpts gid=5,mode=620 0 0\n" s += "tmpfs /dev/shm tmpfs defaults 0 0\n" s += "proc /proc proc defaults 0 0\n" s += "sysfs /sys sysfs defaults 0 0\n" @@ -816,12 +820,11 @@ class LoopImageCreator(ImageCreator): if not base_on is None: shutil.copyfile(base_on, self._image) - self.__instloop = SparseExtLoopbackMount(self._image, - self._instroot, - self.__image_size, - self.__fstype, - self.__blocksize, - self.fslabel) + self.__instloop = ExtDiskMount(SparseLoopbackDisk(self._image, self.__image_size), + self._instroot, + self.__fstype, + self.__blocksize, + self.fslabel) try: self.__instloop.mount() diff --git a/imgcreate/disk.py b/imgcreate/disk.py index a0dd6e9..991db01 100644 --- a/imgcreate/disk.py +++ b/imgcreate/disk.py @@ -144,11 +144,12 @@ class DiskImageCreator(ImageCreator): def _create_grub_config(self): (bootdevnum, rootdevnum, rootdev, prefix) = self._get_grub_boot_config() + # XXX don't hardcode default - see livecd code grub = "" grub += "default=0\n" grub += "timeout=5\n" - grub += "splashimage=(hd0,%-d)%s/grub/splash.xpm.gz\n" % (rootdevnum, prefix) - grub += "hiddenemnu\n" + grub += "splashimage=(hd0,%d)%s/grub/splash.xpm.gz\n" % (bootdevnum, prefix) + grub += "hiddenmenu\n" versions = [] kernels = self._get_kernel_versions() @@ -158,11 +159,11 @@ class DiskImageCreator(ImageCreator): for v in versions: grub += "title Fedora (%s)\n" % v - grub += " root (hd0,%-d)\n" % bootdevnum + grub += " root (hd0,%d)\n" % bootdevnum grub += " kernel %s/vmlinuz-%s ro root=%s\n" % (prefix, v, rootdev) grub += " initrd %s/initrd-%s.img\n" % (prefix, v) - cfg = open(self._instroot + "/boot/grub/grub.cfg", "w") + cfg = open(self._instroot + "/boot/grub/grub.conf", "w") cfg.write(grub) cfg.close() @@ -185,11 +186,16 @@ class DiskImageCreator(ImageCreator): def _install_grub(self): (bootdevnum, rootdevnum, rootdev, prefix) = self._get_grub_boot_config() + # Ensure all data is flushed to disk before doing grub install + subprocess.call(["sync"]) + + stage2 = self._instroot + "/boot/grub/stage2" + setup = "" setup += "device (hd0) %s\n" % (self.__instloop.disk.device) - setup += "root (hd0,%d)\n" % rootdevnum - setup += "configfile (hd0,%d)%s/grub/grub.cfg" % (bootdevnum, prefix) - setup += "setup --prefix=%s/grub (hd0) (hd0,%d)\n" % (prefix, bootdevnum) + setup += "root (hd0,%d)\n" % bootdevnum + setup += "setup --stage2=%s --prefix=%s/grub (hd0)\n" % (stage2, prefix) + setup += "quit\n" grub = subprocess.Popen(["grub", "--batch", "--no-floppy"], stdin=subprocess.PIPE) diff --git a/imgcreate/fs.py b/imgcreate/fs.py index 9ca3a3e..b76f38b 100644 --- a/imgcreate/fs.py +++ b/imgcreate/fs.py @@ -86,42 +86,51 @@ class BindChrootMount: subprocess.call(["/bin/umount", self.dest]) self.mounted = False -class LoopbackMount: - def __init__(self, lofile, mountdir, fstype = None): - self.lofile = lofile - self.mountdir = mountdir - self.fstype = fstype +class Disk: + def __init__(self, size, device = None): + self._device = device + self._size = size - self.mounted = False - self.losetup = False - self.rmdir = False - self.loopdev = None + def create(self): + pass def cleanup(self): - self.unmount() - self.lounsetup() + pass - def unmount(self): - if self.mounted: - rc = subprocess.call(["/bin/umount", self.mountdir]) - if rc == 0: - self.mounted = False + def get_device(self): + return self._device + def set_device(self, path): + self._device = path + device = property(get_device, set_device) + + def get_size(self): + return self._size + size = property(get_size) - if self.rmdir and not self.mounted: - try: - os.rmdir(self.mountdir) - except OSError, e: - pass - self.rmdir = False - def lounsetup(self): - if self.losetup: - rc = subprocess.call(["/sbin/losetup", "-d", self.loopdev]) - self.losetup = False - self.loopdev = None +class RawDisk(Disk): + def __init__(self, size, device): + Disk.__init__(self, size, device) + + def fixed(self): + return True + + def exists(self): + return True + +class LoopbackDisk(Disk): + def __init__(self, lofile, size): + Disk.__init__(self, size) + self.lofile = lofile - def loopsetup(self): - if self.losetup: + def fixed(self): + return False + + def exists(self): + return os.path.exists(self.lofile) + + def create(self): + if self.device is not None: return losetupProc = subprocess.Popen(["/sbin/losetup", "-f"], @@ -132,40 +141,27 @@ class LoopbackMount: raise MountError("Failed to allocate loop device for '%s'" % self.lofile) - self.loopdev = losetupOutput.split()[0] + device = losetupOutput.split()[0] - rc = subprocess.call(["/sbin/losetup", self.loopdev, self.lofile]) + print "Losetup add %s %s" % (device, self.lofile) + rc = subprocess.call(["/sbin/losetup", device, self.lofile]) if rc != 0: raise MountError("Failed to allocate loop device for '%s'" % self.lofile) + self.device = device - self.losetup = True - - def mount(self): - if self.mounted: + def cleanup(self): + if self.device is None: return + print "Losetup remove %s" % self.device + rc = subprocess.call(["/sbin/losetup", "-d", self.device]) + self.device = None - self.loopsetup() - if not os.path.isdir(self.mountdir): - os.makedirs(self.mountdir) - self.rmdir = True - - args = [ "/bin/mount", self.loopdev, self.mountdir ] - if self.fstype: - args.extend(["-t", self.fstype]) - rc = subprocess.call(args) - if rc != 0: - raise MountError("Failed to mount '%s' to '%s'" % - (self.loopdev, self.mountdir)) - - self.mounted = True - -class SparseLoopbackMount(LoopbackMount): - def __init__(self, lofile, mountdir, size, fstype = None): - LoopbackMount.__init__(self, lofile, mountdir, fstype) - self.size = size +class SparseLoopbackDisk(LoopbackDisk): + def __init__(self, lofile, size): + LoopbackDisk.__init__(self, lofile, size) def expand(self, create = False, size = None): flags = os.O_WRONLY @@ -191,10 +187,81 @@ class SparseLoopbackMount(LoopbackMount): def create(self): self.expand(create = True) + LoopbackDisk.create(self) + +class Mount: + def __init__(self, mountdir): + self.mountdir = mountdir + + def cleanup(self): + self.unmount() + + def mount(self): + pass + + def unmount(self): + pass + +class DiskMount(Mount): + def __init__(self, disk, mountdir, fstype = None, rmmountdir = True): + Mount.__init__(self, mountdir) + + self.disk = disk + self.fstype = fstype + self.rmmountdir = rmmountdir + + self.mounted = False + self.rmdir = False + + def cleanup(self): + Mount.cleanup(self) + self.disk.cleanup() + + def unmount(self): + if self.mounted: + rc = subprocess.call(["/bin/umount", self.mountdir]) + if rc == 0: + self.mounted = False + + if self.rmdir and not self.mounted: + try: + os.rmdir(self.mountdir) + except OSError, e: + pass + self.rmdir = False + + + def __create(self): + print "Disk create %s" % str(self) + self.disk.create() + + + def mount(self): + if self.mounted: + return + + print "Disk mount" + if not os.path.isdir(self.mountdir): + os.makedirs(self.mountdir) + self.rmdir = self.rmmountdir + + self.__create() + + print "Do mount %s " % self.disk.device + args = [ "/bin/mount", self.disk.device, self.mountdir ] + if self.fstype: + args.extend(["-t", self.fstype]) + + rc = subprocess.call(args) + if rc != 0: + raise MountError("Failed to mount '%s' to '%s'" % + (self.disk.device, self.mountdir)) + + self.mounted = True -class SparseExtLoopbackMount(SparseLoopbackMount): - def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel): - SparseLoopbackMount.__init__(self, lofile, mountdir, size, fstype) +class ExtDiskMount(DiskMount): + def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True): + DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) self.blocksize = blocksize self.fslabel = fslabel @@ -202,19 +269,15 @@ class SparseExtLoopbackMount(SparseLoopbackMount): rc = subprocess.call(["/sbin/mkfs." + self.fstype, "-F", "-L", self.fslabel, "-m", "1", "-b", str(self.blocksize), - self.lofile, - str(self.size / self.blocksize)]) + self.disk.device]) + # str(self.disk.size / self.blocksize)]) if rc != 0: raise MountError("Error creating %s filesystem" % (self.fstype,)) subprocess.call(["/sbin/tune2fs", "-c0", "-i0", "-Odir_index", - "-ouser_xattr,acl", self.lofile]) - - def create(self): - SparseLoopbackMount.create(self) - self.__format_filesystem() + "-ouser_xattr,acl", self.disk.device]) - def resize(self, size = None): - current_size = os.stat(self.lofile)[stat.ST_SIZE] + def __resize_filesystem(self, size = None): + current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] if size is None: size = self.size @@ -227,21 +290,31 @@ class SparseExtLoopbackMount(SparseLoopbackMount): self.__fsck() - resize2fs(self.lofile, size) - - if size < current_size: - self.truncate(size) + resize2fs(self.disk.lofile, size) return size - def mount(self): - if not os.path.isfile(self.lofile): - self.create() + def __create(self): + print "Wibble" + resize = False + if not self.disk.fixed() and self.disk.exists(): + resize = True + + #DiskMount.__create(self) + self.disk.create() + + if resize: + print "Fs resie" + self.__resize_filesystem() else: - self.resize() - return SparseLoopbackMount.mount(self) + print "FS format" + self.__format_filesystem() + + def mount(self): + self.__create() + DiskMount.mount(self) def __fsck(self): - subprocess.call(["/sbin/e2fsck", "-f", "-y", self.lofile]) + subprocess.call(["/sbin/e2fsck", "-f", "-y", self.disk.lofile]) def __get_size_from_filesystem(self): def parse_field(output, field): @@ -253,7 +326,7 @@ class SparseExtLoopbackMount(SparseLoopbackMount): dev_null = os.open("/dev/null", os.O_WRONLY) try: - out = subprocess.Popen(['/sbin/dumpe2fs', '-h', self.lofile], + out = subprocess.Popen(['/sbin/dumpe2fs', '-h', self.disk.lofile], stdout = subprocess.PIPE, stderr = dev_null).communicate()[0] finally: @@ -273,7 +346,7 @@ class SparseExtLoopbackMount(SparseLoopbackMount): while top != (bot + 1): t = bot + ((top - bot) / 2) - if not resize2fs(self.lofile, t): + if not resize2fs(self.disk.lofile, t): top = t else: bot = t @@ -281,12 +354,187 @@ class SparseExtLoopbackMount(SparseLoopbackMount): def resparse(self, size = None): self.cleanup() - minsize = self.__resize_to_minimal() + self.disk.truncate(minsize) + return minsize + +class PartitionedMount(Mount): + def __init__(self, disk, mountdir): + Mount.__init__(self, mountdir) + self.disk = disk + self.partitions = [] + self.mapped = False + self.mountOrder = [] + self.unmountOrder = [] + + def add_partition(self, size, mountpoint, fstype = None): + self.partitions.append({'size': size, 'mountpoint': mountpoint, 'fstype': fstype, 'device': None, 'mount': None, 'num': None}) + + def __format_disk(self): + print "Formatting" + rc = subprocess.call(["/sbin/parted", "-s", self.disk.device, "mklabel", "msdos"]) + if rc != 0: + raise MountError("Error writing partition table on %s" % self.disk.device) + + print "Adding parts" + extSize = 0 + num = 1 + for p in self.partitions: + if num > 3: + extSize += p['size'] + num += 1 + + # XXX we should probably work in cylinder units to keep fdisk happier.. + start = 0 + num = 1 + for p in self.partitions: + if num == 4: + print "Added extened part at %d of size %d" %(start, extSize) + rc = subprocess.call(["/sbin/parted", "-s", self.disk.device, "mkpart", "extended", "%dM" % start, "%dM" % (start+extSize)]) + num += 1 + + type = "primary" + if num > 3: + type = "logical" + print "Add %s part at %d of size %d" % (type, start, p['size']) + rc = subprocess.call(["/sbin/parted", "-s", self.disk.device, "mkpart", type, "%dM" % start, "%dM" % (start+p['size'])]) + if rc != 0 and 1 == 0: # XXX disabled because parted always fails to reload part table with loop devices + raise MountError("Error creating partition on %s" % self.disk.device) + start = start + p['size'] + p['num'] = num + num += 1 + + def __map_partitions(self): + if self.mapped: + return + + kpartx = subprocess.Popen(["/sbin/kpartx", "-l", self.disk.device], + stdout=subprocess.PIPE) + + kpartxOutput = kpartx.communicate()[0].split("\n") + # Strip trailing blank + kpartxOutput = kpartxOutput[0:len(kpartxOutput)-1] + print "Run %s" % str(kpartxOutput) + if kpartx.returncode: + raise MountError("Failed to query partition mapping for '%s'" % + self.disk.device) + + if len(kpartxOutput) != len(self.partitions): + raise MountError("Unexpected number of partitions from kpartx: %d != %d", + len(kpartxOutput), len(self.partitions)) + + for i in range(len(kpartxOutput)): + print " Got %s" % (kpartxOutput[i]) + line = kpartxOutput[i] + dev = line.split()[0] + mapperdev = "/dev/mapper/" + dev + loopdev = self.disk.device + dev[-1] + print "Dev %s: %s -> %s" % (dev, loopdev, mapperdev) + self.partitions[i]['device'] = loopdev + + # grub's install wants partitions to be named + # to match their parent device + partition num + # kpartx doesn't work like this, so we add compat + # symlinks to point to /dev/mapper + os.symlink(mapperdev, loopdev) + + print "Mapping" + rc = subprocess.call(["/sbin/kpartx", "-a", self.disk.device]) + if rc != 0: + raise MountError("Failed to map partitions for '%s'" % + self.disk.device) + self.mapped = True + + + def __unmap_partitions(self): + if not self.mapped: + return + + print "Removing compat symlinks" + for p in self.partitions: + if p['device'] != None: + os.unlink(p['device']) + p['device'] = None + + print "Unmapping" + rc = subprocess.call(["/sbin/kpartx", "-d", self.disk.device]) + if rc != 0: + raise MountError("Failed to unmap partitions for '%s'" % + self.disk.device) + + self.mapped = False + + + def __calculate_mountorder(self): + for p in self.partitions: + self.mountOrder.append(p['mountpoint']) + self.unmountOrder.append(p['mountpoint']) + + self.mountOrder.sort() + self.unmountOrder.sort() + self.unmountOrder.reverse() + print str(self.mountOrder) - self.truncate(minsize) + def cleanup(self): + import time + time.sleep(20) + Mount.cleanup(self) + self.__unmap_partitions() + self.disk.cleanup() + + def unmount(self): + for mp in self.unmountOrder: + if mp == 'swap': + continue + p = None + for p1 in self.partitions: + if p1['mountpoint'] == mp: + p = p1 + break + + if p['mount'] != None: + print "Clenaup %s " % p['mountpoint'] + try: + p['mount'].cleanup() + except: + pass + p['mount'] = None + + def mount(self): + self.disk.create() + + self.__format_disk() + self.__map_partitions() + self.__calculate_mountorder() + + for mp in self.mountOrder: + p = None + for p1 in self.partitions: + if p1['mountpoint'] == mp: + p = p1 + break + + if mp == 'swap': + subprocess.call(["/sbin/mkswap", p['device']]) + continue + + print "Submount %s " % p['mountpoint'] + rmmountdir = False + if p['mountpoint'] == "/": + rmmountdir = True + pdisk = ExtDiskMount(RawDisk(p['size'] * 1024 * 1024, p['device']), + self.mountdir + p['mountpoint'], + p['fstype'], + 4096, + p['mountpoint'], + rmmountdir) + pdisk.mount() + p['mount'] = pdisk + + def resparse(self, size = None): + # Can't re-sparse a disk image - too hard + pass - return self.resize(size) class DeviceMapperSnapshot(object): def __init__(self, imgloop, cowloop): @@ -306,8 +554,8 @@ class DeviceMapperSnapshot(object): if self.__created: return - self.imgloop.loopsetup() - self.cowloop.loopsetup() + self.imgloop.create() + self.cowloop.create() self.__name = "imgcreate-%d-%d" % (os.getpid(), random.randint(0, 2**16)) @@ -315,8 +563,8 @@ class DeviceMapperSnapshot(object): size = os.stat(self.imgloop.lofile)[stat.ST_SIZE] table = "0 %d snapshot %s %s p 8" % (size / 512, - self.imgloop.loopdev, - self.cowloop.loopdev) + self.imgloop.device, + self.cowloop.device) args = ["/sbin/dmsetup", "create", self.__name, "--table", table] if subprocess.call(args) != 0: @@ -382,15 +630,14 @@ class DeviceMapperSnapshot(object): # 8) Create a squashfs of the COW # def create_image_minimizer(path, image, minimal_size): - imgloop = LoopbackMount(image, "None") + imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter - cowloop = SparseLoopbackMount(os.path.join(os.path.dirname(path), "osmin"), - None, 64L * 1024L * 1024L) + cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"), + 64L * 1024L * 1024L) snapshot = DeviceMapperSnapshot(imgloop, cowloop) try: - cowloop.create() snapshot.create() resize2fs(snapshot.path, minimal_size) diff --git a/imgcreate/kickstart.py b/imgcreate/kickstart.py index a7e0723..a92f877 100644 --- a/imgcreate/kickstart.py +++ b/imgcreate/kickstart.py @@ -468,6 +468,9 @@ def get_groups(ks, required = []): def get_excluded(ks, required = []): return ks.handler.packages.excludedList + required +def get_partitions(ks, required = []): + return ks.handler.partition.partitions + def ignore_missing(ks): return ks.handler.packages.handleMissing == ksconstants.KS_MISSING_IGNORE diff --git a/imgcreate/live.py b/imgcreate/live.py index bbb17ef..dc2aea9 100644 --- a/imgcreate/live.py +++ b/imgcreate/live.py @@ -130,7 +130,7 @@ class LiveImageCreatorBase(LoopImageCreator): # def __base_on_iso(self, base_on): """helper function to extract ext3 file system from a live CD ISO""" - isoloop = LoopbackMount(base_on, self._mkdtemp()) + isoloop = Mount(LoopbackDisk(base_on), self._mkdtemp()) try: isoloop.mount() @@ -144,10 +144,10 @@ class LiveImageCreatorBase(LoopImageCreator): else: squashimg = isoloop.mountdir + "/LiveOS/squashfs.img" - squashloop = LoopbackMount(squashimg, self._mkdtemp(), "squashfs") + squashloop = Mount(LoopbackDisk(squashimg), self._mkdtemp(), "squashfs") try: - if not os.path.exists(squashloop.lofile): + if not squashloop.disk.exists(): raise CreatorError("'%s' is not a valid live CD ISO : " "squashfs.img doesn't exist" % base_on) diff --git a/livecd-tools.spec b/livecd-tools.spec index a789460..d9118ba 100644 --- a/livecd-tools.spec +++ b/livecd-tools.spec @@ -58,6 +58,7 @@ rm -rf $RPM_BUILD_ROOT %dir %{_datadir}/livecd-tools %{_datadir}/livecd-tools/* %{_bindir}/image-creator +%{_bindir}/disk-creator %dir %{python_sitelib}/imgcreate %{python_sitelib}/imgcreate/*.py %{python_sitelib}/imgcreate/*.pyo