All, This series does some more cleanup of the imagefactory, this time centered on FedoraBuilder. The changes again should be no functional change, although as these change actual code paths they require a bit more careful review. Please review and ACK.
Chris Lalancette
Signed-off-by: Chris Lalancette clalance@redhat.com --- imgfac/builders/FedoraBuilder.py | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/imgfac/builders/FedoraBuilder.py b/imgfac/builders/FedoraBuilder.py index 9f559a8..4d6eed1 100644 --- a/imgfac/builders/FedoraBuilder.py +++ b/imgfac/builders/FedoraBuilder.py @@ -21,6 +21,7 @@ import subprocess import os import re import sys +import shutil import guestfs import string import libxml2 @@ -898,8 +899,7 @@ class FedoraBuilder(BaseBuilder): # Copy to staging location # The os-native cp command in Fedora and RHEL does sparse file detection which is good self.log.debug("Copying (%s) to (%s)" % (input_image, staging_image)) - if subprocess.call(["cp", "-f", input_image, staging_image]): - raise ImageFactoryException("Copy of condorcloud image to staging location (%s) failed" % (staging_image)) + shutil.copyfile(input_image, staging_image)
# Retrieve original XML and write it out to the final dir image_xml_base="/condorimage-" + self.new_image_id + ".xml" @@ -916,8 +916,7 @@ class FedoraBuilder(BaseBuilder): # Now move the image file to the final location final_image = storage + image_base self.log.debug("Moving (%s) to (%s)" % (staging_image, final_image)) - if subprocess.call(["mv", "-f", staging_image, final_image]): - raise ImageFactoryException("Move of condorcloud image to final location (%s) failed" % (final_image)) + shutil.move(staging_image, final_image)
metadata = dict(target_image=target_image_id, provider=provider, icicle="none", target_identifier=self.new_image_id) self.warehouse.create_provider_image(self.new_image_id, metadata=metadata)
Signed-off-by: Chris Lalancette clalance@redhat.com --- imgfac/builders/FedoraBuilder.py | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/imgfac/builders/FedoraBuilder.py b/imgfac/builders/FedoraBuilder.py index 4d6eed1..f5c3bcb 100644 --- a/imgfac/builders/FedoraBuilder.py +++ b/imgfac/builders/FedoraBuilder.py @@ -29,6 +29,7 @@ import httplib2 import traceback import pycurl import json +import gzip from cloudservers import CloudServers import ConfigParser import boto.ec2 @@ -1134,22 +1135,21 @@ class FedoraBuilder(BaseBuilder):
# if the image is already here, great, otherwise grab it from the warehouse - input_image_path=self.app_config['imgdir'] + "/" - input_image_name="ec2-image-" + target_image_id + ".dsk" - input_image=input_image_path + input_image_name + input_image_name = "ec2-image-" + target_image_id + ".dsk" + input_image = self.app_config['imgdir'] + "/" + input_image_name
self.retrieve_image(target_image_id, input_image)
input_image_compressed_name = input_image_name + ".gz" - input_image_compressed=input_image + ".gz" + input_image_compressed = input_image + ".gz"
if not os.path.isfile(input_image_compressed): self.log.debug("No compressed version of image file found - compressing now") - f_out = open(input_image_compressed, 'wb') - retcode = subprocess.call(['gzip', '-c', input_image], stdout=f_out) + f_in = open(input_image, 'rb') + f_out = gzip.open(input_image_compressed, 'wb') + f_out.writelines(f_in) f_out.close() - if retcode: - raise ImageFactoryException("Error while compressing image prior to scp") + f_in.close() self.log.debug("Compression complete")
region=provider
Not only is this less lines of code, it is also faster.
Signed-off-by: Chris Lalancette clalance@redhat.com --- imagefactory.spec.in | 2 +- imgfac/builders/FedoraBuilder.py | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-)
diff --git a/imagefactory.spec.in b/imagefactory.spec.in index d025925..6b9a4ea 100644 --- a/imagefactory.spec.in +++ b/imagefactory.spec.in @@ -8,7 +8,7 @@ Group: Development/Libraries BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot Url: http://www.aeolusproject.org/imagefactory.html BuildArch: noarch -Requires: oz +Requires: oz >= 0.2.0 Requires: euca2ools Requires: python-qmf Requires: python-pycurl diff --git a/imgfac/builders/FedoraBuilder.py b/imgfac/builders/FedoraBuilder.py index f5c3bcb..567609d 100644 --- a/imgfac/builders/FedoraBuilder.py +++ b/imgfac/builders/FedoraBuilder.py @@ -173,14 +173,11 @@ class FedoraBuilder(BaseBuilder): self.log.debug("Doing base install via Oz") libvirt_xml = self.guest.install(self.app_config["timeout"]) self.image = self.guest.diskimage - self.log.debug("Base install complete - Doing customization") - self.percent_complete=30 - self.guest.customize(libvirt_xml) - self.log.debug("Customization complete") - self.percent_complete=50 - self.log.debug("Generating ICICLE") - self.output_descriptor = self.guest.generate_icicle(libvirt_xml) - self.log.debug("ICICLE generation complete") + self.log.debug("Base install complete - Doing customization and ICICLE generation") + self.percent_complete = 30 + self.output_descriptor = self.guest.customize_and_generate_icicle(libvirt_xml) + self.log.debug("Customization and ICICLE generation complete") + self.percent_complete = 50 except: self.log_exc() self.guest.cleanup_old_guest()
Disk prefix and disk filesystem type are *always* "xv" and "ext3" for EC2, as far as I can tell. Remove the useless indirection and just hard-code them.
Signed-off-by: Chris Lalancette clalance@redhat.com --- imgfac/builders/FedoraBuilder.py | 23 +++++++---------------- 1 files changed, 7 insertions(+), 16 deletions(-)
diff --git a/imgfac/builders/FedoraBuilder.py b/imgfac/builders/FedoraBuilder.py index 567609d..130357a 100644 --- a/imgfac/builders/FedoraBuilder.py +++ b/imgfac/builders/FedoraBuilder.py @@ -391,14 +391,7 @@ class FedoraBuilder(BaseBuilder):
# BG - Upload one of two templated fstabs # Input - root device name - # PREFIX is xv unless we are dealing with RHEL5, which is interesting - must mean we hack - # the pv kernel for RHEL5 to present PV block devices as sd*. Is this true? - # UPDATE: clalance indicates that this works but is not advisable - # the xv devices are present in RHEL5, work, and should be used - # Filesystem type - ext3 for now everywhere # TODO: Match OS default behavior and/or what is found in the existing image - prefix="xv" - fstype="ext3"
self.log.info("Modifying and uploading fstab") # Make arch conditional @@ -407,8 +400,6 @@ class FedoraBuilder(BaseBuilder): else: tmpl=self.fstab_32bit
- tmpl = string.replace(tmpl, "#DISK_DEVICE_PREFIX#", prefix) - tmpl = string.replace(tmpl, "#FILESYSTEM_TYPE#", fstype) g.write("/etc/fstab", tmpl)
# BG - Enable networking @@ -1540,18 +1531,18 @@ title #TITLE# initrd /boot/#KERNEL_IMAGE_NAME#-#KERNEL_VERSION#.img """
- fstab_32bit="""LABEL=/ / #FILESYSTEM_TYPE# defaults 1 1 -/dev/#DISK_DEVICE_PREFIX#da2 /mnt ext3 defaults 1 2 -/dev/#DISK_DEVICE_PREFIX#da3 swap swap defaults 0 0 + fstab_32bit="""LABEL=/ / ext3 defaults 1 1 +/dev/xvda2 /mnt ext3 defaults 1 2 +/dev/xvda3 swap swap defaults 0 0 none /dev/pts devpts gid=5,mode=620 0 0 none /dev/shm tmpfs defaults 0 0 none /proc proc defaults 0 0 none /sys sysfs defaults 0 0 """
- fstab_64bit="""LABEL=/ / #FILESYSTEM_TYPE# defaults 1 1 -/dev/#DISK_DEVICE_PREFIX#db /mnt ext3 defaults 0 0 -/dev/#DISK_DEVICE_PREFIX#dc /data ext3 defaults 0 0 + fstab_64bit="""LABEL=/ / ext3 defaults 1 1 +/dev/xvdb /mnt ext3 defaults 0 0 +/dev/xvdc /data ext3 defaults 0 0 none /dev/pts devpts gid=5,mode=620 0 0 none /dev/shm tmpfs defaults 0 0 none /proc proc defaults 0 0 @@ -1559,7 +1550,7 @@ none /sys sysfs defaults 0 0 """
# Dont attempt to be clever with ephemeral devices - leave it to users - fstab_generic="""LABEL=/ / #FILESYSTEM_TYPE# defaults 1 1 + fstab_generic="""LABEL=/ / ext3 defaults 1 1 none /dev/pts devpts gid=5,mode=620 0 0 none /dev/shm tmpfs defaults 0 0 none /proc proc defaults 0 0
Signed-off-by: Chris Lalancette clalance@redhat.com --- imgfac/builders/FedoraBuilder.py | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/imgfac/builders/FedoraBuilder.py b/imgfac/builders/FedoraBuilder.py index 130357a..d187695 100644 --- a/imgfac/builders/FedoraBuilder.py +++ b/imgfac/builders/FedoraBuilder.py @@ -1027,7 +1027,7 @@ class FedoraBuilder(BaseBuilder): _usernodes = ctxt.xpathEval("//provider_credentials/%s_credentials/username" % (self.target)) if len(_usernodes) > 0: self.username = _usernodes[0].content - + self.password = None _passnodes = ctxt.xpathEval("//provider_credentials/%s_credentials/password" % (self.target)) if len(_passnodes) > 0: @@ -1575,22 +1575,22 @@ none /sys sysfs defaults 0 0 # v0.6 of F14 and F15 - dropped F13 for now - also include official public RHEL hourly AMIs for RHEL6 ec2_jeos_amis={ 'ec2-us-east-1': {'Fedora': { '14' : { 'x86_64': 'ami-5b1dd932', 'i386': 'ami-171dd97e' }, - '15' : { 'x86_64': 'ami-c31cd8aa', 'i386': 'ami-b71cd8de' } + '15' : { 'x86_64': 'ami-c31cd8aa', 'i386': 'ami-b71cd8de' } } , 'RHEL-6': { '0' : { 'x86_64': 'ami-80c937e9', 'i386': 'ami-ceb841a7' }, '1' : { 'x86_64': 'ami-5e837b37', 'i386': 'ami-0cbb4265' } } } , 'ec2-us-west-1': {'Fedora': { '14' : { 'x86_64': 'ami-c9693a8c', 'i386': 'ami-c7693a82' }, - '15' : { 'x86_64': 'ami-45a9fb00', 'i386': 'ami-65a9fb20' } + '15' : { 'x86_64': 'ami-45a9fb00', 'i386': 'ami-65a9fb20' } } , 'RHEL-6': { '0' : { 'x86_64': 'ami-a5c695e0', 'i386': 'ami-05e7b440' }, - '1' : { 'x86_64': 'ami-592d7f1c', 'i386': 'ami-dfe7b49a' } - } + '1' : { 'x86_64': 'ami-592d7f1c', 'i386': 'ami-dfe7b49a' } + } # TODO: RHEL5 snapshots fail because of block device mapping issues - fix this # , # 'RHEL-5': { 'U5' : { 'x86_64': 'ami-6fc99a2a', 'i386': 'ami-89e0b3cc' }, # 'U6' : { 'x86_64': 'ami-edc596a8', 'i386': 'ami-73e7b436' } # } - } + } }
aeolus-devel@lists.fedorahosted.org