[Patch][Pungi] To add a generic way of adding files and directories to the tree.

[Joel Andres Granados]

Hi list:

Been working on a general way to add directories and files to the pungi 
tree.  This patch allows what I think is the best way to do it.  It 
allows the user to add whatever directory or file to the tree by means 
of an additional configuration file :( something I did not want to do, 
but saw no other choice) and packages (the packages are the ones that 
contain the data).
I tested the code for the inclusion of files and directories into the 
tree.  All my tests were successful (the script added the files and 
directories that I specified).  I also tried changing the location of 
the Release notes to a docs directory.  The pungi modifications 
performed as expected but I had to change anaconda so it could read from 
the new tree.
The patches are attached (pungi's and anaconda's)

Comments greatly appreciated.
Regards

-- 
Joel Andres Granados


diff -ur --unidirectional-new-file -x '.hg*' pungi/config/extras.conf
pungi-0.3.8/config/extras.conf
--- pungi/config/extras.conf	1970-01-01 01:00:00.000000000 +0100
+++ pungi-0.3.8/config/extras.conf	2007-06-14 13:16:17.000000000 +0200
@@ -0,0 +1,14 @@
+[/]
+filere = eula.txt GPL ^RPM-GPG
+rpms = fedora-release
+commonfiles=yes
+
+[/docs/relnotes]
+filere = fedora.css ^RELEASE-NOTES
+dirre = stylesheet-images
+rpms = fedora-release-notes
+commonfiles=yes
+
+[/docs/isoburning]
+filere = ^README-BURNING-ISO
+rpms = fedora-release-notes
diff -ur --unidirectional-new-file -x '.hg*' pungi/config/f7-everything.i386
pungi-0.3.8/config/f7-everything.i386
--- pungi/config/f7-everything.i386	2007-06-07 12:38:20.000000000 +0200
+++ pungi-0.3.8/config/f7-everything.i386	2007-06-18 10:03:09.000000000 +0200
@@ -17,3 +17,4 @@
 discs = 2
 cdsize = 4608.0
 getsource = no
+extras = /etc/pungi/extras.conf
diff -ur --unidirectional-new-file -x '.hg*' pungi/config/f7-everything.ppc
pungi-0.3.8/config/f7-everything.ppc
--- pungi/config/f7-everything.ppc	2007-06-07 12:38:20.000000000 +0200
+++ pungi-0.3.8/config/f7-everything.ppc	2007-06-18 10:03:09.000000000 +0200
@@ -17,3 +17,4 @@
 discs = 2
 cdsize = 4608.0
 getsource = no
+extras = /etc/pungi/extras.conf
diff -ur --unidirectional-new-file -x '.hg*' pungi/config/f7-everything.x86_64
pungi-0.3.8/config/f7-everything.x86_64
--- pungi/config/f7-everything.x86_64	2007-06-07 12:38:20.000000000 +0200
+++ pungi-0.3.8/config/f7-everything.x86_64	2007-06-18 10:03:09.000000000 +0200
@@ -17,3 +17,4 @@
 discs = 2
 cdsize = 4608.0
 getsource = no
+extras = /etc/pungi/extras.conf
diff -ur --unidirectional-new-file -x '.hg*' pungi/config/f7-fedora.i386
pungi-0.3.8/config/f7-fedora.i386
--- pungi/config/f7-fedora.i386	2007-06-07 12:38:20.000000000 +0200
+++ pungi-0.3.8/config/f7-fedora.i386	2007-06-18 10:03:09.000000000 +0200
@@ -17,3 +17,4 @@
 discs = 1
 # cdsize = 4608.0
 getsource = no
+extras = /etc/pungi/extras.conf
diff -ur --unidirectional-new-file -x '.hg*' pungi/config/f7-fedora.ppc
pungi-0.3.8/config/f7-fedora.ppc
--- pungi/config/f7-fedora.ppc	2007-06-07 12:38:20.000000000 +0200
+++ pungi-0.3.8/config/f7-fedora.ppc	2007-06-18 10:03:09.000000000 +0200
@@ -17,3 +17,4 @@
 discs = 1
 # cdsize = 4608.0
 getsource = no
+extras = /etc/pungi/extras.conf
diff -ur --unidirectional-new-file -x '.hg*' pungi/config/f7-fedora.x86_64
pungi-0.3.8/config/f7-fedora.x86_64
--- pungi/config/f7-fedora.x86_64	2007-06-07 12:38:20.000000000 +0200
+++ pungi-0.3.8/config/f7-fedora.x86_64	2007-06-18 10:03:31.000000000 +0200
@@ -15,5 +15,6 @@
 version = 7
 flavor = Fedora
 discs = 1
+extras = /etc/pungi/extras.conf
 #cdsize = 4608.0
 getsource = no
diff -ur --unidirectional-new-file -x '.hg*' pungi/pungi pungi-0.3.8/pungi
--- pungi/pungi	2007-06-11 18:34:03.000000000 +0200
+++ pungi-0.3.8/pungi	2007-06-18 09:57:20.000000000 +0200
@@ -29,9 +29,6 @@
     debugdir = "debug"
     isodir = "iso"
     cdsize = "685.0"
-    relnotefilere = "eula.txt fedora.css GPL README-BURNING-ISOS-en_US.txt
RELEASE-NOTES-en_US.html ^RPM-GPG"
-    relnotedirre = "images stylesheet-images"
-    relnotepkgs = "fedora-release fedora-release-notes"
     
     (opts, args) = get_arguments()
 
@@ -42,6 +39,11 @@
         print ("Check that the file %s exists and that it has a 'default'
section" % opts.config)
         sys.exit(1)
 
+    # Initialize the extra dir variable
+    config.extras = SafeConfigParser()#A config parser with no sections.
+    if config.has_option('default', 'extras'):
+        config.extras.read(config.get('default', 'extras'))#Fill in the
sections.
+
     if not config.has_option('default', 'flavor'):
         config.set('default', 'flavor', flavor)
 
@@ -60,22 +62,7 @@
     if not config.has_option('default', 'cdsize'):
         config.set('default', 'cdsize', cdsize)
 
-    if not config.has_option('default', 'relnotefilere'):
-        config.set('default', 'relnotefilere', relnotefilere)
-    else:
-        config.set('default', 'relnotefilere',
config.get('default', 'relnotefilere') + " " + relnotefilere)
-
-    if not config.has_option('default', 'relnotedirre'):
-        config.set('default', 'relnotedirre', relnotedirre)
-    else:
-        config.set('default', 'relnotedirre',
config.get('default', 'relnotedirre') + " " +  relnotedirre)
-
-    if not config.has_option('default', 'relnotepkgs'):
-        config.set('default', 'relnotepkgs', relnotepkgs)
-    else:
-        config.set('default', 'relnotepkgs',
config.get('default', 'relnotepkgs') + " " + relnotepkgs)
-
-    # set some other defaults
+   # set some other defaults
     if not config.has_option('default', 'product_path'):
         config.set('default', 'product_path',
config.get('default', 'product_name'))
 
@@ -119,7 +106,7 @@
 
         if opts.do_all or opts.do_buildinstall:
            mypungi.doBuildinstall()
-           mypungi.doGetRelnotes()
+           mypungi.doGetExtras()
 
         if opts.do_all or opts.do_packageorder:
            mypungi.doPackageorder()
diff -ur --unidirectional-new-file -x '.hg*' pungi/pypungi/gather.py
pungi-0.3.8/pypungi/gather.py
--- pungi/pypungi/gather.py	2007-06-07 12:38:20.000000000 +0200
+++ pungi-0.3.8/pypungi/gather.py	2007-06-14 12:52:32.000000000 +0200
@@ -218,6 +218,10 @@
                     self.logger.info('Adding package: %s' % line)
                 addlist.append(line)
 
+        # append pkg in the extra config file.
+        for section in self.config.extras.sections():
+            addlist.extend(self.config.extras.get(section, 'rpms').split())      
     
+
         # First remove the excludes
         self.conf.exclude.extend(excludelist)
         self.excludePackages()
diff -ur --unidirectional-new-file -x '.hg*' pungi/pypungi/pungi.py
pungi-0.3.8/pypungi/pungi.py
--- pungi/pypungi/pungi.py	2007-06-14 14:35:11.000000000 +0200
+++ pungi-0.3.8/pypungi/pungi.py	2007-06-15 16:22:18.000000000 +0200
@@ -146,65 +146,80 @@
         self._doRunCommand(pkgorder, output=pkgorderfile)
         pkgorderfile.close()
 
-    def doGetRelnotes(self):
-        """Get extra files from packages in the tree to put in the topdir
of
-           the tree."""
-
-        docsdir = os.path.join(self.workdir, 'docs')
-        relnoterpms = self.config.get('default', 'relnotepkgs').split()
-
-        fileres = []
-        for pattern in self.config.get('default',
'relnotefilere').split():
-            fileres.append(re.compile(pattern))
-
-        dirres = []
-        for pattern in self.config.get('default',
'relnotedirre').split():
-            dirres.append(re.compile(pattern))
-        
-        os.makedirs(docsdir)
+    def doGetExtras(self):
+        """ Introduce additional files and directories."""
 
-        # Expload the packages we list as relnote packages
+        if not hasattr( self.config, 'extras' ):
+            return #Because we don't have anything to add.
+        
+        # Complete list of the packages.
         pkgs = os.listdir(os.path.join(self.topdir, self.config.get('default',
'product_path')))
-
-        rpm2cpio = ['/usr/bin/rpm2cpio']
-        cpio = ['cpio', '-imud']
-
-        for pkg in pkgs:
-            pkgname = pkg.rsplit('-', 2)[0]
-            for relnoterpm in relnoterpms:
-                if pkgname == relnoterpm:
-                    import pdb
-                    pdb.set_trace()
-                    extraargs = [os.path.join(self.topdir,
self.config.get('default', 'product_path'), pkg)]
+        tempdir = os.path.join(self.workdir, 'temp')
+        os.makedirs(tempdir)
+        cpiodpkg = [] #The packages that have already been passed through the rpm2cpio |
cpio pipe
+        for section in self.config.extras.sections():
+
+            # Initialize some variables.
+            rpms = self.config.extras.get(section, 'rpms').split()
+            rootdir = os.path.normpath(os.path.join(self.topdir,
section.strip('/')))
+           
+            if not os.path.isdir(rootdir): os.makedirs(rootdir)#Why create it if its
already created?
+
+            fileres = []
+            if self.config.extras.has_option(section, 'filere'):
+                for pattern in self.config.extras.get(section,
'filere').split():
+                    fileres.append(re.compile(pattern))
+
+            dirres = []
+            if self.config.extras.has_option(section, 'dirre'):
+                for pattern in self.config.extras.get(section, 'dirre').split():
+                    dirres.append(re.compile(pattern))
+    
+            #Dump everything in temp. 
+            rpm2cpio = ['/usr/bin/rpm2cpio']
+            cpio = ['cpio', '-imud']
+    
+            for pkg in pkgs:
+                pkgname = pkg.rsplit('-', 2)[0]
+                for rpm in rpms:
+                    # Check if the rpm has already been cpioed
                     try:
-                        p1 = subprocess.Popen(rpm2cpio + extraargs, cwd=docsdir,
stdout=subprocess.PIPE)
-                        (out, err) = subprocess.Popen(cpio, cwd=docsdir, stdin=p1.stdout,
stdout=subprocess.PIPE, 
-                            stderr=subprocess.PIPE,
universal_newlines=True).communicate()
+                        cpiodpkg.index(rpm)
+                        continue
                     except:
-                        log.error("Got an error from rpm2cpio")
-                        log.error(err)
-                        raise
-
-                    log.info(out)
-
-        # Walk the tree for our files
-        for dirpath, dirname, filelist in os.walk(docsdir):
-            for filename in filelist:
-                for regex in fileres:
-                    if regex.match(filename) and not
os.path.exists(os.path.join(self.topdir, filename)):
-                        log.info("Copying release note file %s" % filename)
-                        shutil.copy(os.path.join(dirpath, filename),
os.path.join(self.topdir, filename))
-                        self.common_files.append(filename)
-
-        # Walk the tree for our dirs
-        for dirpath, dirname, filelist in os.walk(docsdir):
-            for directory in dirname:
-                for regex in dirres:
-                    if regex.match(directory) and not
os.path.exists(os.path.join(self.topdir, directory)):
-                        log.info("Copying release note dir %s" % directory)
-                        shutil.copytree(os.path.join(dirpath, directory),
os.path.join(self.topdir, directory))
-        
-
+                        pass
+                    #if the package hasnt been cpioed, do it.
+                    if pkgname == rpm:
+                        extraargs = [os.path.join(self.topdir,
self.config.get('default', 'product_path'), pkg)]
+                        try:
+                            p1 = subprocess.Popen(rpm2cpio + extraargs, cwd=tempdir,
stdout=subprocess.PIPE)
+                            (out, err) = subprocess.Popen(cpio, cwd=tempdir,
stdin=p1.stdout, stdout=subprocess.PIPE, 
+                                stderr=subprocess.PIPE,
universal_newlines=True).communicate()
+                        except:
+                            log.error("Got an error from rpm2cpio")
+                            log.error(err)
+                            raise
+    
+                        log.info(out)
+    
+            # Walk the tree for our files
+            for dirpath, dirname, filelist in os.walk(tempdir):
+                for filename in filelist:
+                    for regex in fileres:
+                        if regex.match(filename) and not
os.path.exists(os.path.join(rootdir, filename)):
+                            log.info("Copying extra file %s" % filename)
+                            shutil.copy(os.path.join(dirpath, filename),
os.path.join(rootdir, filename))
+                            if self.config.extras.has_option(section,
'commonfiles'):#The user decides if the files go in all the images.
+                                self.common_files.append(filename)
+    
+            # Walk the tree for our dirs
+            for dirpath, dirname, filelist in os.walk(tempdir):
+                for directory in dirname:
+                    for regex in dirres:
+                        if regex.match(directory) and not
os.path.exists(os.path.join(rootdir, directory)):
+                            log.info("Copying extra dir %s" % directory)
+                            shutil.copytree(os.path.join(dirpath, directory),
os.path.join(rootdir, directory))
+ 
     def doSplittree(self):
         """Use anaconda-runtime's splittree to split the tree into
appropriate
            sized chunks."""

--- temp/release_notes.py	2007-06-15 18:30:41.000000000 +0200
+++ /mnt/WP/temp/anaconda-11.2.0.66/iw/release_notes.py	2007-04-24 22:51:00.000000000
+0200
@@ -61,9 +61,8 @@
 
 		for suffix in suffixList:
 			fn = "RELEASE-NOTES%s" % (suffix,)
-			fn = os.path.join("docs/relnotes/",fn)
 			try:
-				tmpfile = os.path.abspath(self.anaconda.dispatch.method.getFilename(fn))
+				tmpfile = os.path.abspath(self.anaconda.dispatch.method.getFilename(fn,
destdir="/tmp", retry=0))
 				if tmpfile is None:
 					continue