Extend verbosity to the scripts and apps that pungi uses

[Joel Andres Granados]

hi list:

The idea is to allow, via a command line argument, the redirection of 
the output of the applications and scripts that pungi calls to the 
stdout of pungi.  In this way the user sees all that is happening.  
including the things that output of the stuff that pungi calls.  I 
havent tested this extensively and I'm just wondering what you guys 
think of the idea.
diffs attached.

Cheers.

--- gather.py-original	2007-05-03 13:51:43.000000000 -0400
+++ gather.py-JG	2007-05-03 13:51:43.000000000 -0400
@@ -16,6 +16,9 @@
 import os
 import shutil
 import sys
+import logging
+
+log = logging.getLogger("pungi")

class Gather(yum.YumBase):
     def __init__(self, config, pkglist):
@@ -46,7 +49,6 @@
         #self.doSackSetup(arches)
         self.doSackSetup(archlist=arches) # work around temp break in yum api
         self.doSackFilelistPopulate()
-        self.logger = yum.logging.getLogger("yum.verbose.pungi")
         self.pkglist = pkglist
         self.polist = []
         self.srpmlist = []
@@ -56,15 +58,8 @@
         self.compsobj.add(self.config.get('default', 'comps'))

def doLoggingSetup(self, debuglevel, errorlevel):
-        """Setup the logging facility."""
-
-
-        logdir = os.path.join(self.config.get('default', 'destdir'), 'logs')
-        if not os.path.exists(logdir):
-            os.makedirs(logdir)
-        logfile = os.path.join(logdir, '%s.%s.log' % (self.config.get('default', 'flavor'),
-                                                      self.config.get('default', 'arch')))
-        yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile)
+        # We want to use our own logger.
+        pass

def doFileLogSetup(self, uid, logfile):
         # This function overrides a yum function, allowing pungi to control
@@ -76,8 +71,6 @@
            transaction info"""

-        if not self.config.has_option('default', 'quiet'):
-            self.logger.info('Checking deps of %s.%s' % (po.name, po.arch))

reqs = po.requires
         provs = po.provides
@@ -93,13 +86,12 @@

deps = self.whatProvides(r, f, v).returnPackages()
             if deps is None:
-                self.logger.warning("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
+                log.warning("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
                 continue

for dep in deps:
                 self.tsInfo.addInstall(dep)
-                if not self.config.has_option('default', 'quiet'):
-                    self.logger.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))
+                log.debug('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))

self.resolved_deps[req] = None

@@ -124,7 +116,7 @@

# Check if we have the group
         if not self.compsobj.has_group(group):
-            self.logger.error("Group %s not found in comps!" % group)
+            log.error("Group %s not found in comps!" % group)
             return packages

# Get the group object to work with
@@ -175,22 +167,18 @@
         for line in self.pkglist:
             line = line.strip()
             if line.startswith('#'):
-                if not self.config.has_option('default', 'quiet'):
-                    self.logger.info('Skipping comment: %s' % line)
+                log.debug('Skipping comment: %s' % line)
                 continue
             if line.startswith('@'):
-                if not self.config.has_option('default', 'quiet'):
-                    self.logger.info('Adding group: %s' % line)
+                log.debug('Adding group: %s' % line)
                 grouplist.append(line.strip('@'))
                 continue
             if line.startswith('-'):
-                if not self.config.has_option('default', 'quiet'):
-                    self.logger.info('Adding exclude: %s' % line)
+                log.debug('Adding exclude: %s' % line)
                 excludelist.append(line.strip('-'))
                 continue
             else:
-                if not self.config.has_option('default', 'quiet'):
-                    self.logger.info('Adding package: %s' % line)
+                log.debug('Adding package: %s' % line)
                 addlist.append(line)

# First remove the excludes
@@ -219,12 +207,10 @@
         mysack = yum.packageSack.ListPackageSack(matches)
         for match in mysack.returnNewestByNameArch():
             self.tsInfo.addInstall(match)
-            if not self.config.has_option('default', 'quiet'):
-                self.logger.info('Found %s.%s' % (match.name, match.arch))
+            log.debug('Found %s.%s' % (match.name, match.arch))

for pkg in unmatched:
-            if not pkg in matchdict.keys():
-                self.logger.warn('Could not find a match for %s' % pkg)
+            log.warn('Could not find a match for %s' % pkg)

if len(self.tsInfo) == 0:
             raise yum.Errors.MiscError, 'No packages found to download.'
@@ -262,7 +248,7 @@
             for pkg in self.polist:
                 downloads.append('%s.%s' % (pkg.name, pkg.arch))
                 downloads.sort()
-            self.logger.info("Download list: %s" % downloads)
+            log.info("Download list: %s" % downloads)

# Package location within destdir, name subject to change/config
         pkgdir = os.path.join(self.config.get('default', 'destdir'), self.config.get('default', 'version'), 
@@ -282,8 +268,7 @@
             if (os.path.exists(local) and
                 str(os.path.getsize(local)) == pkg.packagesize):

-                if not self.config.has_option('default', 'quiet'):
-                    self.logger.info("%s already exists and appears to be complete" % local)
+                log.info("%s already exists and appears to be complete" % local)
                 target = os.path.join(pkgdir, os.path.basename(remote))
                 if os.path.exists(target):
                     os.remove(target) # avoid traceback after interrupted download
@@ -292,8 +277,7 @@

# Disable cache otherwise things won't download
             repo.cache = 0
-            if not self.config.has_option('default', 'quiet'):
-                self.logger.info('Downloading %s' % os.path.basename(remote))
+            log.info('Downloading %s' % os.path.basename(remote))
             pkg.localpath = local # Hack: to set the localpath to what we want.

# do a little dance for file:// repos...
@@ -348,19 +332,16 @@
             local = os.path.join(self.config.get('default', 'cachedir'), local)
             if os.path.exists(local) and str(os.path.getsize(local)) == pkg.packagesize:

-                if not self.config.has_option('default', 'quiet'):
-                    self.logger.info("%s already exists and appears to be complete" % local)
+                log.info("%s already exists and appears to be complete" % local)
                 if os.path.exists(os.path.join(pkgdir, os.path.basename(remote))) and str(os.path.getsize(os.path.join(pkgdir, os.path.basename(remote)))) == pkg.packagesize:
-                    if not self.config.has_option('default', 'quiet'):
-                        self.logger.info("%s already exists in tree and appears to be complete" % local)
+                    log.info("%s already exists in tree and appears to be complete" % local)
                 else:
                     os.link(local, os.path.join(pkgdir, os.path.basename(remote)))
                 continue

# Disable cache otherwise things won't download
             repo.cache = 0
-            if not self.config.has_option('default', 'quiet'):
-                self.logger.info('Downloading %s' % os.path.basename(remote))
+            log.info('Downloading %s' % os.path.basename(remote))
             pkg.localpath = local # Hack: to set the localpath to what we want.

# do a little dance for file:// repos...

--- pungi.py-original	2007-05-03 13:51:43.000000000 -0400
+++ pungi.py-JG	2007-05-03 14:05:50.000000000 -0400
@@ -21,7 +21,7 @@
 import shutil
 import re

-log = logging.getLogger("pypungi.pungi")
+log = logging.getLogger("pungi")

class Pungi:
     def __init__(self, config):
@@ -69,6 +69,12 @@

log.info("Running %s" % ' '.join(command))

+        #if the verbose flag is set then change the default behaviour of the output and error
+        #change the values only if the devault is set. otherwise leave the value as is.
+        if self.config.get('default','verbosity2') == "True":
+            if output == subprocess.PIPE: output=None
+            if error == subprocess.PIPE: error=None
+ 
         p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error)
         (out, err) = p1.communicate()
         if p1.returncode != 0:

--- pungi-original	2007-05-03 13:51:43.000000000 -0400
+++ pungi-JG	2007-05-03 13:55:51.000000000 -0400
@@ -16,6 +16,7 @@
 import pypungi.gather
 import pypungi.pungi
 import yum
+import logging

from ConfigParser import SafeConfigParser

@@ -79,6 +80,21 @@

destdir = config.get('default', 'destdir')

+    if not config.has_option('default', 'quiet'):quiet=False
+    else:quiet=True
+    config.set('default','quiet',str(quiet))
+    
+    config.set('default','verbosity1',str(opts.verbose))
+    
+    config.set('default','verbosity2',str(opts.Verbose))
+
+    # initialize logging. 
+    initLogger(config.get('default', 'destdir'),
+               '%s.%s.log' % (config.get('default', 'flavor'), config.get('default', 'arch')),
+               quiet=quiet,
+               useVerboseMode=opts.verbose)
+    log = logging.getLogger("pungi")
+
     if not os.path.exists(destdir):
         try:
             os.makedirs(destdir)
@@ -157,7 +173,10 @@
           help="Flag to enable processing the SplitTree stage")
         parser.add_option("-I", action="store_true", default=False, dest="do_createiso",
           help="Flag to enable processing the CreateISO stage")
-
+        parser.add_option("-v", action="store_true", default=False, dest="verbose",
+          help="Flag to enable verbose mode for pungi specific messages")
+        parser.add_option("-V", action="store_true", default=False, dest="Verbose",
+          help="Flag to enable verbose mode for messages of the applications and scripts that pungi uses")

(opts, args) = parser.parse_args()
         if opts.do_gather or opts.do_buildinstall or opts.do_packageorder or opts.do_splittree or opts.do_createiso:
@@ -180,4 +198,31 @@
         manifestfile.close()
         return pkglist

+    def initLogger(path, filename, quiet=True, useVerboseMode=False):
+        """Initialize the log stuff so other files can use it.
+    
+        path - Is the directory where the log will be located
+        filename - Is the log files name
+        useLogFile - Whether to log to the file or not
+        useVerboseMode - Whether to output stuff to pungis stdout
+        """
+        logger = logging.getLogger("pungi")
+        if not quiet:
+            logdir = os.path.join(path , 'logs')
+            if not os.path.exists(logdir):
+                os.makedirs(logdir)
+            logfile = os.path.join(logdir, filename)
+    
+            fileHandler = logging.FileHandler(logfile)
+            fileHandler.setLevel(logging.DEBUG)
+            logger.addHandler(fileHandler) 
+        
+        if useVerboseMode:
+            stdoutHandler = logging.StreamHandler(sys.stdout)
+            stdoutHandler.setLevel(logging.INFO)
+            logger.addHandler(stdoutHandler)
+        
+        logger.setLevel(logging.DEBUG)
+        if quiet and not useVerboseMode:logger.disabled=1
+
     main()

[Jesse Keating]

On Friday 04 May 2007 12:17:57 Joel Andres Granados wrote:
> The idea is to allow, via a command line argument, the redirection of
> the output of the applications and scripts that pungi calls to the
> stdout of pungi.  In this way the user sees all that is happening.  
> including the things that output of the stuff that pungi calls.  I
> havent tested this extensively and I'm just wondering what you guys
> think of the idea.
> diffs attached.

Like my other example, the idea would be that a verbose option to pungi would 
essentially echo what it puts in the log to the CLI at the same time.  Would 
that satisfy this desire?