pungi logger
by Joel Andres Granados
Hi list
Referring to ticket #34 at
https://hosted.fedoraproject.org/projects/pungi/ticket/34 stating that
the logger needed a little work so that it didn't depend on the
gather.py (or at least thats what I understood :) I propose either a
new file (pungiLog.py) located in the pypungi directory or a new
function in the "pungi" file that contains the logging stuff. The log
services would be started somewhere before the line containing "#
Actually do work." of the "pungi" file. The logging root would be
called "pungi" and would be called in each file that logging is needed
with the logging.getlogger("pungi") command.
If "quiet" is specified in the config file the logging will be turned off.
*Diff for the pungi file:*
1. Initializes the logger by calling to the new file.
2. specify quiet value.
3. logging function.
*
Diff for pungi.py file:
*1. use the correct logger.
*
Diff for gather.py file:*
change all the if statements for each logging call.
Files attached...
Comments appreciated.
Regards
--- gather.py-original 2007-05-03 17:01:18.000000000 +0200
+++ gather.py-JG 2007-05-03 17:01:18.000000000 +0200
@@ -16,6 +16,9 @@
import os
import shutil
import sys
+import logging
+
+log = logging.getLogger("pungi")
class Gather(yum.YumBase):
def __init__(self, config, pkglist):
@@ -46,7 +49,6 @@
#self.doSackSetup(arches)
self.doSackSetup(archlist=arches) # work around temp break in yum api
self.doSackFilelistPopulate()
- self.logger = yum.logging.getLogger("yum.verbose.pungi")
self.pkglist = pkglist
self.polist = []
self.srpmlist = []
@@ -56,15 +58,8 @@
self.compsobj.add(self.config.get('default', 'comps'))
def doLoggingSetup(self, debuglevel, errorlevel):
- """Setup the logging facility."""
-
-
- logdir = os.path.join(self.config.get('default', 'destdir'), 'logs')
- if not os.path.exists(logdir):
- os.makedirs(logdir)
- logfile = os.path.join(logdir, '%s.%s.log' % (self.config.get('default', 'flavor'),
- self.config.get('default', 'arch')))
- yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile)
+ # We want to use our own logger.
+ pass
def doFileLogSetup(self, uid, logfile):
# This function overrides a yum function, allowing pungi to control
@@ -76,8 +71,7 @@
transaction info"""
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Checking deps of %s.%s' % (po.name, po.arch))
+ log.info('Checking deps of %s.%s' % (po.name, po.arch))
reqs = po.requires
provs = po.provides
@@ -93,13 +87,12 @@
deps = self.whatProvides(r, f, v).returnPackages()
if deps is None:
- self.logger.warning("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
+ log.warning("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
continue
for dep in deps:
self.tsInfo.addInstall(dep)
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))
+ log.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))
self.resolved_deps[req] = None
@@ -124,7 +117,7 @@
# Check if we have the group
if not self.compsobj.has_group(group):
- self.logger.error("Group %s not found in comps!" % group)
+ log.error("Group %s not found in comps!" % group)
return packages
# Get the group object to work with
@@ -175,22 +168,18 @@
for line in self.pkglist:
line = line.strip()
if line.startswith('#'):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Skipping comment: %s' % line)
+ log.info('Skipping comment: %s' % line)
continue
if line.startswith('@'):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Adding group: %s' % line)
+ log.info('Adding group: %s' % line)
grouplist.append(line.strip('@'))
continue
if line.startswith('-'):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Adding exclude: %s' % line)
+ log.info('Adding exclude: %s' % line)
excludelist.append(line.strip('-'))
continue
else:
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Adding package: %s' % line)
+ log.info('Adding package: %s' % line)
addlist.append(line)
# First remove the excludes
@@ -219,12 +208,10 @@
mysack = yum.packageSack.ListPackageSack(matches)
for match in mysack.returnNewestByNameArch():
self.tsInfo.addInstall(match)
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Found %s.%s' % (match.name, match.arch))
+ log.info('Found %s.%s' % (match.name, match.arch))
for pkg in unmatched:
- if not pkg in matchdict.keys():
- self.logger.warn('Could not find a match for %s' % pkg)
+ log.warn('Could not find a match for %s' % pkg)
if len(self.tsInfo) == 0:
raise yum.Errors.MiscError, 'No packages found to download.'
@@ -262,7 +249,7 @@
for pkg in self.polist:
downloads.append('%s.%s' % (pkg.name, pkg.arch))
downloads.sort()
- self.logger.info("Download list: %s" % downloads)
+ log.info("Download list: %s" % downloads)
# Package location within destdir, name subject to change/config
pkgdir = os.path.join(self.config.get('default', 'destdir'), self.config.get('default', 'version'),
@@ -282,8 +269,7 @@
if (os.path.exists(local) and
str(os.path.getsize(local)) == pkg.packagesize):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info("%s already exists and appears to be complete" % local)
+ log.info("%s already exists and appears to be complete" % local)
target = os.path.join(pkgdir, os.path.basename(remote))
if os.path.exists(target):
os.remove(target) # avoid traceback after interrupted download
@@ -292,8 +278,7 @@
# Disable cache otherwise things won't download
repo.cache = 0
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Downloading %s' % os.path.basename(remote))
+ log.info('Downloading %s' % os.path.basename(remote))
pkg.localpath = local # Hack: to set the localpath to what we want.
# do a little dance for file:// repos...
@@ -348,19 +333,16 @@
local = os.path.join(self.config.get('default', 'cachedir'), local)
if os.path.exists(local) and str(os.path.getsize(local)) == pkg.packagesize:
- if not self.config.has_option('default', 'quiet'):
- self.logger.info("%s already exists and appears to be complete" % local)
+ log.info("%s already exists and appears to be complete" % local)
if os.path.exists(os.path.join(pkgdir, os.path.basename(remote))) and str(os.path.getsize(os.path.join(pkgdir, os.path.basename(remote)))) == pkg.packagesize:
- if not self.config.has_option('default', 'quiet'):
- self.logger.info("%s already exists in tree and appears to be complete" % local)
+ log.info("%s already exists in tree and appears to be complete" % local)
else:
os.link(local, os.path.join(pkgdir, os.path.basename(remote)))
continue
# Disable cache otherwise things won't download
repo.cache = 0
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Downloading %s' % os.path.basename(remote))
+ log.info('Downloading %s' % os.path.basename(remote))
pkg.localpath = local # Hack: to set the localpath to what we want.
# do a little dance for file:// repos...
--- pungi.py-original 2007-05-03 11:01:18.000000000 -0400
+++ pungi.py-JG 2007-05-03 11:17:00.000000000 -0400
@@ -21,7 +21,7 @@
import shutil
import re
-log = logging.getLogger("pypungi.pungi")
+log = logging.getLogger("pungi")
class Pungi:
def __init__(self, config):
--- pungi-original 2007-05-03 11:01:18.000000000 -0400
+++ pungi-JG 2007-05-03 11:47:12.000000000 -0400
@@ -16,6 +16,7 @@
import pypungi.gather
import pypungi.pungi
import yum
+import logging
from ConfigParser import SafeConfigParser
@@ -79,6 +80,16 @@
destdir = config.get('default', 'destdir')
+ if not config.has_option('default', 'quiet'):quiet=False
+ else:quiet=True
+ config.set('default','quiet',str(quiet))
+
+ # initialize logging.
+ initLogger(config.get('default', 'destdir'),
+ '%s.%s.log' % (config.get('default', 'flavor'), config.get('default', 'arch')),
+ quiet=quiet)
+ log = logging.getLogger("pungi")
+
if not os.path.exists(destdir):
try:
os.makedirs(destdir)
@@ -180,4 +190,25 @@
manifestfile.close()
return pkglist
+ def initLogger(path, filename, quiet=True):
+ """Initialize the log stuff so other files can use it.
+
+ path - Is the directory where the log will be located
+ filename - Is the log files name
+ useLogFile - Whether to log to the file or not
+ """
+ logger = logging.getLogger("pungi")
+ if not quiet:
+ logdir = os.path.join(path , 'logs')
+ if not os.path.exists(logdir):
+ os.makedirs(logdir)
+ logfile = os.path.join(logdir, filename)
+
+ fileHandler = logging.FileHandler(logfile)
+ fileHandler.setLevel(logging.DEBUG)
+ logger.addHandler(fileHandler)
+ logger.setLevel(logging.DEBUG)
+ else:
+ logger.disable = 1
+
main()
16 years, 5 months
Pungis work directory.
by Joel Andres Granados
I ran pungi once and created a fedora iso. When I ran it again with the
same config file, and forgot to erase the previous work directory
files, pungi shows a traceback telling me that some file already exists.
I have two proposals for this situation.
1. create a timestamp directory inside the working directory for each
pungi run. The problem with this approach is that the use is left with
a bunch of directories that are named after whatever time.time() spit
out (not very pretty). But alas it does away with the ugly traceback
message.
The diff for the first solution:
--- pungi-0.3.0 2007-04-02 23:17:25.000000000 +0200
+++ pungi-0.3.1-JG 2007-04-13 14:27:22.000000000 +0200
@@ -18,6 +18,7 @@
import yum
from ConfigParser import SafeConfigParser
+from time import time
def main():
# Set some default variables, can be overrided in config file
@@ -76,7 +77,8 @@
if not opts.destdir == "*CONFFILE*":
config.set('default', 'destdir', opts.destdir)
-
+ config.set('default', 'destdir',
os.path.join(config.get('default','destdir'),
+ str(int(time()))))
destdir = config.get('default', 'destdir')
if not os.path.exists(destdir):
2. Just tell the user to erase the things he/she has in the directory.
The diff for the second solution:
--- pungi-0.3.0 2007-04-13 15:03:26.000000000 +0200
+++ pungi-0.3.1-JG 2007-04-13 15:02:22.000000000 +0200
@@ -18,6 +18,7 @@
import yum
from ConfigParser import SafeConfigParser
+from os.path import isdir
def main():
# Set some default variables, can be overrided in config file
@@ -77,7 +78,7 @@
if not opts.destdir == "*CONFFILE*":
config.set('default', 'destdir', opts.destdir)
- destdir = config.get('default', 'destdir')
+ destdir = check_destdir(config.get('default', 'destdir'))
if not os.path.exists(destdir):
try:
@@ -180,4 +181,10 @@
manifestfile.close()
return pkglist
+ def check_destdir(workpath):
+ if isdir(workpath) and len(os.listdir(workpath)) > 0:
+ print >> sys.stderr, "Please erase all items from the
directory %s before continuing." % workpath
+ exit(1)
+ return workpath
+
main()
16 years, 6 months
pungi fails in the createiso stage
by Joel Andres Granados
Hi list:
Been working on pungi quite a bit and while I was testing some stuff it
showed some strange behaviour. In my box it stops in the createiso
stage when `mkisofs` is called. The error messages states that there is
no isolinux directory ( and when I go look for it where its suppose to
be... its not there :)
To make sure that it was not what I had done I downloaded a new tar file
and installed it in my box. To my surprise it showed the same behavior.
I looked at pungi and the anaconda-runtime scripts which I suspect are
the cause of all this. To get a better look at things I stopped pungi
just before executing the buildinstall command and executed it with the
--debug option to see what came out. A bunch of messages popped out ( I
will list just a small portion):
/.../upd-instroot: line 989: /.../fixmtime.py: No such file or directory
/.../upd-instroot: line 989: /.../fixmtime.py: No such file or directory
cat: /.../lang-table*: No such file or directory
/.../upd-instroot: line 1011: cd: /.../locale: No such file or directory
These where the first ones. There are more but I think the root of the
problem is here.
I looked in the anaconda-runtime scripts for the bug with no success.
Finally I changed pungies yum repository to point to a repository that I
had lying around for some time (the repository had
anaconda-11.2.0.57-1.x86_64.rpm and
anaconda-runtime-11.2.0.57-1.x86_64.rpm they were old versions ). It
worked like a charm.
FYI, the repository that I was using for my previous tests was
"mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=rawhide&arch=..."
So I diffed the upd-insroot from the 11.2.0.57-1.x86_64 version with the
11.2.0.66-1.x86_64 one and IMHO found no change that would give the
behavior that I'm experiencing.
Anybody else in this situation???
Comments greatly appreciated
Regards
--
Joel Andres Granados
16 years, 6 months
[PATCH][PUNGI] name of minimal manifest is inconsistent
by Joel Andres Granados
Hi list:
The minimal manifest in the config file appears from pungi-0.3.5-1.fc7
with a hyphen "minimal-manifest". The file itself appears with a period
"/etc/pungi/minimal.manifest". Just change the config file. or the name
of the file :)
Regards.
--- /etc/pungi/pungi.conf 2007-05-17 15:14:27.000000000 +0200
+++ /etc/pungi/pungi.conf.rpmnew 2007-05-23 17:17:49.000000000 +0200
@@ -8,7 +8,7 @@
iso_basename = F ; The first part of the iso file name
bugurl = http://bugzilla.redhat.com ; Used for betanag
comps = /etc/pungi/comps-f7.xml ; Used to define package groupings and
default installs
-manifest = /etc/pungi/minimal.manifest ; Used to determine what to
bring in. Supports Kickstart syntax
+manifest = /etc/pungi/minimal-manifest ; Used to determine what to
bring in. Supports Kickstart syntax
yumconf = /etc/pungi/yum.conf.f7.x86_64 ; Used to determine where to
gather packages from
destdir = /srv/pungi/Fedora ; Top level compose directory, must be clean
cachedir = /srv/pungi/cache ; Cache used for repeat runs
--
Joel Andres Granados
16 years, 6 months
[Patch][Pungi] log file formating
by Joel Andres Granados
Hi list:
I have noticed that the resulting log file contains some strange looking
output. Mainly the output of createrepo and mksquashfs from the
buildinstall call. AFAIK the problem is caused because these two
applications use the '\r' in there outputs. When using the
universal_newlines option the log file improves a little bit.
The diff is attached
regards.
diff --git a/pypungi/pungi.py b/pypungi/pungi.py
index 3f74669..018bb31 100755
--- a/pypungi/pungi.py
+++ b/pypungi/pungi.py
@@ -69,7 +69,7 @@ class Pungi:
log.info("Running %s" % ' '.join(command))
- p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error)
+ p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error, universal_newlines=True)
(out, err) = p1.communicate()
if p1.returncode != 0:
log.error("Got an error from %s" % command[0])
16 years, 6 months
createrepo in packages/n/v/r dir?
by Roland McGrath
It would be nice if koji ran createrepo in .../packages/name/version/release
when a build finishes, and the same in scratch/user/task dirs. It should be
quick and cheap since it's just a few rpms (I'm suggesting one run at the
top covering all arch's). Especially for packages with many subpackages, it
would be real handy to be able to try a build using a one-off .repo file
pointed at http://koji.fedoraproject.org/packages/foo/v/r/ with yum
--enablerepo, where one now downloads all the rpms and uses rpm by hand.
Thanks,
Roland
16 years, 6 months
Extend verbosity to the scripts and apps that pungi uses
by Joel Andres Granados
hi list:
The idea is to allow, via a command line argument, the redirection of
the output of the applications and scripts that pungi calls to the
stdout of pungi. In this way the user sees all that is happening.
including the things that output of the stuff that pungi calls. I
havent tested this extensively and I'm just wondering what you guys
think of the idea.
diffs attached.
Cheers.
--- gather.py-original 2007-05-03 13:51:43.000000000 -0400
+++ gather.py-JG 2007-05-03 13:51:43.000000000 -0400
@@ -16,6 +16,9 @@
import os
import shutil
import sys
+import logging
+
+log = logging.getLogger("pungi")
class Gather(yum.YumBase):
def __init__(self, config, pkglist):
@@ -46,7 +49,6 @@
#self.doSackSetup(arches)
self.doSackSetup(archlist=arches) # work around temp break in yum api
self.doSackFilelistPopulate()
- self.logger = yum.logging.getLogger("yum.verbose.pungi")
self.pkglist = pkglist
self.polist = []
self.srpmlist = []
@@ -56,15 +58,8 @@
self.compsobj.add(self.config.get('default', 'comps'))
def doLoggingSetup(self, debuglevel, errorlevel):
- """Setup the logging facility."""
-
-
- logdir = os.path.join(self.config.get('default', 'destdir'), 'logs')
- if not os.path.exists(logdir):
- os.makedirs(logdir)
- logfile = os.path.join(logdir, '%s.%s.log' % (self.config.get('default', 'flavor'),
- self.config.get('default', 'arch')))
- yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile)
+ # We want to use our own logger.
+ pass
def doFileLogSetup(self, uid, logfile):
# This function overrides a yum function, allowing pungi to control
@@ -76,8 +71,6 @@
transaction info"""
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Checking deps of %s.%s' % (po.name, po.arch))
reqs = po.requires
provs = po.provides
@@ -93,13 +86,12 @@
deps = self.whatProvides(r, f, v).returnPackages()
if deps is None:
- self.logger.warning("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
+ log.warning("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
continue
for dep in deps:
self.tsInfo.addInstall(dep)
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))
+ log.debug('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))
self.resolved_deps[req] = None
@@ -124,7 +116,7 @@
# Check if we have the group
if not self.compsobj.has_group(group):
- self.logger.error("Group %s not found in comps!" % group)
+ log.error("Group %s not found in comps!" % group)
return packages
# Get the group object to work with
@@ -175,22 +167,18 @@
for line in self.pkglist:
line = line.strip()
if line.startswith('#'):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Skipping comment: %s' % line)
+ log.debug('Skipping comment: %s' % line)
continue
if line.startswith('@'):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Adding group: %s' % line)
+ log.debug('Adding group: %s' % line)
grouplist.append(line.strip('@'))
continue
if line.startswith('-'):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Adding exclude: %s' % line)
+ log.debug('Adding exclude: %s' % line)
excludelist.append(line.strip('-'))
continue
else:
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Adding package: %s' % line)
+ log.debug('Adding package: %s' % line)
addlist.append(line)
# First remove the excludes
@@ -219,12 +207,10 @@
mysack = yum.packageSack.ListPackageSack(matches)
for match in mysack.returnNewestByNameArch():
self.tsInfo.addInstall(match)
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Found %s.%s' % (match.name, match.arch))
+ log.debug('Found %s.%s' % (match.name, match.arch))
for pkg in unmatched:
- if not pkg in matchdict.keys():
- self.logger.warn('Could not find a match for %s' % pkg)
+ log.warn('Could not find a match for %s' % pkg)
if len(self.tsInfo) == 0:
raise yum.Errors.MiscError, 'No packages found to download.'
@@ -262,7 +248,7 @@
for pkg in self.polist:
downloads.append('%s.%s' % (pkg.name, pkg.arch))
downloads.sort()
- self.logger.info("Download list: %s" % downloads)
+ log.info("Download list: %s" % downloads)
# Package location within destdir, name subject to change/config
pkgdir = os.path.join(self.config.get('default', 'destdir'), self.config.get('default', 'version'),
@@ -282,8 +268,7 @@
if (os.path.exists(local) and
str(os.path.getsize(local)) == pkg.packagesize):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info("%s already exists and appears to be complete" % local)
+ log.info("%s already exists and appears to be complete" % local)
target = os.path.join(pkgdir, os.path.basename(remote))
if os.path.exists(target):
os.remove(target) # avoid traceback after interrupted download
@@ -292,8 +277,7 @@
# Disable cache otherwise things won't download
repo.cache = 0
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Downloading %s' % os.path.basename(remote))
+ log.info('Downloading %s' % os.path.basename(remote))
pkg.localpath = local # Hack: to set the localpath to what we want.
# do a little dance for file:// repos...
@@ -348,19 +332,16 @@
local = os.path.join(self.config.get('default', 'cachedir'), local)
if os.path.exists(local) and str(os.path.getsize(local)) == pkg.packagesize:
- if not self.config.has_option('default', 'quiet'):
- self.logger.info("%s already exists and appears to be complete" % local)
+ log.info("%s already exists and appears to be complete" % local)
if os.path.exists(os.path.join(pkgdir, os.path.basename(remote))) and str(os.path.getsize(os.path.join(pkgdir, os.path.basename(remote)))) == pkg.packagesize:
- if not self.config.has_option('default', 'quiet'):
- self.logger.info("%s already exists in tree and appears to be complete" % local)
+ log.info("%s already exists in tree and appears to be complete" % local)
else:
os.link(local, os.path.join(pkgdir, os.path.basename(remote)))
continue
# Disable cache otherwise things won't download
repo.cache = 0
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Downloading %s' % os.path.basename(remote))
+ log.info('Downloading %s' % os.path.basename(remote))
pkg.localpath = local # Hack: to set the localpath to what we want.
# do a little dance for file:// repos...
--- pungi.py-original 2007-05-03 13:51:43.000000000 -0400
+++ pungi.py-JG 2007-05-03 14:05:50.000000000 -0400
@@ -21,7 +21,7 @@
import shutil
import re
-log = logging.getLogger("pypungi.pungi")
+log = logging.getLogger("pungi")
class Pungi:
def __init__(self, config):
@@ -69,6 +69,12 @@
log.info("Running %s" % ' '.join(command))
+ #if the verbose flag is set then change the default behaviour of the output and error
+ #change the values only if the devault is set. otherwise leave the value as is.
+ if self.config.get('default','verbosity2') == "True":
+ if output == subprocess.PIPE: output=None
+ if error == subprocess.PIPE: error=None
+
p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error)
(out, err) = p1.communicate()
if p1.returncode != 0:
--- pungi-original 2007-05-03 13:51:43.000000000 -0400
+++ pungi-JG 2007-05-03 13:55:51.000000000 -0400
@@ -16,6 +16,7 @@
import pypungi.gather
import pypungi.pungi
import yum
+import logging
from ConfigParser import SafeConfigParser
@@ -79,6 +80,21 @@
destdir = config.get('default', 'destdir')
+ if not config.has_option('default', 'quiet'):quiet=False
+ else:quiet=True
+ config.set('default','quiet',str(quiet))
+
+ config.set('default','verbosity1',str(opts.verbose))
+
+ config.set('default','verbosity2',str(opts.Verbose))
+
+ # initialize logging.
+ initLogger(config.get('default', 'destdir'),
+ '%s.%s.log' % (config.get('default', 'flavor'), config.get('default', 'arch')),
+ quiet=quiet,
+ useVerboseMode=opts.verbose)
+ log = logging.getLogger("pungi")
+
if not os.path.exists(destdir):
try:
os.makedirs(destdir)
@@ -157,7 +173,10 @@
help="Flag to enable processing the SplitTree stage")
parser.add_option("-I", action="store_true", default=False, dest="do_createiso",
help="Flag to enable processing the CreateISO stage")
-
+ parser.add_option("-v", action="store_true", default=False, dest="verbose",
+ help="Flag to enable verbose mode for pungi specific messages")
+ parser.add_option("-V", action="store_true", default=False, dest="Verbose",
+ help="Flag to enable verbose mode for messages of the applications and scripts that pungi uses")
(opts, args) = parser.parse_args()
if opts.do_gather or opts.do_buildinstall or opts.do_packageorder or opts.do_splittree or opts.do_createiso:
@@ -180,4 +198,31 @@
manifestfile.close()
return pkglist
+ def initLogger(path, filename, quiet=True, useVerboseMode=False):
+ """Initialize the log stuff so other files can use it.
+
+ path - Is the directory where the log will be located
+ filename - Is the log files name
+ useLogFile - Whether to log to the file or not
+ useVerboseMode - Whether to output stuff to pungis stdout
+ """
+ logger = logging.getLogger("pungi")
+ if not quiet:
+ logdir = os.path.join(path , 'logs')
+ if not os.path.exists(logdir):
+ os.makedirs(logdir)
+ logfile = os.path.join(logdir, filename)
+
+ fileHandler = logging.FileHandler(logfile)
+ fileHandler.setLevel(logging.DEBUG)
+ logger.addHandler(fileHandler)
+
+ if useVerboseMode:
+ stdoutHandler = logging.StreamHandler(sys.stdout)
+ stdoutHandler.setLevel(logging.INFO)
+ logger.addHandler(stdoutHandler)
+
+ logger.setLevel(logging.DEBUG)
+ if quiet and not useVerboseMode:logger.disabled=1
+
main()
16 years, 6 months
verbose command option
by Joel Andres Granados
Hi list:
How about a verbose option "-v"?
On the one hand it is very useful for the impatient user that needs some
sort of immediate feedback. on the other hand it could be useful for
other applications that might need to analyze pungis output (like
revisor). The idea is to have the option off by defect and when the
user specifies "-v" at the command line, all the logging information is
showed. With the output I did a very simple progress bar and every
thing seem to work correctly (the progress bar example is not attached,
it works but its not pretty :).
The verbose option contains the logging patch I posted earlier.
The diffs are attached.
--- gather.py-original 2007-05-03 18:35:02.000000000 +0200
+++ gather.py-JG 2007-05-04 17:25:21.000000000 +0200
@@ -16,6 +16,9 @@
import os
import shutil
import sys
+import logging
+
+log = logging.getLogger("pungi")
class Gather(yum.YumBase):
def __init__(self, config, pkglist):
@@ -46,7 +49,6 @@
#self.doSackSetup(arches)
self.doSackSetup(archlist=arches) # work around temp break in yum api
self.doSackFilelistPopulate()
- self.logger = yum.logging.getLogger("yum.verbose.pungi")
self.pkglist = pkglist
self.polist = []
self.srpmlist = []
@@ -56,15 +58,8 @@
self.compsobj.add(self.config.get('default', 'comps'))
def doLoggingSetup(self, debuglevel, errorlevel):
- """Setup the logging facility."""
-
-
- logdir = os.path.join(self.config.get('default', 'destdir'), 'logs')
- if not os.path.exists(logdir):
- os.makedirs(logdir)
- logfile = os.path.join(logdir, '%s.%s.log' % (self.config.get('default', 'flavor'),
- self.config.get('default', 'arch')))
- yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile)
+ # We want to use our own logger.
+ pass
def doFileLogSetup(self, uid, logfile):
# This function overrides a yum function, allowing pungi to control
@@ -76,8 +71,7 @@
transaction info"""
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Checking deps of %s.%s' % (po.name, po.arch))
+ log.info('Checking deps of %s.%s' % (po.name, po.arch))
reqs = po.requires
provs = po.provides
@@ -93,13 +87,12 @@
deps = self.whatProvides(r, f, v).returnPackages()
if deps is None:
- self.logger.warning("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
+ log.warning("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
continue
for dep in deps:
self.tsInfo.addInstall(dep)
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))
+ log.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))
self.resolved_deps[req] = None
@@ -124,7 +117,7 @@
# Check if we have the group
if not self.compsobj.has_group(group):
- self.logger.error("Group %s not found in comps!" % group)
+ log.error("Group %s not found in comps!" % group)
return packages
# Get the group object to work with
@@ -175,22 +168,18 @@
for line in self.pkglist:
line = line.strip()
if line.startswith('#'):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Skipping comment: %s' % line)
+ log.info('Skipping comment: %s' % line)
continue
if line.startswith('@'):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Adding group: %s' % line)
+ log.info('Adding group: %s' % line)
grouplist.append(line.strip('@'))
continue
if line.startswith('-'):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Adding exclude: %s' % line)
+ log.info('Adding exclude: %s' % line)
excludelist.append(line.strip('-'))
continue
else:
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Adding package: %s' % line)
+ log.info('Adding package: %s' % line)
addlist.append(line)
# First remove the excludes
@@ -219,12 +208,10 @@
mysack = yum.packageSack.ListPackageSack(matches)
for match in mysack.returnNewestByNameArch():
self.tsInfo.addInstall(match)
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Found %s.%s' % (match.name, match.arch))
+ log.info('Found %s.%s' % (match.name, match.arch))
for pkg in unmatched:
- if not pkg in matchdict.keys():
- self.logger.warn('Could not find a match for %s' % pkg)
+ log.warn('Could not find a match for %s' % pkg)
if len(self.tsInfo) == 0:
raise yum.Errors.MiscError, 'No packages found to download.'
@@ -262,7 +249,7 @@
for pkg in self.polist:
downloads.append('%s.%s' % (pkg.name, pkg.arch))
downloads.sort()
- self.logger.info("Download list: %s" % downloads)
+ log.info("Download list: %s" % downloads)
# Package location within destdir, name subject to change/config
pkgdir = os.path.join(self.config.get('default', 'destdir'), self.config.get('default', 'version'),
@@ -282,8 +269,7 @@
if (os.path.exists(local) and
str(os.path.getsize(local)) == pkg.packagesize):
- if not self.config.has_option('default', 'quiet'):
- self.logger.info("%s already exists and appears to be complete" % local)
+ log.info("%s already exists and appears to be complete" % local)
target = os.path.join(pkgdir, os.path.basename(remote))
if os.path.exists(target):
os.remove(target) # avoid traceback after interrupted download
@@ -292,8 +278,7 @@
# Disable cache otherwise things won't download
repo.cache = 0
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Downloading %s' % os.path.basename(remote))
+ log.info('Downloading %s' % os.path.basename(remote))
pkg.localpath = local # Hack: to set the localpath to what we want.
# do a little dance for file:// repos...
@@ -348,19 +333,16 @@
local = os.path.join(self.config.get('default', 'cachedir'), local)
if os.path.exists(local) and str(os.path.getsize(local)) == pkg.packagesize:
- if not self.config.has_option('default', 'quiet'):
- self.logger.info("%s already exists and appears to be complete" % local)
+ log.info("%s already exists and appears to be complete" % local)
if os.path.exists(os.path.join(pkgdir, os.path.basename(remote))) and str(os.path.getsize(os.path.join(pkgdir, os.path.basename(remote)))) == pkg.packagesize:
- if not self.config.has_option('default', 'quiet'):
- self.logger.info("%s already exists in tree and appears to be complete" % local)
+ log.info("%s already exists in tree and appears to be complete" % local)
else:
os.link(local, os.path.join(pkgdir, os.path.basename(remote)))
continue
# Disable cache otherwise things won't download
repo.cache = 0
- if not self.config.has_option('default', 'quiet'):
- self.logger.info('Downloading %s' % os.path.basename(remote))
+ log.info('Downloading %s' % os.path.basename(remote))
pkg.localpath = local # Hack: to set the localpath to what we want.
# do a little dance for file:// repos...
--- pungi.py-original 2007-05-03 18:34:48.000000000 +0200
+++ pungi.py-JG 2007-05-04 17:33:30.000000000 +0200
@@ -21,12 +21,16 @@
import shutil
import re
-log = logging.getLogger("pypungi.pungi")
+log = logging.getLogger("pungi")
class Pungi:
def __init__(self, config):
self.config = config
self.prodpath = 'Fedora' # Probably should be defined elsewhere
+ #self.config.set('default', 'destdir', os.path.join(int(time()),
+ # self.config.get('default','destdir')))
+ #import pdb
+ #pdb.set_trace()
self.destdir = self.config.get('default', 'destdir')
self.archdir = os.path.join(self.destdir,
self.config.get('default', 'version'),
--- pungi-original 2007-05-03 18:34:32.000000000 +0200
+++ pungi-JG 2007-05-04 17:36:38.000000000 +0200
@@ -16,6 +16,7 @@
import pypungi.gather
import pypungi.pungi
import yum
+import logging
from ConfigParser import SafeConfigParser
@@ -79,6 +80,19 @@
destdir = config.get('default', 'destdir')
+ if not config.has_option('default', 'quiet'):quiet=False
+ else:quiet=True
+ config.set('default','quiet',str(quiet))
+
+ config.set('default','verbose',str(opts.verbose))
+
+ # initialize logging.
+ initLogger(config.get('default', 'destdir'),
+ '%s.%s.log' % (config.get('default', 'flavor'), config.get('default', 'arch')),
+ quiet=quiet,
+ useVerboseMode= opts.verbose)
+ log = logging.getLogger("pungi")
+
if not os.path.exists(destdir):
try:
os.makedirs(destdir)
@@ -157,6 +171,8 @@
help="Flag to enable processing the SplitTree stage")
parser.add_option("-I", action="store_true", default=False, dest="do_createiso",
help="Flag to enable processing the CreateISO stage")
+ parser.add_option("-v", action="store_true", default=False, dest="verbose",
+ help="Flag to enable verbose mode")
(opts, args) = parser.parse_args()
@@ -180,4 +195,31 @@
manifestfile.close()
return pkglist
+ def initLogger(path, filename, quiet=True, useVerboseMode=False):
+ """Initialize the log stuff so other files can use it.
+
+ path - Is the directory where the log will be located
+ filename - Is the log files name
+ useLogFile - Whether to log to the file or not
+ useVerboseMode - Whether to output stuff to pungis stdout
+ """
+ logger = logging.getLogger("pungi")
+ if not quiet:
+ logdir = os.path.join(path , 'logs')
+ if not os.path.exists(logdir):
+ os.makedirs(logdir)
+ logfile = os.path.join(logdir, filename)
+
+ fileHandler = logging.FileHandler(logfile)
+ fileHandler.setLevel(logging.DEBUG)
+ logger.addHandler(fileHandler)
+
+ if useVerboseMode:
+ stdoutHandler = logging.StreamHandler(sys.stdout)
+ stdoutHandler.setLevel(logging.INFO)
+ logger.addHandler(stdoutHandler)
+
+ logger.setLevel(logging.DEBUG)
+ if quiet and not useVerboseMode:logger.disabled=1
+
main()
16 years, 6 months
configuration file not found
by Joel Andres Granados
Hi list: Its me again.
Bug (I think): Pungi fails with a traceback when the config file is not
correctly specified. It says
It does not find the default section.
Reason: Its a config parser error. The ConfigParser reads the existing
files and ingnores the
not existing ones. In any case if the file does not exist or if
it exists but there is no "default"
section, pungi will fail with a traceback.
IMO catching the mistake before the traceback and telling the user what
is happening is the right thing to do.
I don't know about the message though. Its the best thing that my
little mind could come up with :)
The diff:
@@ -37,7 +37,10 @@
config = SafeConfigParser()
config.read(opts.config)
-
+ if "default" not in config.sections():
+ print ("Check that the file %s exists and that "
+ "has a 'default' section" % opts.config)
+ exit(0)
if not config.has_option('default', 'flavor'):
config.set('default', 'flavor', flavor)
16 years, 6 months
mkisofs VolumeID 32 character limitation
by Joel Andres Granados
Hi list:
still pocking at pungi...
Bug(I think): Pungi fails sometimes with a message that the mkisofs
command failed. On further investigation
of the pungi log it seems that the label given to the iso is to
long. Looked at the mkisofs
man page and found out that it only supports 32 characters in the
iso label. In my configuration
it had a label of 33 "Fedora development x86_64 rescue"
Reason: There is a possibility for the cds iso images label to be Too
long. In the end the user
defines the label because it is constructed from the pungi.conf
variables:Prodcut_name, arch,
version...
Possible fix: make sure to snip off the label before creating the
isoimage. With the patch I'm trying to cut the id to an allowable
size. Although I *do* cut the original name and I don't know what the
effects are in the installations. I did run pungi and it ended with no
errors.
Comments: when the was shortened from the config file pungi executed
successfully. comments greatly appreciated.
The diff:
@@ -356,8 +359,8 @@
extraargs.append(os.path.join('%s-disc%s' %
(self.topdir, disc), "ppc/mac"))
extraargs.append('-V')
- extraargs.append('"%s %s %s Disc %s"' %
(self.config.get('default', 'product_name'),
- self.config.get('default', 'version'),
self.config.get('default', 'arch'), disc))
+ extraargs.append(('"%s %s %s Disc %s"' %
(self.config.get('default', 'product_name'),
+ self.config.get('default', 'version'),
self.config.get('default', 'arch'), disc))[:32])
extraargs.append('-o')
extraargs.append(isofile)
@@ -417,8 +422,8 @@
extraargs.append(os.path.join('%s-disc%s' %
(self.topdir, disc), "ppc/mac"))
extraargs.append('-V')
- extraargs.append('"%s %s %s DVD"' %
(self.config.get('default', 'product_name'),
- self.config.get('default', 'version'),
self.config.get('default', 'arch')))
+ extraargs.append(('"%s %s %s DVD"' %
(self.config.get('default', 'product_name'),
+ self.config.get('default', 'version'),
self.config.get('default', 'arch')))[:32])
extraargs.append('-o')
extraargs.append(isofile)
16 years, 6 months
