cas October 2008

cas@lists.fedorahosted.org

2 participants
16 discussions

cas lib/cas version
by Adam Stokes 30 Oct '08

30 Oct '08

cas | 11 ++++++++--- lib/cas/utilities.py | 26 +++++++++++++++----------- version | 2 +- 3 files changed, 24 insertions(+), 15 deletions(-) New commits: commit 7bb15d20e43a4f54280321940b608c0431e52b39 Author: Adam Stokes <adam(a)conans.battleaxe> Date: Thu Oct 30 12:02:04 2008 -0400 finalized on the downloader, altered cas to accept both localfiles and urlbased files. support for http and ftp only at this time diff --git a/cas b/cas index 95f1e11..1590628 100755 --- a/cas +++ b/cas @@ -7,7 +7,7 @@ import os import ConfigParser import shutil -from cas.utilities import Utilities, CoreTool, dprint, sprint +from cas.utilities import Utilities, CoreTool, dprint, sprint, Download from cas.rpmutils import Tools if sys.version_info[:2] < (2,4): @@ -27,6 +27,8 @@ class CoreHandler(object): self.tool = CoreTool() def run(self): + if (self.filename.startswith("http") or self.filename.startswith("ftp")): + self.filename = Download(self.filename, self.dst).get() if not os.path.isfile(self.filename): print("Unable to find file %s" % (self.filename,)) sys.exit(1) @@ -50,6 +52,7 @@ class TimestampHandler(object): def run(self): rpmDB = self.util.load(RPMS) coreTimestamp = self.tool.timestamp(self.corefile) + dprint(coreTimestamp, DPRINT) if coreTimestamp: for k,v in rpmDB.iteritems(): for coreObj in rpmDB[k]: @@ -59,7 +62,8 @@ class TimestampHandler(object): else: dprint("Cannot process a timestamp from: %s" % (self.corefile,), DPRINT) sys.exit(1) - return False + dprint("Unable to match (%s,%s) with debugKernel" % (self.corefile, coreTimestamp), DPRINT) + sys.exit(1) class CasApplication(object): def __init__(self, args): @@ -90,13 +94,14 @@ class CasApplication(object): if not os.path.isdir(self.storagePath): os.makedirs(self.storagePath) corefile = CoreHandler(self.filename, self.storagePath).run() + dprint(corefile, DPRINT) debuginfo, debugKernel = TimestampHandler(corefile).run() filterString = "*/%s" % (debugKernel,) self.rpmTool.extract(debuginfo, self.storagePath, filter=filterString, return_results=False) - self.tool.buildCrashFile(self.storagePath, vmcore, debugKernel) + self.tool.buildCrashFile(self.storagePath, corefile, debugKernel) print("Job on %s complete and located in %s." % (self.filename,self.storagePath)) return diff --git a/lib/cas/utilities.py b/lib/cas/utilities.py index 074b3c5..937d8c9 100755 --- a/lib/cas/utilities.py +++ b/lib/cas/utilities.py @@ -8,6 +8,7 @@ import sys import shutil import tarfile import logging +import urlparse import urlgrabber.grabber as grabber from subprocess import Popen, PIPE @@ -181,8 +182,8 @@ class Utilities(object): # Build crash input file crashInputPath = os.path.join(dst, file_in) crashInputFH = open(crashInputPath, "w") - crashInFH.write(",".join(crashInputCmds)) - crashInFH.close() + crashInputFH.write("".join(crashInputCmds)) + crashInputFH.close() vmcorePath = os.path.join(dst, vmcore) crashCmd = "#!/bin/sh\ncrash %s %s -i %s -s\n" % (vmcorePath,debug, @@ -210,8 +211,11 @@ class CoreTool(object): raise CoreException("%s : is not a properly compressed tarfile." % (self.filepath,)) else: tar = tarfile.open(self.filepath, "r") - tar.extractall(self.dst) - for root, dirs, files in self.util.directoryList(dst): + # python 2.4 doesn't have extractall method + for tarinfo in tar: + tar.extract(tarfinfo.name, self.dst) + #tar.extractall(self.dst) + for root, dirs, files in self.util.directoryList(self.dst): for file in files: if self.isCorefile(file): return os.path.join(root,file) @@ -245,8 +249,8 @@ class CoreTool(object): raise CoreException("Unable to retrieve timestamp from: %s" % (path,)) class Download(object): -""" borrowed from anaconda's urlinstall method -""" + """ borrowed from anaconda's urlinstall method + """ def __init__(self, url, dst): self.url = url self.dst = dst @@ -255,14 +259,14 @@ class Download(object): self.file = os.path.basename(path) self.output = os.path.join(self.dst, self.file) - def status(self, cur_bytes, tot_bytes): - sys.stdout.write("Downloaded %s of %s" % (cur_bytes, tot_bytes) + "\r") + def status(self, cur_percent): + sys.stdout.write("Downloading %3d%%" % (cur_percent) + "\r") sys.stdout.flush() def get(self): try: url = grabber.urlopen(self.url) - except: grabber.URLGrabError, e: + except grabber.URLGrabError, e: raise IOError (e.errno, e.strerror) # check size @@ -280,9 +284,9 @@ class Download(object): tot = len(buf) while len(buf) > 0: if filesize is not None: - self.status((100*tot)/filesize, filesize) + self.status((100*tot)/filesize) else: - self.status(tot/1024, filesize) + self.status(tot/1024) f.write(buf) buf = url.read(65535) tot += len(buf) diff --git a/version b/version index d5fc183..dd9b839 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.13 70 +0.13 71

1 0

lib/cas
by Adam Stokes 30 Oct '08

30 Oct '08

lib/cas/utilities.py | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) New commits: commit a8da19b400f740cc2e66541fbd3f0bc75b4bfb5e Author: Adam Stokes <adam(a)conans.battleaxe> Date: Thu Oct 30 10:50:54 2008 -0400 download support added diff --git a/lib/cas/utilities.py b/lib/cas/utilities.py index fea79d6..074b3c5 100755 --- a/lib/cas/utilities.py +++ b/lib/cas/utilities.py @@ -8,6 +8,7 @@ import sys import shutil import tarfile import logging +import urlgrabber.grabber as grabber from subprocess import Popen, PIPE @@ -180,7 +181,7 @@ class Utilities(object): # Build crash input file crashInputPath = os.path.join(dst, file_in) crashInputFH = open(crashInputPath, "w") - crashInFH.write(crashInputCmds) + crashInFH.write(",".join(crashInputCmds)) crashInFH.close() vmcorePath = os.path.join(dst, vmcore) @@ -244,12 +245,48 @@ class CoreTool(object): raise CoreException("Unable to retrieve timestamp from: %s" % (path,)) class Download(object): +""" borrowed from anaconda's urlinstall method +""" def __init__(self, url, dst): self.url = url self.dst = dst + (scheme, netloc, path, query, fragid) = urlparse.urlsplit(self.url) + self.file = os.path.basename(path) + self.output = os.path.join(self.dst, self.file) + def status(self, cur_bytes, tot_bytes): - print("Downloaded %s of %s" % (cur_bytes, tot_bytes) + "\r") + sys.stdout.write("Downloaded %s of %s" % (cur_bytes, tot_bytes) + "\r") + sys.stdout.flush() def get(self): - pass + try: + url = grabber.urlopen(self.url) + except: grabber.URLGrabError, e: + raise IOError (e.errno, e.strerror) + + # check size + try: + filesize = int(url.info()["Content-Length"]) + if filesize == 0: + filesize = None + except: + filesize = None + + #write output + f = open(self.output, "w+") + + buf = url.read(65535) + tot = len(buf) + while len(buf) > 0: + if filesize is not None: + self.status((100*tot)/filesize, filesize) + else: + self.status(tot/1024, filesize) + f.write(buf) + buf = url.read(65535) + tot += len(buf) + + f.close() + url.close() + return self.output

1 0

cas lib/cas
by Adam Stokes 30 Oct '08

30 Oct '08

cas | 9 +-------- lib/cas/utilities.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) New commits: commit 78162295c6852f938b03bef73bba02133cb9aa70 Author: Adam Stokes <adam(a)conans.battleaxe> Date: Wed Oct 29 21:10:51 2008 -0400 build out crash file to use the -i switch for a more automated approach. this gets rid of the need for expect, however, i may need to use some sort of distributed control system like func in order to process the cores depending on the architecture type. diff --git a/cas b/cas index 620238c..95f1e11 100755 --- a/cas +++ b/cas @@ -96,14 +96,7 @@ class CasApplication(object): filter=filterString, return_results=False) - crash_cmd = "#!/bin/sh\ncrash %s %s\n" % (os.path.join(self.storagePath, - debugKernel), - corefile) - crash_exe = os.path.join(self.storagePath,"crash") - fh = open(crash_exe,"w") - fh.write(crash_cmd) - fh.close() - self.tool.make_exe(crash_exe) + self.tool.buildCrashFile(self.storagePath, vmcore, debugKernel) print("Job on %s complete and located in %s." % (self.filename,self.storagePath)) return diff --git a/lib/cas/utilities.py b/lib/cas/utilities.py index a9bea61..fea79d6 100755 --- a/lib/cas/utilities.py +++ b/lib/cas/utilities.py @@ -169,6 +169,30 @@ class Utilities(object): return supportArch[k] return False + def buildCrashFile(self, dst, vmcore, debug, file_in="crash.in"): + """ build crash file with predefined debug + commands + """ + crashInputCmds = ['bt -a > bt.out\n','sys > sys.out\n','sys -c > sys-c.out\n', + 'kmem -f > kmem-f.out\n', 'log > log.out\n', + 'mod > mod.out\n', 'exit'] + + # Build crash input file + crashInputPath = os.path.join(dst, file_in) + crashInputFH = open(crashInputPath, "w") + crashInFH.write(crashInputCmds) + crashInFH.close() + + vmcorePath = os.path.join(dst, vmcore) + crashCmd = "#!/bin/sh\ncrash %s %s -i %s -s\n" % (vmcorePath,debug, + crashInputPath) + crashExe = os.path.join(dst,"crash") + fh = open(crashExe,"w") + fh.write(crashCmd) + fh.close() + self.make_exe(crashExe) + return + class CoreException(Exception): pass @@ -218,3 +242,14 @@ class CoreTool(object): if out: return out raise CoreException("Unable to retrieve timestamp from: %s" % (path,)) + +class Download(object): + def __init__(self, url, dst): + self.url = url + self.dst = dst + + def status(self, cur_bytes, tot_bytes): + print("Downloaded %s of %s" % (cur_bytes, tot_bytes) + "\r") + + def get(self): + pass

1 0

cas version
by Adam Stokes 25 Oct '08

25 Oct '08

cas | 3 --- version | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) New commits: commit 7ad38167081db03b39adc4f8a39c6833fdc7d2fc Author: Adam Stokes <adam(a)conans.battleaxe> Date: Fri Oct 24 18:46:24 2008 -0400 working copy of cas complete. next step, add support for processing queues for that automated touch diff --git a/cas b/cas index 5d5d16f..620238c 100755 --- a/cas +++ b/cas @@ -56,9 +56,6 @@ class TimestampHandler(object): debugKernel, timestamp = coreObj if timestamp and coreTimestamp in timestamp: return (k, debugKernel) - else: - dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT) - sys.exit(1) else: dprint("Cannot process a timestamp from: %s" % (self.corefile,), DPRINT) sys.exit(1) diff --git a/version b/version index 2e9ff79..d5fc183 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.13 69 +0.13 70

1 0

cas cas.conf casprint lib/cas version
by Adam Stokes 24 Oct '08

24 Oct '08

cas | 7 ++++--- cas.conf | 14 ++++++++++++-- casprint | 37 +++++++++++++++++++++++-------------- lib/cas/rpmutils.py | 3 +-- version | 2 +- 5 files changed, 41 insertions(+), 22 deletions(-) New commits: commit 588f80a3f10bdaeb5bd3b220d9ca1c294035d62e Author: Adam Stokes <adam(a)conans.battleaxe> Date: Thu Oct 23 19:08:33 2008 -0400 casprint seems to capture all extracted debugs and timestamp. db tree is sorted in the form of RPM '-> [ ( debugKernel, timestamp ) ( debugKernel, timestamp ) ] added support for find's emacs regex for fine tuning kernel-debuginfo searches. cas is about 85% complete with has the ability to detect timestamps within cores, test for proper corefile and extraction. builds out a crash wrapper for ease of use when loading the necessary files for crash to proceed. diff --git a/cas b/cas index 007878c..5d5d16f 100755 --- a/cas +++ b/cas @@ -51,10 +51,11 @@ class TimestampHandler(object): rpmDB = self.util.load(RPMS) coreTimestamp = self.tool.timestamp(self.corefile) if coreTimestamp: - dprint(coreTimestamp, DPRINT) for k,v in rpmDB.iteritems(): - if rpmDB[k].timestamp and coreTimestamp in rpmDB[k].timestamp: - return (k, rpmDB[k].debugKernel) + for coreObj in rpmDB[k]: + debugKernel, timestamp = coreObj + if timestamp and coreTimestamp in timestamp: + return (k, debugKernel) else: dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT) sys.exit(1) diff --git a/cas.conf b/cas.conf index 6de30b4..3d995a5 100644 --- a/cas.conf +++ b/cas.conf @@ -1,19 +1,29 @@ [settings] # Where kernels are stored -# NOTE: this _can_ include symlinked directories, just be careful they are indefinately -# recursive +# NOTE: this _can_ include symlinked directories, just be careful they are +# indefinately recursive kernels=/mnt/kernels + # Database which houses all kernel found in $kernels rpms=/var/db/cas/rpms.db + +# Compose emacs regular expression for determing what +# kernel debug rpms you wish to search for +rpmFilter=.*kernel-debuginfo-[0-9].*\.rpm + # Define where to store necessary debug information # NOTE: This is more than likely going to be temporary storage # so you could essentially set this to /tmp if space permitted debugs=/cores/debugs + # debug print on? (True/False) dprint=True + # define work directory workDirectory=/cores/processed + # database to house cas servers servers=/var/db/cas/servers.db + # define processing jobs db jobs=/var/db/cas/jobs.db diff --git a/casprint b/casprint index 4fcff4e..adf568d 100755 --- a/casprint +++ b/casprint @@ -6,7 +6,6 @@ import ConfigParser import optparse import sys -from cas.db import Core from cas.utilities import Utilities, CoreTool, dprint, sprint from cas.rpmutils import Analyze, Tools from subprocess import Popen, PIPE @@ -20,6 +19,7 @@ config = ConfigParser.ConfigParser() config.read("/etc/cas.conf") KERNELS = config.get("settings","kernels") RPMS = config.get("settings","rpms") +RPMFILTER = config.get("settings","rpmFilter") DEBUGS = config.get("settings","debugs") DPRINT = config.get("settings","dprint") @@ -27,43 +27,52 @@ class CasDatabaseHandler(object): def __init__(self): self.util = Utilities() if os.path.isfile(RPMS): - self.debugList = self.util.load(RPMS) + self.rpmDB = self.util.load(RPMS) else: - self.debugList = {} + self.rpmDB = {} def rpmExist(self, rpm): """ checks existence of rpm in db """ - if self.debugList.has_key(rpm): + if self.rpmDB.has_key(rpm): return True return False def run(self): rpms = [] - cmd = ["find", "-L", KERNELS, "-iname", "kernel-debuginfo*rpm"] + # Uses emacs regex -- see `man find` + cmd = ["find", "-L", KERNELS, "-iregex", RPMFILTER] pipe = Popen(cmd, stdout=PIPE, stderr=PIPE) for line in pipe.stdout: rpms.append(line.strip()) sprint("(found) %-100s" % (os.path.basename(line.strip()),)) totalRpms = len(rpms) count = 0 + """ Build database out in the form of + RPM - [( DebugKernel, Timestamp), + ( DebugKernel2, Timestamp2)] + """ for x in rpms: if not self.rpmExist(x): + self.rpmDB[x] = [] count = count + 1 dst = os.path.join(DEBUGS, str(count)) rpmTool = Tools() - coreObj = Core() sprint("(extracting) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x))) results = rpmTool.extract(x, dst) + # Sort through extracted debug for each type + # e.g. hugemem, PAE, smp, largesmp for item in results: vmlinux = item.strip() stamper = CoreTool() - coreObj.debugKernel = os.path.normpath(vmlinux) - coreObj.timestamp = stamper.timestamp(coreObj.debugKernel) - self.debugList[x] = coreObj - sprint("(timestamp) %-100s" % (coreObj.debugKernel,)) - rmtree(dst) - self.util.save(self.debugList, RPMS) + debugKernel = os.path.normpath(vmlinux) + timestamp = stamper.timestamp(debugKernel) + # Build tuple of each debug type + self.rpmDB[x].append((debugKernel, timestamp)) + sprint("(timestamp) %-100s" % (debugKernel,)) + self.util.save(self.rpmDB, RPMS) + # Cleanup extracted debugs + rmtree(dst) return class CasprintApplication(object): @@ -89,10 +98,10 @@ class CasprintApplication(object): util.mkdir(DEBUGS) if self.opts.buildDB: - print("::: Building CAS DB instance. :::") + print("::: Starting CAS DB instance. :::") dbHandler = CasDatabaseHandler().run() else: - print("Please define -b switch to update the CAS database.") + print("Please define -b switch to initialize/update the CAS database.") raise sys.exit(1) if __name__=="__main__": diff --git a/lib/cas/rpmutils.py b/lib/cas/rpmutils.py index 44c8f9b..7c6d03e 100644 --- a/lib/cas/rpmutils.py +++ b/lib/cas/rpmutils.py @@ -1,4 +1,3 @@ -# rpm.py # rpm utilities import rpm @@ -67,7 +66,7 @@ class Tools(object): stdout=PIPE,stderr=PIPE) out, err = p2.communicate() if return_results: - tmp = err.split("\n")[:1] + tmp = err.splitlines()[:-1] for item in tmp: self.filter_results.append(item) return self.filter_results diff --git a/version b/version index d4d75d0..2e9ff79 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.13 65 +0.13 69

1 0

cas lib/cas version
by Adam Stokes 23 Oct '08

23 Oct '08

cas | 42 +++++++++++++++++++++++------------------- lib/cas/utilities.py | 8 ++++---- version | 2 +- 3 files changed, 28 insertions(+), 24 deletions(-) New commits: commit bf425bd75095b2d12ea91f16990d2f74a2997437 Author: Adam Stokes <adam(a)conans.battleaxe> Date: Thu Oct 23 11:41:30 2008 -0400 rpmutils is failing to capture all extracted kernels from stdout, needs fixing diff --git a/cas b/cas index 88a9074..007878c 100755 --- a/cas +++ b/cas @@ -8,6 +8,7 @@ import ConfigParser import shutil from cas.utilities import Utilities, CoreTool, dprint, sprint +from cas.rpmutils import Tools if sys.version_info[:2] < (2,4): raise SystemExit("Python >= 2.4 required") @@ -36,25 +37,29 @@ class CoreHandler(object): corepath = self.tool.extractCore(self.filename, self.dst) if corepath: return os.path.realpath(corepath) - return False + else: + dprint("Unable to determine corefile", DPRINT) + raise sys.exit(1) class TimestampHandler(object): - def __init__(self, corefile, db): + def __init__(self, corefile): self.corefile = corefile - self.db = db self.util = Utilities() self.tool = CoreTool() def run(self): - rpmDB = self.util.load(self.db) + rpmDB = self.util.load(RPMS) coreTimestamp = self.tool.timestamp(self.corefile) if coreTimestamp: dprint(coreTimestamp, DPRINT) - for k,v in rpmDB: - if coreTimestamp and coreTimestamp in rpmDB[k].coreObj.timestamp: - return rpmDB[k] + for k,v in rpmDB.iteritems(): + if rpmDB[k].timestamp and coreTimestamp in rpmDB[k].timestamp: + return (k, rpmDB[k].debugKernel) + else: + dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT) + sys.exit(1) else: - dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT) + dprint("Cannot process a timestamp from: %s" % (self.corefile,), DPRINT) sys.exit(1) return False @@ -62,6 +67,7 @@ class CasApplication(object): def __init__(self, args): self.parse_options(args) self.tool = Utilities() + self.rpmTool = Tools() def parse_options(self, args): parser = optparse.OptionParser(usage="cas -i ID " @@ -86,23 +92,21 @@ class CasApplication(object): if not os.path.isdir(self.storagePath): os.makedirs(self.storagePath) corefile = CoreHandler(self.filename, self.storagePath).run() - if not corefile: - dprint("Unable to determine corefile : %s" % (corefile,), DPRINT) - sys.exit(1) - coreObj = TimestampHandler(corefile, RPMS).run() - try: - # extract debug kernel to workDirectory - except: - dprint("Unable to perform symlink %s" % (os.path.basename(debug)), DPRINT) - sys.exit(1) + debuginfo, debugKernel = TimestampHandler(corefile).run() + filterString = "*/%s" % (debugKernel,) + self.rpmTool.extract(debuginfo, self.storagePath, + filter=filterString, + return_results=False) - crash_cmd = "/bin/sh\ncrash %s %s" % (symlink_dst, corefile) + crash_cmd = "#!/bin/sh\ncrash %s %s\n" % (os.path.join(self.storagePath, + debugKernel), + corefile) crash_exe = os.path.join(self.storagePath,"crash") fh = open(crash_exe,"w") fh.write(crash_cmd) fh.close() self.tool.make_exe(crash_exe) - dprint("Job on %s complete and located in %s." % (self.filename,self.storagePath), DPRINT) + print("Job on %s complete and located in %s." % (self.filename,self.storagePath)) return if __name__=="__main__": diff --git a/lib/cas/utilities.py b/lib/cas/utilities.py old mode 100644 new mode 100755 index 9b40d59..a9bea61 --- a/lib/cas/utilities.py +++ b/lib/cas/utilities.py @@ -20,7 +20,7 @@ logging.basicConfig(level=logging.DEBUG, def dprint(msg, debug=True): if debug: sys.stderr.write("(debug) %s" % (msg,)) - logging.error(msg) + logging.debug(msg) def sprint(msg): """ function to print status messages @@ -182,7 +182,7 @@ class CoreTool(object): self.dst = dst self.filepath = filepath if not tarfile.is_tarfile(self.filepath): - raise CoreException("%s : is not a properly compressed tarfile." % (self.filepath,)) + raise CoreException("%s : is not a properly compressed tarfile." % (self.filepath,)) else: tar = tarfile.open(self.filepath, "r") tar.extractall(self.dst) @@ -207,10 +207,10 @@ class CoreTool(object): def timestamp(self, path): """ captures fingerprint from core """ - match='Linux\sversion.*\d{1,4}|#1\s.*\d{1,2}' + match='Linux\sversion.*20\d{1,2}|#1\s.*20\d{1,2}' try: fd=open('%s' % (path)) - except IOError: + except IOError: return False fd.seek(0) b = os.read(fd.fileno(),54000000) diff --git a/version b/version index 1312e50..d4d75d0 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.13 64 +0.13 65

1 0

casprint lib/cas version
by Adam Stokes 23 Oct '08

23 Oct '08

casprint | 2 +- lib/cas/rpmutils.py | 2 +- version | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) New commits: commit d554231bd2a365d4f29eb2a7ad9fa132f49475af Author: Adam Stokes <adam(a)conans.battleaxe> Date: Wed Oct 22 22:55:31 2008 -0400 casprint stores proper path to vmlinux within cpio archive incremental saves to rpmdb to help with speed when updating database. diff --git a/casprint b/casprint index bcfb0aa..4fcff4e 100755 --- a/casprint +++ b/casprint @@ -63,7 +63,7 @@ class CasDatabaseHandler(object): self.debugList[x] = coreObj sprint("(timestamp) %-100s" % (coreObj.debugKernel,)) rmtree(dst) - self.util.save(self.debugList, RPMS) + self.util.save(self.debugList, RPMS) return class CasprintApplication(object): diff --git a/lib/cas/rpmutils.py b/lib/cas/rpmutils.py index abd7306..44c8f9b 100644 --- a/lib/cas/rpmutils.py +++ b/lib/cas/rpmutils.py @@ -69,7 +69,7 @@ class Tools(object): if return_results: tmp = err.split("\n")[:1] for item in tmp: - self.filter_results.append(os.path.join(self.dst,os.path.normpath(item))) + self.filter_results.append(item) return self.filter_results return diff --git a/version b/version index 5362fd9..1312e50 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.13 63 +0.13 64

1 0

cas
by Adam Stokes 23 Oct '08

23 Oct '08

cas | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) New commits: commit b2c15170ee3602ce580e685575dff87c6416ec8f Author: Adam Stokes <adam(a)conans.battleaxe> Date: Wed Oct 22 22:54:13 2008 -0400 *** empty log message *** diff --git a/cas b/cas index 2172479..88a9074 100755 --- a/cas +++ b/cas @@ -46,14 +46,13 @@ class TimestampHandler(object): self.tool = CoreTool() def run(self): - debugList = self.util.load(self.db) + rpmDB = self.util.load(self.db) coreTimestamp = self.tool.timestamp(self.corefile) if coreTimestamp: dprint(coreTimestamp, DPRINT) - for item in debugList: - debug, timestamp = item - if coreTimestamp and coreTimestamp in timestamp: - return debug + for k,v in rpmDB: + if coreTimestamp and coreTimestamp in rpmDB[k].coreObj.timestamp: + return rpmDB[k] else: dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT) sys.exit(1) @@ -90,11 +89,9 @@ class CasApplication(object): if not corefile: dprint("Unable to determine corefile : %s" % (corefile,), DPRINT) sys.exit(1) - debug = TimestampHandler(corefile, RPMS).run() + coreObj = TimestampHandler(corefile, RPMS).run() try: - symlink_dst = os.path.join(self.storagePath, - os.path.basename(debug)) - os.symlink(debug,symlink_dst) + # extract debug kernel to workDirectory except: dprint("Unable to perform symlink %s" % (os.path.basename(debug)), DPRINT) sys.exit(1)

1 0

cas cas.conf casprint lib/cas version
by Adam Stokes 22 Oct '08

22 Oct '08

cas | 4 ++-- cas.conf | 7 ++++--- casprint | 16 ++++++++++------ lib/cas/rpmutils.py | 12 +++++++----- version | 2 +- 5 files changed, 24 insertions(+), 17 deletions(-) New commits: commit 0f8ae2fc50a228fbc562c0dba38a394699184314 Author: Adam Stokes <adam(a)conans.battleaxe> Date: Wed Oct 22 12:19:23 2008 -0400 Reworked rpmutils library to return debug kernel results if asked Altered casprint to cleanup after itself, we are going to continue storing all related information in the debuginfo packages for now diff --git a/cas b/cas index 099ea6e..2172479 100755 --- a/cas +++ b/cas @@ -27,11 +27,11 @@ class CoreHandler(object): def run(self): if not os.path.isfile(self.filename): - dprint("Unable to find file specified by -f FILENAME.", DPRINT) + print("Unable to find file %s" % (self.filename,)) sys.exit(1) if self.tool.isCorefile(self.filename): shutil.move(self.filename, - os.path.join(self.dst,self.filename)) + os.path.join(self.dst, self.filename)) return os.path.realpath(self.filename) corepath = self.tool.extractCore(self.filename, self.dst) if corepath: diff --git a/cas.conf b/cas.conf index 2e558a2..6de30b4 100644 --- a/cas.conf +++ b/cas.conf @@ -5,14 +5,15 @@ kernels=/mnt/kernels # Database which houses all kernel found in $kernels rpms=/var/db/cas/rpms.db -# Database to house all kernel fingperint # Define where to store necessary debug information +# NOTE: This is more than likely going to be temporary storage +# so you could essentially set this to /tmp if space permitted debugs=/cores/debugs # debug print on? (True/False) dprint=True # define work directory workDirectory=/cores/processed # database to house cas servers -servers=/var/db/cas/casservers.db +servers=/var/db/cas/servers.db # define processing jobs db -jobs=/var/db/cas/casjobs.db +jobs=/var/db/cas/jobs.db diff --git a/casprint b/casprint index eca970b..bcfb0aa 100755 --- a/casprint +++ b/casprint @@ -10,6 +10,7 @@ from cas.db import Core from cas.utilities import Utilities, CoreTool, dprint, sprint from cas.rpmutils import Analyze, Tools from subprocess import Popen, PIPE +from shutil import rmtree if sys.version_info[:2] < (2,4): raise SystemExit("Python >= 2.4 required") @@ -51,14 +52,17 @@ class CasDatabaseHandler(object): count = count + 1 dst = os.path.join(DEBUGS, str(count)) rpmTool = Tools() + coreObj = Core() + sprint("(extracting) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x))) results = rpmTool.extract(x, dst) - sprint("(extracted) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x))) for item in results: vmlinux = item.strip() stamper = CoreTool() - timestamp = stamper.timestamp(vmlinux) - self.debugList[x] = (vmlinux, timestamp) - sprint("(timestamp) %-100s" % (os.path.realpath(vmlinux),)) + coreObj.debugKernel = os.path.normpath(vmlinux) + coreObj.timestamp = stamper.timestamp(coreObj.debugKernel) + self.debugList[x] = coreObj + sprint("(timestamp) %-100s" % (coreObj.debugKernel,)) + rmtree(dst) self.util.save(self.debugList, RPMS) return @@ -85,10 +89,10 @@ class CasprintApplication(object): util.mkdir(DEBUGS) if self.opts.buildDB: - dprint("Building CAS DB instance.", DPRINT) + print("::: Building CAS DB instance. :::") dbHandler = CasDatabaseHandler().run() else: - dprint("Please define -b switch to update the CAS database\n", DPRINT) + print("Please define -b switch to update the CAS database.") raise sys.exit(1) if __name__=="__main__": diff --git a/lib/cas/rpmutils.py b/lib/cas/rpmutils.py index 5cecce4..abd7306 100644 --- a/lib/cas/rpmutils.py +++ b/lib/cas/rpmutils.py @@ -53,7 +53,7 @@ class Tools(object): self.filter = filter self.rpm2cpio = tool self.cpio = "/bin/cpio" - self.cpio_args = "-imud" + self.cpio_args = "-imudv" self.filter_results = [] # change to destination directory @@ -63,11 +63,13 @@ class Tools(object): # pipe to handle extraction, e.g. rpm2cpio kernel.rpm | cpio -imud */vmlinux p1 = Popen([self.rpm2cpio, self.rpm], stdout=PIPE) - (out, err) = Popen([self.cpio,self.cpio_args,self.filter], stdin=p1.stdout, - stdout=open(os.devnull, "w"),stderr=PIPE).communicate() + p2 = Popen([self.cpio,self.cpio_args,self.filter], stdin=p1.stdout, + stdout=PIPE,stderr=PIPE) + out, err = p2.communicate() if return_results: - for item in out.stderr.readlines(): - self.filter_results.append(os.path.join(self.dst,item)) + tmp = err.split("\n")[:1] + for item in tmp: + self.filter_results.append(os.path.join(self.dst,os.path.normpath(item))) return self.filter_results return diff --git a/version b/version index 6b6bf85..5362fd9 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.13 61 +0.13 63

1 0

2 commits - casprint lib/cas
by Adam Stokes 22 Oct '08

22 Oct '08

casprint | 68 ++++++++++++++++++++++------------------------------ lib/cas/rpmutils.py | 16 ++++++++---- 2 files changed, 41 insertions(+), 43 deletions(-) New commits: commit ba98f0cfab10a3291f70298ae02d889059e7fa57 Author: Adam Stokes <adam(a)conans.battleaxe> Date: Tue Oct 21 21:35:34 2008 -0400 re-worked casprint diff --git a/lib/cas/rpmutils.py b/lib/cas/rpmutils.py index 23b024b..5cecce4 100644 --- a/lib/cas/rpmutils.py +++ b/lib/cas/rpmutils.py @@ -37,14 +37,15 @@ class Analyze(object): return ".%s" % (i,) else: return hdr["FILENAMES"] - + class Tools(object): """ provide file alteration tools """ def __init__(self): pass - def extract(self, rpm, dst, tool="/usr/bin/rpm2cpio", filter="*/vmlinux"): + def extract(self, rpm, dst, tool="/usr/bin/rpm2cpio", + filter="*/vmlinux", return_results=True): """ extract file(s) from rpm """ self.rpm = rpm @@ -53,15 +54,20 @@ class Tools(object): self.rpm2cpio = tool self.cpio = "/bin/cpio" self.cpio_args = "-imud" + self.filter_results = [] # change to destination directory if not os.path.isdir(self.dst): os.makedirs(self.dst) os.chdir(self.dst) - + # pipe to handle extraction, e.g. rpm2cpio kernel.rpm | cpio -imud */vmlinux p1 = Popen([self.rpm2cpio, self.rpm], stdout=PIPE) - (out, err) = Popen([self.cpio,self.cpio_args,self.filter], stdin=p1.stdout, - stdout=open(os.devnull, "w"),stderr=open(os.devnull,"w")).communicate() + (out, err) = Popen([self.cpio,self.cpio_args,self.filter], stdin=p1.stdout, + stdout=open(os.devnull, "w"),stderr=PIPE).communicate() + if return_results: + for item in out.stderr.readlines(): + self.filter_results.append(os.path.join(self.dst,item)) + return self.filter_results return commit a605acbf91a5c0b2fb123d675605dd6b4a5cab3b Author: Adam Stokes <adam(a)conans.battleaxe> Date: Tue Oct 21 21:34:48 2008 -0400 reworking casprint to not store debugs locally and use the debuginfo rpm packages as main storage. diff --git a/casprint b/casprint index e8656da..eca970b 100755 --- a/casprint +++ b/casprint @@ -22,31 +22,20 @@ RPMS = config.get("settings","rpms") DEBUGS = config.get("settings","debugs") DPRINT = config.get("settings","dprint") -class CasTimeStampHandler(object): - def __init__(self): - self.cmd = ["find", "-L", DEBUGS, "-iname", "vmlinux"] - self.tool = CoreTool() - self.util = Utilities() - self.debugList = [] - - def run(self): - pipe = Popen(self.cmd, stdout=PIPE, stderr=PIPE) - for line in pipe.stdout: - vmlinux = line.strip() - tstamp = CoreTool() - timestamp = tstamp.timestamp(vmlinux) - self.debugList.append((vmlinux,timestamp)) - sprint("(timestamp) %-100s" % (os.path.realpath(vmlinux),)) - self.util.save(self.debugList, RPMS) - return - class CasDatabaseHandler(object): def __init__(self): self.util = Utilities() if os.path.isfile(RPMS): self.debugList = self.util.load(RPMS) else: - self.debugList = [] + self.debugList = {} + + def rpmExist(self, rpm): + """ checks existence of rpm in db + """ + if self.debugList.has_key(rpm): + return True + return False def run(self): rpms = [] @@ -58,11 +47,19 @@ class CasDatabaseHandler(object): totalRpms = len(rpms) count = 0 for x in rpms: - count = count + 1 - dst = os.path.join(DEBUGS, str(count)) - rpmTool = Tools() - rpmTool.extract(x, dst) - sprint("(extracted) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x))) + if not self.rpmExist(x): + count = count + 1 + dst = os.path.join(DEBUGS, str(count)) + rpmTool = Tools() + results = rpmTool.extract(x, dst) + sprint("(extracted) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x))) + for item in results: + vmlinux = item.strip() + stamper = CoreTool() + timestamp = stamper.timestamp(vmlinux) + self.debugList[x] = (vmlinux, timestamp) + sprint("(timestamp) %-100s" % (os.path.realpath(vmlinux),)) + self.util.save(self.debugList, RPMS) return class CasprintApplication(object): @@ -70,11 +67,9 @@ class CasprintApplication(object): self.parse_options(args) def parse_options(self, args): - parser = optparse.OptionParser(usage="casprint -n -t") - parser.add_option("-n","--new", dest="newCasDB", - help="Build new CAS database", action="store_true") - parser.add_option("-t","--timestamp", dest="timestamp", - help="Timestamp existing debug kernels", action="store_true") + parser = optparse.OptionParser(usage="casprint -b") + parser.add_option("-b","--build", dest="buildDB", + help="Build CAS DB", action="store_true") self.opts, args = parser.parse_args() def run(self): @@ -84,20 +79,17 @@ class CasprintApplication(object): util = Utilities() if os.getuid() is not 0: raise RuntimeError, "You must be root(0), instead you are id(%d)" % (os.getuid()) - if not os.path.isdir(os.path.dirname(RPMS)): + if not os.path.isdir(os.path.dirname(RPMS)): util.mkdir(RPMS) - if not os.path.isdir(DEBUGS): + if not os.path.isdir(DEBUGS): util.mkdir(DEBUGS) - if self.opts.newCasDB: - dprint("Running a new cas database instance.", DPRINT) + if self.opts.buildDB: + dprint("Building CAS DB instance.", DPRINT) dbHandler = CasDatabaseHandler().run() - elif self.opts.timestamp: - dprint("Building an updated cache of timestamps on existing debugs.", DPRINT) - timestampHandler = CasTimeStampHandler().run() else: - raise RuntimeError("You must define either -n for a new CAS database or \n" - "-t to update existing debugs") + dprint("Please define -b switch to update the CAS database\n", DPRINT) + raise sys.exit(1) if __name__=="__main__": app = CasprintApplication(sys.argv[1:])

1 0

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

cas October 2008