cas | 7 +++---- cas-admin | 23 +++++++++++++++++++++-- cas.conf | 17 ++++++++++++----- cas.sql | 27 ++++++++++----------------- lib/cas/db.py | 15 ++++++++++----- 5 files changed, 56 insertions(+), 33 deletions(-)
New commits: commit f6c4ec270ffedd4d21e9802eee2b26caa4acd433 Author: Adam Stokes adam.stokes@gmail.com Date: Thu Mar 19 21:07:25 2009 -0400
- conf update - sql update to better reflect the relationships involved - focus on cas-admin to incorporate maintenance
diff --git a/cas b/cas index 7303a54..e255e5e 100755 --- a/cas +++ b/cas @@ -40,7 +40,6 @@ config = ConfigParser.ConfigParser() config.read("/etc/cas.conf") WORKDIRECTORY = config.get("settings","workDirectory") DEBUGLEVEL = config.get("settings","debugLevel") -SERVERS = config.get("settings", "servers") SMTPHOST = config.get("settings", "mailServer") # Read database parameters CASDB_CONF = {'dbhost': config.get("database","dbhost"), @@ -143,8 +142,7 @@ class CasApplication(object): parser.add_option("-f","--file", dest="filename", help="Filename") parser.add_option("-e","--email", dest="email", - help="Define email for results (must be valid!)", - action="store") + help="Define email for results") parser.add_option("-m","--modules", dest="kernel_modules", help="Extract associated kernel modules", action="store_true") @@ -241,7 +239,8 @@ class CasApplication(object): # TODO: Randomize server selection # TODO: Verify remote server is reachable casProcessMachine = serverList[debugKernelArch][0] - self.casLog.info("Machine %s found, processing crash output" % (casProcessMachine,)) + self.casLog.info("Machine %s found, processing " \ + "crash output" % (casProcessMachine,)) cmd = os.path.join(self.storagePath,"crash") client = fc.Overlord(casProcessMachine) clientDict = client.command.run(cmd) diff --git a/cas-admin b/cas-admin index 8bd9819..e394c9b 100755 --- a/cas-admin +++ b/cas-admin @@ -23,6 +23,7 @@ import urlparse from cas.core import CoreBase from cas.util import UtilBase, Logging from cas.rpmutils import RPMBase +from cas.db import CasStorage, CasStorageException from subprocess import Popen, PIPE from cas.cas_shutil import rmtree
@@ -33,11 +34,13 @@ if sys.version_info[:2] < (2,4): config = ConfigParser.ConfigParser() config.read("/etc/cas.conf") KERNELS = config.get("settings","kernels") -RPMS = config.get("settings","rpms") RPMFILTER = config.get("settings","rpmFilter") DEBUGS = config.get("settings","debugs") DEBUGLEVEL = config.get("settings","debugLevel") -SERVERS = config.get("settings","servers") + +# read maintenance options +PURGELIMIT = config.get("maintenance","purgeLimit") +AUTOPURGE = config.get("maintenance","autoPurge")
class CasDatabaseHandler(object): def __init__(self, logger): @@ -151,9 +154,15 @@ class CasAdminApplication(object): help="Build CAS DB", action="store_true", default=False) parser.add_option("-s","--server", dest="server_init", help="Build SERVER DB", action="store_true", default=False) + parser.add_option("-p","--purge", dest="purgeData", + help="Purge files", action="store_true", default=False) + parser.add_option("-d","--days", dest="purgeDataDays", + help="Set how many days back to purge data") (self.opts, args) = parser.parse_args() self.buildDB = self.opts.buildDB self.server_init = self.opts.server_init + self.purgeData = self.opts.purgeData + self.purgeDataDays = self.opts.purgeDataDays
def run(self): """ Make sure necessary directories and configuration is setup @@ -166,6 +175,16 @@ class CasAdminApplication(object): if not os.path.isdir(DEBUGS): os.makedirs(DEBUGS)
+ if self.purgeData: + ans = raw_input("You are about to purge data, is this what you " \ + "really want to do? [Y/y/N/n]: ") + if ans=='Y' or ans=='y': + if not self.purgeDataDays: + self.purgeDataDays = PURGELIMIT + self.casLog.info("Beginning Purge going back %s days" % (self.purgeDataDays,)) + else: + sys.exit(0) + if self.buildDB: self.casLog.info("Starting CAS DB instance.") dbHandler = CasDatabaseHandler(self.casLog).run() diff --git a/cas.conf b/cas.conf index 82d2ce8..5b8808a 100644 --- a/cas.conf +++ b/cas.conf @@ -4,9 +4,6 @@ # indefinately recursive kernels=/mnt/kernels
-# Database which houses all kernel found in $kernels -rpms=/var/db/cas/rpms.db - # Compose emacs regular expression for determing what # kernel debug rpms you wish to search for rpmFilter=.*kernel-debuginfo-[0-9].*.rpm @@ -16,21 +13,31 @@ rpmFilter=.*kernel-debuginfo-[0-9].*.rpm # cpio is run against an rpm and extract the debug kernel # for a small amount of time. This could be something like # /tmp if space permitted -debugs=/cores/debugs +debugs=/tmp
# debug level (DEBUG, INFO) debugLevel=DEBUG
# define work directory -workDirectory=/cores/processed +workDirectory=/cores
# Mail server, e.g. mail.example.com # Provides job results via email mailServer=mail.example.com
+[maintenance] +# Time in days to keep jobs and its associated files +purgeLimit=90 + +# Enable if want to automatically check and purge data +# whenever cas-admin is run +autoPurge=False + [database] # database connection info (mysql only) # TODO: perhaps add orm support for multiple backends? +# A database needs to be manually created +# $ mysqladmin -uroot -predhat create $dbname dbname='casdb' dbhost='localhost' dbuser='root' diff --git a/cas.sql b/cas.sql index 5246295..0448425 100644 --- a/cas.sql +++ b/cas.sql @@ -2,52 +2,45 @@ -- cas jobs -- CREATE TABLE jobs ( - id MEDIUMINT NOT NULL AUTO_INCREMENT, + id INTEGER PRIMARY KEY AUTO_INCREMENT, submitter VARCHAR(255), creation TIMESTAMP DEFAULT CURRENT_TIMESTAMP, jobpath TEXT, corefile TEXT, - debug TEXT, - PRIMARY KEY (id) + debug TEXT )
-- -- cas servers -- CREATE TABLE architecture ( - id MEDIUMINT NOT NULL AUTO_INCREMENT, - arch VARCHAR(255), - PRIMARY KEY (id) + id INTEGER PRIMARY KEY AUTO_INCREMENT, + arch VARCHAR(255) )
-- -- one to many relationship with architectures -- CREATE TABLE servers ( - id MEDIUMINT NOT NULL AUTO_INCREMENT, + id INTEGER PRIMARY KEY AUTO_INCREMENT, hostname VARCHAR(255), - arch_id INTEGER, - PRIMARY KEY (id), - FOREIGN KEY (arch_id) + arch_id INTEGER )
-- -- debug rpm filename -- CREATE TABLE debuginforpm ( - id MEDIUMINT NOT NULL AUTO_INCREMENT, - filename VARCHAR(255), - PRIMARY KEY (id) + id INTEGER PRIMARY KEY AUTO_INCREMENT, + filename VARCHAR(255) )
-- -- debug kernels -- CREATE TABLE debugkernels ( - id MEDIUMINT NOT NULL AUTO_INCREMENT, + id INTEGER PRIMARY KEY AUTO_INCREMENT, debugpath TEXT, debugtimestamp TEXT, - debuginforpm_id INTEGER, - PRIMARY KEY (id), - FOREIGN KEY (debuginforpm_id) + debuginforpm_id INTEGER ) diff --git a/lib/cas/db.py b/lib/cas/db.py index 175a3bb..57313d2 100644 --- a/lib/cas/db.py +++ b/lib/cas/db.py @@ -1,5 +1,7 @@ import MySQLdb
+class CasStorageException(Exception): pass + class CasStorage(object): def __init__(self, sql_server, username, password, database): """ setup database connection and return db cursor for @@ -12,11 +14,14 @@ class CasStorage(object):
def connect(self): """ execute connection """ - db_connect = MySQLdb.connect(host=self.sql_server, - user=self.username, - passwd=self.password, - db=self.db) - self.cursor = db_connect.cursor() + try: + db_connect = MySQLdb.connect(host=self.sql_server, + user=self.username, + passwd=self.password, + db=self.db) + self.cursor = db_connect.cursor() + except OperationalError, e: + raise CasStorageException(e) return
def getAllJobs(self):