commit 6f801de1708d10bdd08a67ad9218c75ee1c08300 Author: Konstantin Ryabitsev icon@mricon.com Date: Thu Jan 23 21:42:28 2003 +0000
Big logfile handler rewrite.
epylog | 1 + py/epylog/__init__.py | 154 ++++++----------------- py/epylog/log.py | 337 +++++++++++++++++++++++++++++++++++++++---------- py/epylog/module.py | 110 ++++++++-------- py/epylog/report.py | 96 ++++++++------- 5 files changed, 412 insertions(+), 286 deletions(-) --- diff --git a/epylog b/epylog index 5e2ae37..81f1bfc 100755 --- a/epylog +++ b/epylog @@ -89,6 +89,7 @@ def main(args): logger.puthang(1, 'Restoring log offsets') epylog.restore_log_offsets() logger.endhang(1, 'done') + sys.exit(1) logger.put(1, 'Invoking the module execution routines:') epylog.process_modules() logger.put(1, 'Finished processing modules') diff --git a/py/epylog/__init__.py b/py/epylog/__init__.py index 5c72d0d..27684c4 100644 --- a/py/epylog/__init__.py +++ b/py/epylog/__init__.py @@ -9,7 +9,7 @@ import time
from report import Report from module import Module -from log import LogFile +from log import LogTracker
VERSION = 'Epylog-0.8'
@@ -48,16 +48,15 @@ class SysCallError(exceptions.Exception): logger.put(2, 'Raising SysCallError with message: %s' % message) self.args = message
+class NoSuchLogError(exceptions.Exception): + def __init__(self, message, logger): + logger.put(2, 'Raising NoSuchLogError with message: %s' % message) + self.args = message + class Epylog: def __init__(self, cfgfile, logger): - logger.puthang(2, 'Starting the epylog object initialization') - logger.put(5, 'Sticking logger into the object') self.logger = logger - - logger.put(3, 'Setting some defaults') - self.report = None - self.modules = [] - self.offsets = None + logger.put(5, 'Entering Epylog.__init__')
config = ConfigParser.ConfigParser() logger.puthang(3, 'Reading the config file "%s"' % cfgfile) @@ -71,17 +70,20 @@ class Epylog: self.cfgdir = config.get('main', 'cfgdir') self.vardir = config.get('main', 'vardir') except: - raise ConfigError(('Could not parse the main config file "%s"' - % cfgfile), logger) + msg = 'Could not parse the main config file "%s"' % cfgfile + raise ConfigError(msg, logger) logger.put(4, 'cfgdir=%s' % self.cfgdir) logger.put(4, 'vardir=%s' % self.vardir) logger.endhang(3)
logger.put(3, 'Checking if we can write to vardir') if not os.access(self.vardir, os.W_OK): - raise ConfigError('Write access required for vardir "%s"' - % self.vardir, logger) + msg = 'Write access required for vardir "%s"' % self.vardir + raise ConfigError(msg, logger)
+ ## + # Set up a safe temp dir + # logger.put(3, 'Setting up a temporary directory') try: tmpdir = config.get('main', 'tmpdir') @@ -92,23 +94,38 @@ class Epylog: try: tmpprefix = tempfile.mkdtemp('EPYLOG') except: - raise ConfigError('Could not create a temp directory in "%s"' - % tmpprefix, logger) + msg = 'Could not create a safe temp directory in "%s"' % tmpprefix + raise ConfigError(msg, logger) self.tmpprefix = tmpprefix tempfile.tempdir = tmpprefix logger.put(3, 'Temporary directory created in "%s"' % tmpprefix) logger.put(3, 'Sticking tmpprefix into config to pass to other objs') config.tmpprefix = self.tmpprefix
+ ## + # Initialize the Report object + # logger.puthang(3, 'Initializing the Report') self.report = Report(config, logger) logger.endhang(3)
+ ## + # Initialize the LogTracker object + # + logger.puthang(3, 'Initializing the log tracker object') + logtracker = LogTracker(config, logger) + self.logtracker = logtracker + logger.endhang(3) + + ## + # Process module configurations + # + self.modules = [] modcfgdir = os.path.join(self.cfgdir, 'modules.d') logger.put(3, 'Checking if module config dir "%s" exists' % modcfgdir) if not os.path.isdir(modcfgdir): - raise ConfigError('Module configuration directory "%s" not found' - % modcfgdir, logger) + msg = 'Module configuration directory "%s" not found' % modcfgdir + raise ConfigError(msg, logger) logger.put(3, 'Looking for module configs in %s' % modcfgdir) for file in os.listdir(modcfgdir): cfgfile = os.path.join(modcfgdir, file) @@ -120,116 +137,23 @@ class Epylog: continue logger.put(3, 'Ends in .conf all right.') logger.puthang(3, 'Calling the Module init routines') - module = Module(cfgfile, logger) + module = Module(cfgfile, logtracker, logger) logger.endhang(3) if module.enabled: - logger.put(3, 'Module "%s" is enabled' % module.name) + logger.put(2, 'Module "%s" is enabled' % module.name) logger.put(3, 'Checking "%s" for sanity' % module.name) module.sanity_check() logger.put(3, 'Sanity checks passed. Remembering module') self.modules.append(module) else: - logger.put(3, 'Module "%s" is not enabled, ignoring' + logger.put(2, 'Module "%s" is not enabled, ignoring' % module.name) else: logger.put(3, '%s is not a regular file, ignoring' % cfgfile) logger.put(3, 'Total of %d modules initialized' % len(self.modules)) - logger.put(3, 'Looking at what logfiles to initialize') - logmap = {} - for module in self.modules: - logger.put(3, 'Looking in module "%s"' % module.name) - logger.put(5, module.lognames) - for logname in module.lognames: - logger.put(3, 'Found logfile declaration "%s"' % logname[0]) - if logmap.has_key(logname[0]): - logger.put(3, 'Log file "%s" already initialized' - % logname[0]) - logger.put(3, 'Sticking it into the module') - module.logs.append(logmap[logname[0]]) - else: - logger.put(3, '%s is not yet initialized. Initializing.' - % logname[0]) - try: - logger.puthang(3, 'Calling LogFile init routines') - logobj = LogFile(logname[0], self.tmpprefix, logger) - logger.endhang(3) - except AccessError: - raise ConfigError(('Log file "%s" does not exist or ' + - 'is not accessible for reading') - % logname[0], logger) - logger.put(3, 'Opening the logfile "%s"' % logname[0]) - logobj.initfile() - if logname[1] is not None: - logger.put(3, 'Initializing the rotated logfile "%s"' - % logname[1]) - try: - rotobj = LogFile(logname[1], self.tmpprefix, - logger) - logger.put(3, ('Sticking rotated logfile object' + - ' into the log object')) - logobj.rotated = rotobj - except AccessError: - logger.put(3, 'Error opening. Hmm... Oh well.') - logger.put(3, 'Ignoring rotated logfile "%s"' - % logname[1]) - logger.put(3, 'Sticking the log object into the module') - module.logs.append(logobj) - logger.put(3, 'Copying object into the logs map') - logmap[logname[0]] = logobj - - self.logmap = logmap - if self.report.unparsed: - logger.put(3, 'Creating a temporary file for filtered strings') - filt_fh = open(tempfile.mktemp('FILT'), 'w') - logger.put(3, 'Filtered strings file created in "%s"' - % filt_fh.name) - logger.put(3, 'Sticking filt_fh into the report object') - self.report.filt_fh = filt_fh - logger.endhang(2) - - def restore_log_offsets(self): - logger = self.logger - logger.put(2, 'Invoking the restore_log_offsets routine') - logger.put(2, 'Setting up the iterator') - iter = self.logmap.iteritems() - earliest_stamp = None - latest_stamp = None - while 1: - try: - (logname, logobj) = iter.next() - logger.put(2, 'Processing logname "%s"' % logname) - self.__restore_log_offset(logobj) - start_stamp = logobj.get_offset_start_stamp() - end_stamp = logobj.get_offset_end_stamp() - if earliest_stamp is None: - earliest_stamp = start_stamp - if latest_stamp is None: - latest_stamp = end_stamp - if start_stamp < earliest_stamp: - earliest_stamp = start_stamp - if end_stamp > latest_stamp: - latest_stamp = end_stamp - except StopIteration: - logger.put(2, 'Iteration finished') - break - self.report.start_stamp = earliest_stamp - self.report.end_stamp = latest_stamp - - def store_log_offsets(self): - logger = self.logger - logger.put(2, 'Invoking the store_log_offsets routine') - logger.put(2, 'Setting up the iterator') - iter = self.logmap.iteritems() - while 1: - try: - (logname, logobj) = iter.next() - logger.put(2, 'Processing logname "%s"' % logname) - self.__store_log_offset(logobj) - except StopIteration: - logger.put(2, 'Iteration finished') - break - logger.put(2, 'Calling the pickling routine') - self.__pickle_offsets() + if len(self.modules) == 0: + raise ConfigError('No modules are enabled. Exiting.', logger) + logger.put(5, 'Exiting Epylog.__init__')
def process_modules(self): logger = self.logger diff --git a/py/epylog/log.py b/py/epylog/log.py index fa61a67..89c756c 100644 --- a/py/epylog/log.py +++ b/py/epylog/log.py @@ -4,83 +4,273 @@ import re import string import time
-class LogFile: - - def __init__(self, logfile, tmpprefix, logger): - logger.put(5, 'Entering LogFile.__init__') - logger.put(2, 'Starting LogFile object initialization for logfile "%s"' - % logfile) - logger.put(3, 'Sticking logger into object') +class LogTracker: + def __init__(self, config, logger): + self.logger = logger + logger.put(5, 'Entering LogTracker.__init__') + self.tmpprefix = config.tmpprefix + self.entries = [] + self.logs = [] + logger.put(5, 'Exiting LogTracker.__init__') + + def getlog(self, entry): + logger = self.logger + logger.put(5, 'Entering LogTracker.getlog') + logger.put(5, 'Checking if we have a log for entry "%s"' % entry) + if entry in self.entries: + logger.put(5, 'Yes, returning that log') + log = self.__get_log_by_entry(entry) + else: + logger.put(5, 'Logfile for "%s" not yet initialized' % entry) + log = self.__init_log_by_entry(entry) + logger.put(5, 'Exiting LogTracker.getlog') + return log + + def get_offset_map(self): + logger = self.logger + logger.put(5, 'Entering LogTracker.get_offset_map') + omap = {} + for log in self.logs: + key = log.entry + inode = log.getinode() + if log.orange.endix != 0: + offset = 0 + else: + offset = log.orange.end_offset + omap[key] = [inode, offset] + logger.put(5, 'omap follows') + logger.put(5, omap) + logger.put(5, 'Exiting LogTracker.get_offset_map') + return omap + + def set_start_offset_by_entry(self, entry, inode, offset): + logger = self.logger + logger.put(5, '>LogTracker.set_offset_by_entry') + logger.put(5, 'entry=%s' % entry) + logger.put(5, 'inode=%d' % inode) + logger.put(5, 'offset=%d' % offset) + if entry in self.entries: + log = self.__get_log_by_entry(entry) + if log.getinode() != inode: + logger.put(5, 'Inodes do not match. Assuming logrotation') + try: + log.set_range_param(1, offset, 0) + except epylog.OutOfRangeError: + logger.put(5, 'No rotated file in place. Set offset to 0') + log.set_range_param(0, 0, 0) + else: + logger.put(5, 'Inodes match, setting offset to "%d"' % offset) + log.set_range_param(0, offset, 0) + else: + msg = 'No such log entry "%s"' % entry + raise epylog.NoSuchLogError(msg, logger) + logger.put(5, '<LogTracker.set_offset_by_entry') + + def __init_log_by_entry(self, entry): + logger = self.logger + logger.put(5, 'Entering LogTracker.__init_log_by_entry') + logger.puthang(5, 'Initializing log object for entry "%s"' % entry) + log = Log(entry, self.tmpprefix, self.logger) + logger.endhang(5) + self.entries.append(entry) + self.logs.append(log) + logger.put(5, 'Exiting LogTracker.__init_log_by_entry') + return log + + def __get_log_by_entry(self, entry): + logger = self.logger + logger.put(5, 'Entering LogTracker.__get_log_by_entry') + for log in self.logs: + if log.entry == entry: + logger.put(5, 'Found log object "%s"' % entry) + logger.put(5, 'Exiting LogTracker.__get_log_by_entry') + return log + logger.put(5, 'No such log file! Returning None. How did this happen?') + return None + + +class OffsetRange: + def __init__(self, startix, start_offset, endix, end_offset, logger): self.logger = logger + logger.put(5, 'Entering OffsetRange.__init__') + self.startix = startix + self.endix = endix + self.start_offset = start_offset + self.end_offset = end_offset + logger.put(5, 'startix=%d' % self.startix) + logger.put(5, 'start_offset=%d' % self.start_offset) + logger.put(5, 'endix=%d' % self.endix) + logger.put(5, 'end_offset=%d' % self.end_offset) + logger.put(5, 'Exiting OffsetRange.__init__') + + def setstart(self, ix, offset): + logger = self.logger + logger.put(5, 'Entering OffsetRange.setstart') + self.startix = ix + self.start_offset = offset + logger.put(5, 'new startix=%d' % self.startix) + logger.put(5, 'new start_offset=%d' % self.start_offset) + logger.put(5, 'Exiting OffsetRange.setstart')
+ def setend(self, ix, offset): + logger = self.logger + logger.put(5, 'Entering OffsetRange.setend') + self.endix = ix + self.end_offset = offset + logger.put(5, 'new endix=%d' % self.endix) + logger.put(5, 'new end_offset=%d' % self.end_offset) + logger.put(5, 'Exiting OffsetRange.setend') + +class Log: + def __init__(self, entry, tmpprefix, logger): + logger.put(5, 'Entering Log.__init__') + logger.puthang(3, 'Initializing Log object for entry "%s"' % entry) + self.logger = logger self.tmpprefix = tmpprefix - logger.put(3, 'Setting some defaults') - self.fh = None - self.rotated = None - self.monthmap = None - self.log_start_stamp = None - self.log_end_stamp = None - self.start_offset = 0 - self.end_offset = None - self.log_end_offset = None + self.entry = entry + filename = self.__get_filename() + logger.puthang(4, 'Initializing the logfile "%s"' % filename) + logfile = LogFile(filename, tmpprefix, logger) + logger.endhang(4) + logger.put(3, 'Appending logfile to the loglist') + self.loglist = [logfile] + self.orange = OffsetRange(0, 0, 0, logfile.end_offset, logger) + logger.endhang(3) + logger.put(5, 'Exiting Log.__init__') + + def set_range_param(self, ix, offset, whence=0): + logger = self.logger + logger.put(5, 'Entering Log.set_range_param') + logger.put(5, 'ix=%d' % ix) + logger.put(5, 'offset=%d' % offset) + logger.put(5, 'whence=%d' % whence) + try: + while not self.__is_valid_ix(ix): + self.__init_next_logfile() + except epylog.NoSuchLogError: + msg = 'Invalid index "%d" for log entry "%s"' % (ix, self.entry) + raise epylog.OutOfRangeError(msg, logger) + if whence: + logger.put(5, 'Setting range END for entry "%s"' % self.entry) + self.orange.setend(ix, offset) + else: + logger.put(5, 'Setting range START for entry "%s"' % self.entry) + self.orange.setstart(ix, offset) + logger.put(5, 'Exiting Log.set_range_param') + + def __is_valid_ix(self, ix): + logger = self.logger + logger.put(5, 'Entering Log.__is_valid_ix') + ixlen = len(self.loglist) - 1 + isvalid = 1 + if ix > ixlen: + logger.put(5, 'index %d is not valid' % ix) + isvalid = 0 + logger.put(5, 'Exiting Log.__is_valid_ix') + return isvalid + + def __init_next_rotfile(self): + logger = self.logger + logger.put(5, '>Log.__init_next_rotfile') + ix = len(self.loglist) + rotname = self.__get_rotname_by_ix(ix) + try: + logger.put(5, 'Initializing log for rotated file "%s"' % rotname) + rotlog = LogFile(rotname, self.tmpprefix, logger) + except epylog.AccessError: + msg = 'No further rotated files for entry "%s"' % self.entry + raise epylog.NoSuchLogError(msg, logger) + self.loglist.append(rotlog) + logger.put(5, '<Log.__init_next_rotfile') + + def __get_rotname_by_ix(self, ix): + logger = self.logger + logger.put(5, '>Log.__get_rotname_by_ix') + logger.put(5, 'ix=%d' % ix) + rotname = re.sub(re.compile('[|]'), '', self.entry) + rotname = re.sub(re.compile('#'), str(ix), rotname) + logger.put(5, 'rotname=%s' % rotname) + logger.put(5, '<Log.__get_rotname_by_ix') + return rotname + + def __get_filename(self): + logger = self.logger + logger.put(5, '>Log.__get_filename') + logger.put(5, 'entry=%s' % self.entry) + filename = re.sub(re.compile('[.*?]'), '', self.entry) + logger.put(5, 'filename=%s' % filename) + logger.put(5, '<Log.__get_filename') + return filename + + def getinode(self): + logger = self.logger + logger.put(5, 'Entering Log.getinode') + logfile = self.loglist[0] + inode = logfile.getinode() + logger.put(5, 'inode=%d' % inode) + logger.put(5, 'Exiting Log.getinode') + return inode
- self.filename = logfile +class LogFile: + def __init__(self, filename, tmpprefix, logger): + self.logger = logger + logger.put(5, 'Entering LogFile.__init__') + self.tmpprefix = tmpprefix + self.filename = filename + ## + # start_stamp: the timestamp at the start of the log + # end_stamp: the timestamp at the end of the log + # end_offset: this is where the end of the log is + # + self.start_stamp = None + self.end_stamp = None + self.end_offset = None + + logger.put(3, 'Running sanity checks on the logfile') self.__accesscheck() - logger.put(2, 'All checks passed') - self.inode = os.stat(logfile).st_ino - logger.put(3, 'inode=%d' % self.inode) - logger.put(2, 'Finished LogFile object initialization for "%s"' - % logfile) + logger.put(3, 'All checks passed') + logger.put(5, 'Initializing the file') + self.__initfile() logger.put(5, 'Exiting LogFile.__init__')
- def initfile(self): + def __initfile(self): logger = self.logger - logger.put(5, 'Entering LogFile.initfile') - logger.put(2, 'Checking if we are already initialized') - if self.fh is None: - logger.put(2, 'Not inited yet, initing') - logger.put(2, 'Checking if we are gzipped (ends in .gz)') - if re.compile('.gz$').search(self.filename, 1): - logger.put(2, 'Ends in .gz. Using GzipFile to open') - import gzip - import epylog.mytempfile as tempfile - tempfile.tmpdir = self.tmpprefix - ungzfile = tempfile.mktemp('UNGZ') - logger.put(3, 'Creating a tempfile in "%s"' % ungzfile) - ungzfh = open(tempfile.mktemp('UNGZ'), 'w+') - try: - gzfh = gzip.open(self.filename) - except: - raise epylog.ConfigError(('Could not open file "%s" with' - + ' gzip handler. Not gzipped?') - % self.filename, logger) - logger.put(2, 'Putting the contents of the gzlog into ungzlog') - while 1: - chunk = gzfh.read(1024) - if chunk: - ungzfh.write(chunk) - logger.put(5, 'Read "%s" bytes from gzfh' % len(chunk)) - else: - logger.put(5, 'Reached EOF') - break - gzfh.close() - self.fh = ungzfh - else: - logger.put(2, 'Does not end in .gz, assuming plain text') - logger.put(2, 'Opening logfile "%s"' % self.filename) - self.fh = open(self.filename) - logger.put(2, 'Finding the end offset') - self.fh.seek(0, 2) - self.__set_at_line_start() - self.end_offset = self.fh.tell() - self.log_end_offset = self.fh.tell() - if self.end_offset == 0: - logger.put(2, 'This logfile is empty!') - logger.put(2, 'log_end_offset=%d' % self.log_end_offset) + logger.put(5, 'Entering LogFile.__initfile') + logger.put(5, 'Checking if we are gzipped (ends in .gz)') + if re.compile('.gz$').search(self.filename, 1): + logger.put(5, 'Ends in .gz. Using GzipFile to open') + import gzip + import epylog.mytempfile as tempfile + tempfile.tmpdir = self.tmpprefix + ungzfile = tempfile.mktemp('UNGZ') + logger.put(5, 'Creating a tempfile in "%s"' % ungzfile) + ungzfh = open(tempfile.mktemp('UNGZ'), 'w+') + try: + gzfh = gzip.open(self.filename) + except: + raise epylog.ConfigError(('Could not open file "%s" with' + + ' gzip handler. Not gzipped?') + % self.filename, logger) + logger.put(5, 'Putting the contents of the gzlog into ungzlog') + while 1: + chunk = gzfh.read(1024) + if chunk: + ungzfh.write(chunk) + logger.put(5, 'Read "%s" bytes from gzfh' % len(chunk)) + else: + logger.put(5, 'Reached EOF') + break + gzfh.close() + self.fh = ungzfh else: - logger.put(2, 'Already initialized, ignoring') - pass - logger.put(5, 'Exiting LogFile.initfile') + logger.put(5, 'Does not end in .gz, assuming plain text') + logger.put(5, 'Opening logfile "%s"' % self.filename) + self.fh = open(self.filename) + logger.put(5, 'Finding the end offset') + self.fh.seek(0, 2) + self.__set_at_line_start() + self.end_offset = self.fh.tell() + logger.put(5, 'Exiting LogFile.__initfile')
def set_init_offset(self): logger = self.logger @@ -139,6 +329,13 @@ class LogFile: return self.get_log_end_stamp() else: return self.get_stamp_at_offset(self.end_offset) + + def getinode(self): + self.logger.put(5, 'Entering LogFile.getinode') + inode = os.stat(self.filename).st_ino + self.logger.put(5, 'inode=%d' % inode) + self.logger.put(5, 'Exiting LogFile.getinode') + return inode
def find_offset_by_timestamp(self, searchstamp): logger = self.logger @@ -417,13 +614,13 @@ class LogFile: else: logger.put(2, 'Path "%s" does not exist' % logfile) raise epylog.AccessError('Log file "%s" does not exist' - % logfile, logger) + % logfile, logger) if os.access(logfile, os.R_OK): logger.put(2, 'File "%s" is readable' % logfile) else: logger.put(2, 'Logfile "%s" is not readable' % logfile) raise epylog.AccessError('Logfile "%s" is not readable' - % logfile, logger) + % logfile, logger) logger.put(5, 'Exiting LogFile.__accesscheck')
def __set_at_line_start(self): diff --git a/py/epylog/module.py b/py/epylog/module.py index 13e447e..f9a1f61 100644 --- a/py/epylog/module.py +++ b/py/epylog/module.py @@ -1,93 +1,88 @@ import ConfigParser import epylog import os -import os.path import mytempfile as tempfile
class Module: """epylog Module class"""
- def __init__(self, cfgfile, logger): - self.lognames = [] - self.logs = [] + def __init__(self, cfgfile, logtracker, logger): + self.logger = logger + logger.put(5, 'Entering Module.__init__') self.logreport = None self.logfilter = None logger.put(2, 'Initializing module for cfgfile %s' % cfgfile) - logger.put(3, 'Sticking logger into object') - self.logger = logger config = ConfigParser.ConfigParser() logger.put(2, 'Reading in the cfgfile %s' % cfgfile) config.read(cfgfile) try: - self.name = config.get('module', 'name') + self.name = config.get('module', 'desc') except: self.name = 'Unnamed Module' - logger.put(2, 'name=%s' % self.name) - try: - self.executable = config.get('module', 'exec') - except: - raise epylog.ConfigError('Did not find executable name in %s' - % cfgfile, logger) - logger.put(2, 'executable=%s' % self.executable) try: self.enabled = config.getboolean('module', 'enabled') except: self.enabled = 0 - logger.put(2, 'enabled=%d' % self.enabled) + if not self.enabled: + logger.put(2, 'This module is not enabled. Skipping init.') + return + try: + self.executable = config.get('module', 'exec') + except: + raise epylog.ConfigError('Did not find executable name in %s' + % cfgfile, logger) try: self.python = config.getboolean('module', 'python') except: self.python = 0 - logger.put(2, 'python=%d' % self.python) try: - logs = config.get('logs', 'files') + logentries = config.get('logs', 'files') except: raise epylog.ConfigError(('Cannot find log definitions in ' + - 'module config "%s"') % cfgfile) + 'module config "%s"') % cfgfile) try: - rots = config.get('logs', 'rotated') + rotdir = config.get('logs', 'rotdir') except: - rots = 0 - logger.put(3, 'logs=%s' % logs) - logger.put(3, 'rots=%s' % rots) - loglist = logs.split(',') - if rots: - rotlist = rots.split(',') - if rots and len(loglist) is not len(rotlist): - raise epylog.ConfigError(('%d logs specified, but %d rotated logs' + - ' specifications found in module config' + - ' "%s"') - % (len(loglist), len(rotlist), cfgfile), - logger) - logger.put(5, 'Loglist follows') - logger.put(5, loglist) - logger.put(5, 'self.lognames before the loop') - logger.put(5, self.lognames) - for log in loglist: - log = log.strip() - if rots: - rot = rotlist.pop(0) - rot = rot.strip() - else: - rot = None - logger.put(2, 'tuple: [%s, %s]' % (log, str(rot))) - self.lognames.append([log, rot]) - logger.put(5, 'self.lognames:') - logger.put(5, self.lognames) - + rotdir = None try: self.outhtml = config.getboolean('output', 'html') except: self.outhtml = 0 + + logger.put(5, 'name=%s' % self.name) + logger.put(5, 'executable=%s' % self.executable) + logger.put(5, 'enabled=%d' % self.enabled) + logger.put(5, 'python=%d' % self.python) + logger.put(5, 'logentries=%s' % logentries) logger.put(2, 'outhtml=%d' % self.outhtml) - logger.put(2, 'Finished with Module object initialization') + + logger.put(3, 'Figuring out the logfiles from the log list') + entrylist = logentries.split(',') + self.logs = [] + for entry in entrylist: + entry = entry.strip() + logger.put(5, 'entry=%s' % entry) + logger.put(3, 'Getting a log object from entry "%s"' % entry) + try: + log = logtracker.getlog(entry) + except epylog.AccessError, e: + ## + # Do not die, but disable this module and complain loudly + # + logger.put(0, 'Could not init logfile for entry "%s"' % entry) + self.enabled = 0 + logger.put(0, 'Module "%s" disabled' % self.name) + return + logger.put(5, 'Appending the log object to self.logs[]') + self.logs.append(log) + logger.put(5, 'Exiting Module.__init__')
def is_python(self): if self.python: - self.logger.put(2, 'This module is python') + self.logger.put(5, 'This module is python') return 1 else: - self.logger.put(2, 'This module is not python') + self.logger.put(5, 'This module is not python') return 0
def invoke_external_module(self, tmpprefix, cfgdir): @@ -142,14 +137,17 @@ class Module: logger.put(2, 'Checking if executable "%s" is sane' % self.executable) if not os.access(self.executable, os.F_OK): raise epylog.ModuleSanityError(('Executable "%s" for module "%s" ' - + 'does not exist') - % (self.executable, self.name), - logger) + + 'does not exist') + % (self.executable, self.name), + logger) + if self.is_python(): + logger.put(5, 'Module is python, not checking for exec bit') + return if not os.access(self.executable, os.X_OK): raise epylog.ModuleSanityError(('Executable "%s" for module "%s" ' - + 'is not set to execute') - % (self.executable, self.name), - logger) + + 'is not set to execute') + % (self.executable, self.name), + logger)
def get_html_report(self): logger = self.logger diff --git a/py/epylog/report.py b/py/epylog/report.py index 8d3e435..d355e94 100644 --- a/py/epylog/report.py +++ b/py/epylog/report.py @@ -6,87 +6,93 @@ import time from publishers import *
class Report: - section = 'report' - def __init__(self, config, logger): + logger.put(5, 'Entering Report.__init__') logger.put(2, 'Starting Report object intialization') - logger.put(3, 'Storing logger in object') self.logger = logger - - logger.put(3, 'Setting some defaults') + ## + # publishers: a tuple of publisher objects + # filt_fh: where the filtered strings from modules will go + # useful: tells epylog if the report is of any use or not. + # self.publishers = [] - self.module_reports = {} self.filt_fh = None - self.start_stamp = None - self.end_stamp = None self.useful = 0
self.tmpprefix = config.tmpprefix - sec = self.section - logger.put(2, 'Section name is %s' % sec) self.runtime = time.localtime() - logger.put(2, 'Timestamp is "%s"' % time.strftime('%c', self.runtime)) + sec = 'report' try: title = config.get(sec, 'title') except: title = '%HOSTNAME% system events: %LOCALTIME%' - logger.put(2, 'Title as defined in config is: "%s"' % title) - - import socket - title = re.sub(re.compile('%HOSTNAME%', re.M), - socket.gethostname(), title) - title = re.sub(re.compile('%LOCALTIME%', re.M), - time.strftime('%c', self.runtime), title) - self.title = title - logger.put(2, 'Title regexed into: "%s"' % self.title) - try: - template = config.get(sec, 'template').strip() + self.template = config.get(sec, 'template').strip() except: raise epylog.ConfigError('Report template not specified', logger) - if not os.access(template, os.R_OK): - raise epylog.AccessError(('Report template "%s" does not exist or' - + ' is not readable') % template, logger) - self.template = template - logger.put(3, 'template=%s' % self.template) - + if not os.access(self.template, os.R_OK): + msg = 'Report template "%s" is not readable' % self.template + raise epylog.AccessError(msg, logger) try: self.unparsed = config.getboolean(sec, 'include_unparsed') except: self.unparsed = 1 - logger.put(3, 'unparsed=%d' % self.unparsed) - - logger.put(2, 'Looking at defined publishers') try: publishers = config.get(sec, 'publishers') except: - raise epylog.ConfigError('No publishers defined in the "%s" section' - % sec, logger) + msg = 'No publishers defined in "%s"' % sec + raise epylog.ConfigError(msg, logger) + + logger.put(3, 'Title as defined in config is: "%s"' % title) + hregex = re.compile('@@HOSTNAME@@') + tregex = re.compile('@@LOCALTIME@@') + if hregex.search(title): + import socket + hostname = socket.gethostname() + logger.put(3, 'Regexing @@HOSTNAME@@ into "%s"' % hostname) + title = re.sub(hregex, hostname, title) + if tregex.search(title): + timestr = time.strftime('%c', self.runtime) + logger.put(3, 'Regexing @@LOCALTIME@@ into "%s"' % timestr) + title = re.sub(tregex, timestr, title) + self.title = title + logger.put(3, 'Final title is: "%s"' % self.title) + + logger.put(3, 'template=%s' % self.template) + logger.put(3, 'unparsed=%d' % self.unparsed) + if self.unparsed: + logger.put(3, 'Creating a temporary file for filtered strings') + import epylog.mytempfile as tempfile + tempfile.tmpdir = self.tmpprefix + filen = tempfile.mktemp('FILT') + self.filt_fh = open(filen, 'w+') + logger.put(3, 'Filtered strings file created in "%s"' % filen)
- logger.put(2, 'Publishers: "%s"' % publishers) - logger.put(2, 'Initializing publishers') + logger.put(3, 'Publishers: "%s"' % publishers) + logger.put(3, 'Initializing publishers') for sec in publishers.split(','): sec = sec.strip() - logger.put(2, 'Looking for section definition "%s"' % sec) + logger.put(3, 'Looking for section definition "%s"' % sec) if sec not in config.sections(): - raise epylog.ConfigError('Could not find publisher section "%s"' - % sec, logger) - logger.put(2, 'Looking for method declaration') + message = 'Required publisher section "%s" not found' % sec + raise epylog.ConfigError(message, logger) + logger.put(3, 'Looking for method declaration') try: method = config.get(sec, 'method') except: - raise epylog.ConfigError('Publishing method not found in "%s"' - % sec, logger) - logger.put(2, 'Found method "%s"' % method) + msg = 'Publishing method not found in "%s"' % sec + raise epylog.ConfigError(msg, logger) + logger.put(3, 'Found method "%s"' % method) if method == 'file': publisher = FilePublisher(sec, config, logger) elif method == 'mail': publisher = MailPublisher(sec, config, logger) else: - raise epylog.ConfigError('Publishing method "%s" not supported' - % method, logger) + msg = 'Publishing method "%s" not supported' % method + raise epylog.ConfigError(msg, logger) self.publishers.append(publisher) - logger.put(2, 'Finished with Report object initialization') + logger.put(3, 'Finished with Report object initialization') + logger.put(5, 'Exiting Report.__init__')
def append_module_report(self, module_name, module_report): if len(module_report) > 0: