commit 6f801de1708d10bdd08a67ad9218c75ee1c08300
Author: Konstantin Ryabitsev <icon(a)mricon.com>
Date: Thu Jan 23 21:42:28 2003 +0000
Big logfile handler rewrite.
epylog | 1 +
py/epylog/__init__.py | 154 ++++++-----------------
py/epylog/log.py | 337 +++++++++++++++++++++++++++++++++++++++----------
py/epylog/module.py | 110 ++++++++--------
py/epylog/report.py | 96 ++++++++-------
5 files changed, 412 insertions(+), 286 deletions(-)
---
diff --git a/epylog b/epylog
index 5e2ae37..81f1bfc 100755
--- a/epylog
+++ b/epylog
@@ -89,6 +89,7 @@ def main(args):
logger.puthang(1, 'Restoring log offsets')
epylog.restore_log_offsets()
logger.endhang(1, 'done')
+ sys.exit(1)
logger.put(1, 'Invoking the module execution routines:')
epylog.process_modules()
logger.put(1, 'Finished processing modules')
diff --git a/py/epylog/__init__.py b/py/epylog/__init__.py
index 5c72d0d..27684c4 100644
--- a/py/epylog/__init__.py
+++ b/py/epylog/__init__.py
@@ -9,7 +9,7 @@ import time
from report import Report
from module import Module
-from log import LogFile
+from log import LogTracker
VERSION = 'Epylog-0.8'
@@ -48,16 +48,15 @@ class SysCallError(exceptions.Exception):
logger.put(2, 'Raising SysCallError with message: %s' % message)
self.args = message
+class NoSuchLogError(exceptions.Exception):
+ def __init__(self, message, logger):
+ logger.put(2, 'Raising NoSuchLogError with message: %s' % message)
+ self.args = message
+
class Epylog:
def __init__(self, cfgfile, logger):
- logger.puthang(2, 'Starting the epylog object initialization')
- logger.put(5, 'Sticking logger into the object')
self.logger = logger
-
- logger.put(3, 'Setting some defaults')
- self.report = None
- self.modules = []
- self.offsets = None
+ logger.put(5, 'Entering Epylog.__init__')
config = ConfigParser.ConfigParser()
logger.puthang(3, 'Reading the config file "%s"' % cfgfile)
@@ -71,17 +70,20 @@ class Epylog:
self.cfgdir = config.get('main', 'cfgdir')
self.vardir = config.get('main', 'vardir')
except:
- raise ConfigError(('Could not parse the main config file "%s"'
- % cfgfile), logger)
+ msg = 'Could not parse the main config file "%s"' % cfgfile
+ raise ConfigError(msg, logger)
logger.put(4, 'cfgdir=%s' % self.cfgdir)
logger.put(4, 'vardir=%s' % self.vardir)
logger.endhang(3)
logger.put(3, 'Checking if we can write to vardir')
if not os.access(self.vardir, os.W_OK):
- raise ConfigError('Write access required for vardir "%s"'
- % self.vardir, logger)
+ msg = 'Write access required for vardir "%s"' % self.vardir
+ raise ConfigError(msg, logger)
+ ##
+ # Set up a safe temp dir
+ #
logger.put(3, 'Setting up a temporary directory')
try:
tmpdir = config.get('main', 'tmpdir')
@@ -92,23 +94,38 @@ class Epylog:
try:
tmpprefix = tempfile.mkdtemp('EPYLOG')
except:
- raise ConfigError('Could not create a temp directory in "%s"'
- % tmpprefix, logger)
+ msg = 'Could not create a safe temp directory in "%s"' % tmpprefix
+ raise ConfigError(msg, logger)
self.tmpprefix = tmpprefix
tempfile.tempdir = tmpprefix
logger.put(3, 'Temporary directory created in "%s"' % tmpprefix)
logger.put(3, 'Sticking tmpprefix into config to pass to other objs')
config.tmpprefix = self.tmpprefix
+ ##
+ # Initialize the Report object
+ #
logger.puthang(3, 'Initializing the Report')
self.report = Report(config, logger)
logger.endhang(3)
+ ##
+ # Initialize the LogTracker object
+ #
+ logger.puthang(3, 'Initializing the log tracker object')
+ logtracker = LogTracker(config, logger)
+ self.logtracker = logtracker
+ logger.endhang(3)
+
+ ##
+ # Process module configurations
+ #
+ self.modules = []
modcfgdir = os.path.join(self.cfgdir, 'modules.d')
logger.put(3, 'Checking if module config dir "%s" exists' % modcfgdir)
if not os.path.isdir(modcfgdir):
- raise ConfigError('Module configuration directory "%s" not found'
- % modcfgdir, logger)
+ msg = 'Module configuration directory "%s" not found' % modcfgdir
+ raise ConfigError(msg, logger)
logger.put(3, 'Looking for module configs in %s' % modcfgdir)
for file in os.listdir(modcfgdir):
cfgfile = os.path.join(modcfgdir, file)
@@ -120,116 +137,23 @@ class Epylog:
continue
logger.put(3, 'Ends in .conf all right.')
logger.puthang(3, 'Calling the Module init routines')
- module = Module(cfgfile, logger)
+ module = Module(cfgfile, logtracker, logger)
logger.endhang(3)
if module.enabled:
- logger.put(3, 'Module "%s" is enabled' % module.name)
+ logger.put(2, 'Module "%s" is enabled' % module.name)
logger.put(3, 'Checking "%s" for sanity' % module.name)
module.sanity_check()
logger.put(3, 'Sanity checks passed. Remembering module')
self.modules.append(module)
else:
- logger.put(3, 'Module "%s" is not enabled, ignoring'
+ logger.put(2, 'Module "%s" is not enabled, ignoring'
% module.name)
else:
logger.put(3, '%s is not a regular file, ignoring' % cfgfile)
logger.put(3, 'Total of %d modules initialized' % len(self.modules))
- logger.put(3, 'Looking at what logfiles to initialize')
- logmap = {}
- for module in self.modules:
- logger.put(3, 'Looking in module "%s"' % module.name)
- logger.put(5, module.lognames)
- for logname in module.lognames:
- logger.put(3, 'Found logfile declaration "%s"' % logname[0])
- if logmap.has_key(logname[0]):
- logger.put(3, 'Log file "%s" already initialized'
- % logname[0])
- logger.put(3, 'Sticking it into the module')
- module.logs.append(logmap[logname[0]])
- else:
- logger.put(3, '%s is not yet initialized. Initializing.'
- % logname[0])
- try:
- logger.puthang(3, 'Calling LogFile init routines')
- logobj = LogFile(logname[0], self.tmpprefix, logger)
- logger.endhang(3)
- except AccessError:
- raise ConfigError(('Log file "%s" does not exist or ' +
- 'is not accessible for reading')
- % logname[0], logger)
- logger.put(3, 'Opening the logfile "%s"' % logname[0])
- logobj.initfile()
- if logname[1] is not None:
- logger.put(3, 'Initializing the rotated logfile "%s"'
- % logname[1])
- try:
- rotobj = LogFile(logname[1], self.tmpprefix,
- logger)
- logger.put(3, ('Sticking rotated logfile object' +
- ' into the log object'))
- logobj.rotated = rotobj
- except AccessError:
- logger.put(3, 'Error opening. Hmm... Oh well.')
- logger.put(3, 'Ignoring rotated logfile "%s"'
- % logname[1])
- logger.put(3, 'Sticking the log object into the module')
- module.logs.append(logobj)
- logger.put(3, 'Copying object into the logs map')
- logmap[logname[0]] = logobj
-
- self.logmap = logmap
- if self.report.unparsed:
- logger.put(3, 'Creating a temporary file for filtered strings')
- filt_fh = open(tempfile.mktemp('FILT'), 'w')
- logger.put(3, 'Filtered strings file created in "%s"'
- % filt_fh.name)
- logger.put(3, 'Sticking filt_fh into the report object')
- self.report.filt_fh = filt_fh
- logger.endhang(2)
-
- def restore_log_offsets(self):
- logger = self.logger
- logger.put(2, 'Invoking the restore_log_offsets routine')
- logger.put(2, 'Setting up the iterator')
- iter = self.logmap.iteritems()
- earliest_stamp = None
- latest_stamp = None
- while 1:
- try:
- (logname, logobj) = iter.next()
- logger.put(2, 'Processing logname "%s"' % logname)
- self.__restore_log_offset(logobj)
- start_stamp = logobj.get_offset_start_stamp()
- end_stamp = logobj.get_offset_end_stamp()
- if earliest_stamp is None:
- earliest_stamp = start_stamp
- if latest_stamp is None:
- latest_stamp = end_stamp
- if start_stamp < earliest_stamp:
- earliest_stamp = start_stamp
- if end_stamp > latest_stamp:
- latest_stamp = end_stamp
- except StopIteration:
- logger.put(2, 'Iteration finished')
- break
- self.report.start_stamp = earliest_stamp
- self.report.end_stamp = latest_stamp
-
- def store_log_offsets(self):
- logger = self.logger
- logger.put(2, 'Invoking the store_log_offsets routine')
- logger.put(2, 'Setting up the iterator')
- iter = self.logmap.iteritems()
- while 1:
- try:
- (logname, logobj) = iter.next()
- logger.put(2, 'Processing logname "%s"' % logname)
- self.__store_log_offset(logobj)
- except StopIteration:
- logger.put(2, 'Iteration finished')
- break
- logger.put(2, 'Calling the pickling routine')
- self.__pickle_offsets()
+ if len(self.modules) == 0:
+ raise ConfigError('No modules are enabled. Exiting.', logger)
+ logger.put(5, 'Exiting Epylog.__init__')
def process_modules(self):
logger = self.logger
diff --git a/py/epylog/log.py b/py/epylog/log.py
index fa61a67..89c756c 100644
--- a/py/epylog/log.py
+++ b/py/epylog/log.py
@@ -4,83 +4,273 @@ import re
import string
import time
-class LogFile:
-
- def __init__(self, logfile, tmpprefix, logger):
- logger.put(5, 'Entering LogFile.__init__')
- logger.put(2, 'Starting LogFile object initialization for logfile "%s"'
- % logfile)
- logger.put(3, 'Sticking logger into object')
+class LogTracker:
+ def __init__(self, config, logger):
+ self.logger = logger
+ logger.put(5, 'Entering LogTracker.__init__')
+ self.tmpprefix = config.tmpprefix
+ self.entries = []
+ self.logs = []
+ logger.put(5, 'Exiting LogTracker.__init__')
+
+ def getlog(self, entry):
+ logger = self.logger
+ logger.put(5, 'Entering LogTracker.getlog')
+ logger.put(5, 'Checking if we have a log for entry "%s"' % entry)
+ if entry in self.entries:
+ logger.put(5, 'Yes, returning that log')
+ log = self.__get_log_by_entry(entry)
+ else:
+ logger.put(5, 'Logfile for "%s" not yet initialized' % entry)
+ log = self.__init_log_by_entry(entry)
+ logger.put(5, 'Exiting LogTracker.getlog')
+ return log
+
+ def get_offset_map(self):
+ logger = self.logger
+ logger.put(5, 'Entering LogTracker.get_offset_map')
+ omap = {}
+ for log in self.logs:
+ key = log.entry
+ inode = log.getinode()
+ if log.orange.endix != 0:
+ offset = 0
+ else:
+ offset = log.orange.end_offset
+ omap[key] = [inode, offset]
+ logger.put(5, 'omap follows')
+ logger.put(5, omap)
+ logger.put(5, 'Exiting LogTracker.get_offset_map')
+ return omap
+
+ def set_start_offset_by_entry(self, entry, inode, offset):
+ logger = self.logger
+ logger.put(5, '>LogTracker.set_offset_by_entry')
+ logger.put(5, 'entry=%s' % entry)
+ logger.put(5, 'inode=%d' % inode)
+ logger.put(5, 'offset=%d' % offset)
+ if entry in self.entries:
+ log = self.__get_log_by_entry(entry)
+ if log.getinode() != inode:
+ logger.put(5, 'Inodes do not match. Assuming logrotation')
+ try:
+ log.set_range_param(1, offset, 0)
+ except epylog.OutOfRangeError:
+ logger.put(5, 'No rotated file in place. Set offset to 0')
+ log.set_range_param(0, 0, 0)
+ else:
+ logger.put(5, 'Inodes match, setting offset to "%d"' % offset)
+ log.set_range_param(0, offset, 0)
+ else:
+ msg = 'No such log entry "%s"' % entry
+ raise epylog.NoSuchLogError(msg, logger)
+ logger.put(5, '<LogTracker.set_offset_by_entry')
+
+ def __init_log_by_entry(self, entry):
+ logger = self.logger
+ logger.put(5, 'Entering LogTracker.__init_log_by_entry')
+ logger.puthang(5, 'Initializing log object for entry "%s"' % entry)
+ log = Log(entry, self.tmpprefix, self.logger)
+ logger.endhang(5)
+ self.entries.append(entry)
+ self.logs.append(log)
+ logger.put(5, 'Exiting LogTracker.__init_log_by_entry')
+ return log
+
+ def __get_log_by_entry(self, entry):
+ logger = self.logger
+ logger.put(5, 'Entering LogTracker.__get_log_by_entry')
+ for log in self.logs:
+ if log.entry == entry:
+ logger.put(5, 'Found log object "%s"' % entry)
+ logger.put(5, 'Exiting LogTracker.__get_log_by_entry')
+ return log
+ logger.put(5, 'No such log file! Returning None. How did this happen?')
+ return None
+
+
+class OffsetRange:
+ def __init__(self, startix, start_offset, endix, end_offset, logger):
self.logger = logger
+ logger.put(5, 'Entering OffsetRange.__init__')
+ self.startix = startix
+ self.endix = endix
+ self.start_offset = start_offset
+ self.end_offset = end_offset
+ logger.put(5, 'startix=%d' % self.startix)
+ logger.put(5, 'start_offset=%d' % self.start_offset)
+ logger.put(5, 'endix=%d' % self.endix)
+ logger.put(5, 'end_offset=%d' % self.end_offset)
+ logger.put(5, 'Exiting OffsetRange.__init__')
+
+ def setstart(self, ix, offset):
+ logger = self.logger
+ logger.put(5, 'Entering OffsetRange.setstart')
+ self.startix = ix
+ self.start_offset = offset
+ logger.put(5, 'new startix=%d' % self.startix)
+ logger.put(5, 'new start_offset=%d' % self.start_offset)
+ logger.put(5, 'Exiting OffsetRange.setstart')
+ def setend(self, ix, offset):
+ logger = self.logger
+ logger.put(5, 'Entering OffsetRange.setend')
+ self.endix = ix
+ self.end_offset = offset
+ logger.put(5, 'new endix=%d' % self.endix)
+ logger.put(5, 'new end_offset=%d' % self.end_offset)
+ logger.put(5, 'Exiting OffsetRange.setend')
+
+class Log:
+ def __init__(self, entry, tmpprefix, logger):
+ logger.put(5, 'Entering Log.__init__')
+ logger.puthang(3, 'Initializing Log object for entry "%s"' % entry)
+ self.logger = logger
self.tmpprefix = tmpprefix
- logger.put(3, 'Setting some defaults')
- self.fh = None
- self.rotated = None
- self.monthmap = None
- self.log_start_stamp = None
- self.log_end_stamp = None
- self.start_offset = 0
- self.end_offset = None
- self.log_end_offset = None
+ self.entry = entry
+ filename = self.__get_filename()
+ logger.puthang(4, 'Initializing the logfile "%s"' % filename)
+ logfile = LogFile(filename, tmpprefix, logger)
+ logger.endhang(4)
+ logger.put(3, 'Appending logfile to the loglist')
+ self.loglist = [logfile]
+ self.orange = OffsetRange(0, 0, 0, logfile.end_offset, logger)
+ logger.endhang(3)
+ logger.put(5, 'Exiting Log.__init__')
+
+ def set_range_param(self, ix, offset, whence=0):
+ logger = self.logger
+ logger.put(5, 'Entering Log.set_range_param')
+ logger.put(5, 'ix=%d' % ix)
+ logger.put(5, 'offset=%d' % offset)
+ logger.put(5, 'whence=%d' % whence)
+ try:
+ while not self.__is_valid_ix(ix):
+ self.__init_next_logfile()
+ except epylog.NoSuchLogError:
+ msg = 'Invalid index "%d" for log entry "%s"' % (ix, self.entry)
+ raise epylog.OutOfRangeError(msg, logger)
+ if whence:
+ logger.put(5, 'Setting range END for entry "%s"' % self.entry)
+ self.orange.setend(ix, offset)
+ else:
+ logger.put(5, 'Setting range START for entry "%s"' % self.entry)
+ self.orange.setstart(ix, offset)
+ logger.put(5, 'Exiting Log.set_range_param')
+
+ def __is_valid_ix(self, ix):
+ logger = self.logger
+ logger.put(5, 'Entering Log.__is_valid_ix')
+ ixlen = len(self.loglist) - 1
+ isvalid = 1
+ if ix > ixlen:
+ logger.put(5, 'index %d is not valid' % ix)
+ isvalid = 0
+ logger.put(5, 'Exiting Log.__is_valid_ix')
+ return isvalid
+
+ def __init_next_rotfile(self):
+ logger = self.logger
+ logger.put(5, '>Log.__init_next_rotfile')
+ ix = len(self.loglist)
+ rotname = self.__get_rotname_by_ix(ix)
+ try:
+ logger.put(5, 'Initializing log for rotated file "%s"' % rotname)
+ rotlog = LogFile(rotname, self.tmpprefix, logger)
+ except epylog.AccessError:
+ msg = 'No further rotated files for entry "%s"' % self.entry
+ raise epylog.NoSuchLogError(msg, logger)
+ self.loglist.append(rotlog)
+ logger.put(5, '<Log.__init_next_rotfile')
+
+ def __get_rotname_by_ix(self, ix):
+ logger = self.logger
+ logger.put(5, '>Log.__get_rotname_by_ix')
+ logger.put(5, 'ix=%d' % ix)
+ rotname = re.sub(re.compile('\[|\]'), '', self.entry)
+ rotname = re.sub(re.compile('#'), str(ix), rotname)
+ logger.put(5, 'rotname=%s' % rotname)
+ logger.put(5, '<Log.__get_rotname_by_ix')
+ return rotname
+
+ def __get_filename(self):
+ logger = self.logger
+ logger.put(5, '>Log.__get_filename')
+ logger.put(5, 'entry=%s' % self.entry)
+ filename = re.sub(re.compile('\[.*?\]'), '', self.entry)
+ logger.put(5, 'filename=%s' % filename)
+ logger.put(5, '<Log.__get_filename')
+ return filename
+
+ def getinode(self):
+ logger = self.logger
+ logger.put(5, 'Entering Log.getinode')
+ logfile = self.loglist[0]
+ inode = logfile.getinode()
+ logger.put(5, 'inode=%d' % inode)
+ logger.put(5, 'Exiting Log.getinode')
+ return inode
- self.filename = logfile
+class LogFile:
+ def __init__(self, filename, tmpprefix, logger):
+ self.logger = logger
+ logger.put(5, 'Entering LogFile.__init__')
+ self.tmpprefix = tmpprefix
+ self.filename = filename
+ ##
+ # start_stamp: the timestamp at the start of the log
+ # end_stamp: the timestamp at the end of the log
+ # end_offset: this is where the end of the log is
+ #
+ self.start_stamp = None
+ self.end_stamp = None
+ self.end_offset = None
+
+ logger.put(3, 'Running sanity checks on the logfile')
self.__accesscheck()
- logger.put(2, 'All checks passed')
- self.inode = os.stat(logfile).st_ino
- logger.put(3, 'inode=%d' % self.inode)
- logger.put(2, 'Finished LogFile object initialization for "%s"'
- % logfile)
+ logger.put(3, 'All checks passed')
+ logger.put(5, 'Initializing the file')
+ self.__initfile()
logger.put(5, 'Exiting LogFile.__init__')
- def initfile(self):
+ def __initfile(self):
logger = self.logger
- logger.put(5, 'Entering LogFile.initfile')
- logger.put(2, 'Checking if we are already initialized')
- if self.fh is None:
- logger.put(2, 'Not inited yet, initing')
- logger.put(2, 'Checking if we are gzipped (ends in .gz)')
- if re.compile('\.gz$').search(self.filename, 1):
- logger.put(2, 'Ends in .gz. Using GzipFile to open')
- import gzip
- import epylog.mytempfile as tempfile
- tempfile.tmpdir = self.tmpprefix
- ungzfile = tempfile.mktemp('UNGZ')
- logger.put(3, 'Creating a tempfile in "%s"' % ungzfile)
- ungzfh = open(tempfile.mktemp('UNGZ'), 'w+')
- try:
- gzfh = gzip.open(self.filename)
- except:
- raise epylog.ConfigError(('Could not open file "%s" with'
- + ' gzip handler. Not gzipped?')
- % self.filename, logger)
- logger.put(2, 'Putting the contents of the gzlog into ungzlog')
- while 1:
- chunk = gzfh.read(1024)
- if chunk:
- ungzfh.write(chunk)
- logger.put(5, 'Read "%s" bytes from gzfh' % len(chunk))
- else:
- logger.put(5, 'Reached EOF')
- break
- gzfh.close()
- self.fh = ungzfh
- else:
- logger.put(2, 'Does not end in .gz, assuming plain text')
- logger.put(2, 'Opening logfile "%s"' % self.filename)
- self.fh = open(self.filename)
- logger.put(2, 'Finding the end offset')
- self.fh.seek(0, 2)
- self.__set_at_line_start()
- self.end_offset = self.fh.tell()
- self.log_end_offset = self.fh.tell()
- if self.end_offset == 0:
- logger.put(2, 'This logfile is empty!')
- logger.put(2, 'log_end_offset=%d' % self.log_end_offset)
+ logger.put(5, 'Entering LogFile.__initfile')
+ logger.put(5, 'Checking if we are gzipped (ends in .gz)')
+ if re.compile('\.gz$').search(self.filename, 1):
+ logger.put(5, 'Ends in .gz. Using GzipFile to open')
+ import gzip
+ import epylog.mytempfile as tempfile
+ tempfile.tmpdir = self.tmpprefix
+ ungzfile = tempfile.mktemp('UNGZ')
+ logger.put(5, 'Creating a tempfile in "%s"' % ungzfile)
+ ungzfh = open(tempfile.mktemp('UNGZ'), 'w+')
+ try:
+ gzfh = gzip.open(self.filename)
+ except:
+ raise epylog.ConfigError(('Could not open file "%s" with'
+ + ' gzip handler. Not gzipped?')
+ % self.filename, logger)
+ logger.put(5, 'Putting the contents of the gzlog into ungzlog')
+ while 1:
+ chunk = gzfh.read(1024)
+ if chunk:
+ ungzfh.write(chunk)
+ logger.put(5, 'Read "%s" bytes from gzfh' % len(chunk))
+ else:
+ logger.put(5, 'Reached EOF')
+ break
+ gzfh.close()
+ self.fh = ungzfh
else:
- logger.put(2, 'Already initialized, ignoring')
- pass
- logger.put(5, 'Exiting LogFile.initfile')
+ logger.put(5, 'Does not end in .gz, assuming plain text')
+ logger.put(5, 'Opening logfile "%s"' % self.filename)
+ self.fh = open(self.filename)
+ logger.put(5, 'Finding the end offset')
+ self.fh.seek(0, 2)
+ self.__set_at_line_start()
+ self.end_offset = self.fh.tell()
+ logger.put(5, 'Exiting LogFile.__initfile')
def set_init_offset(self):
logger = self.logger
@@ -139,6 +329,13 @@ class LogFile:
return self.get_log_end_stamp()
else:
return self.get_stamp_at_offset(self.end_offset)
+
+ def getinode(self):
+ self.logger.put(5, 'Entering LogFile.getinode')
+ inode = os.stat(self.filename).st_ino
+ self.logger.put(5, 'inode=%d' % inode)
+ self.logger.put(5, 'Exiting LogFile.getinode')
+ return inode
def find_offset_by_timestamp(self, searchstamp):
logger = self.logger
@@ -417,13 +614,13 @@ class LogFile:
else:
logger.put(2, 'Path "%s" does not exist' % logfile)
raise epylog.AccessError('Log file "%s" does not exist'
- % logfile, logger)
+ % logfile, logger)
if os.access(logfile, os.R_OK):
logger.put(2, 'File "%s" is readable' % logfile)
else:
logger.put(2, 'Logfile "%s" is not readable' % logfile)
raise epylog.AccessError('Logfile "%s" is not readable'
- % logfile, logger)
+ % logfile, logger)
logger.put(5, 'Exiting LogFile.__accesscheck')
def __set_at_line_start(self):
diff --git a/py/epylog/module.py b/py/epylog/module.py
index 13e447e..f9a1f61 100644
--- a/py/epylog/module.py
+++ b/py/epylog/module.py
@@ -1,93 +1,88 @@
import ConfigParser
import epylog
import os
-import os.path
import mytempfile as tempfile
class Module:
"""epylog Module class"""
- def __init__(self, cfgfile, logger):
- self.lognames = []
- self.logs = []
+ def __init__(self, cfgfile, logtracker, logger):
+ self.logger = logger
+ logger.put(5, 'Entering Module.__init__')
self.logreport = None
self.logfilter = None
logger.put(2, 'Initializing module for cfgfile %s' % cfgfile)
- logger.put(3, 'Sticking logger into object')
- self.logger = logger
config = ConfigParser.ConfigParser()
logger.put(2, 'Reading in the cfgfile %s' % cfgfile)
config.read(cfgfile)
try:
- self.name = config.get('module', 'name')
+ self.name = config.get('module', 'desc')
except:
self.name = 'Unnamed Module'
- logger.put(2, 'name=%s' % self.name)
- try:
- self.executable = config.get('module', 'exec')
- except:
- raise epylog.ConfigError('Did not find executable name in %s'
- % cfgfile, logger)
- logger.put(2, 'executable=%s' % self.executable)
try:
self.enabled = config.getboolean('module', 'enabled')
except:
self.enabled = 0
- logger.put(2, 'enabled=%d' % self.enabled)
+ if not self.enabled:
+ logger.put(2, 'This module is not enabled. Skipping init.')
+ return
+ try:
+ self.executable = config.get('module', 'exec')
+ except:
+ raise epylog.ConfigError('Did not find executable name in %s'
+ % cfgfile, logger)
try:
self.python = config.getboolean('module', 'python')
except:
self.python = 0
- logger.put(2, 'python=%d' % self.python)
try:
- logs = config.get('logs', 'files')
+ logentries = config.get('logs', 'files')
except:
raise epylog.ConfigError(('Cannot find log definitions in ' +
- 'module config "%s"') % cfgfile)
+ 'module config "%s"') % cfgfile)
try:
- rots = config.get('logs', 'rotated')
+ rotdir = config.get('logs', 'rotdir')
except:
- rots = 0
- logger.put(3, 'logs=%s' % logs)
- logger.put(3, 'rots=%s' % rots)
- loglist = logs.split(',')
- if rots:
- rotlist = rots.split(',')
- if rots and len(loglist) is not len(rotlist):
- raise epylog.ConfigError(('%d logs specified, but %d rotated logs' +
- ' specifications found in module config' +
- ' "%s"')
- % (len(loglist), len(rotlist), cfgfile),
- logger)
- logger.put(5, 'Loglist follows')
- logger.put(5, loglist)
- logger.put(5, 'self.lognames before the loop')
- logger.put(5, self.lognames)
- for log in loglist:
- log = log.strip()
- if rots:
- rot = rotlist.pop(0)
- rot = rot.strip()
- else:
- rot = None
- logger.put(2, 'tuple: [%s, %s]' % (log, str(rot)))
- self.lognames.append([log, rot])
- logger.put(5, 'self.lognames:')
- logger.put(5, self.lognames)
-
+ rotdir = None
try:
self.outhtml = config.getboolean('output', 'html')
except:
self.outhtml = 0
+
+ logger.put(5, 'name=%s' % self.name)
+ logger.put(5, 'executable=%s' % self.executable)
+ logger.put(5, 'enabled=%d' % self.enabled)
+ logger.put(5, 'python=%d' % self.python)
+ logger.put(5, 'logentries=%s' % logentries)
logger.put(2, 'outhtml=%d' % self.outhtml)
- logger.put(2, 'Finished with Module object initialization')
+
+ logger.put(3, 'Figuring out the logfiles from the log list')
+ entrylist = logentries.split(',')
+ self.logs = []
+ for entry in entrylist:
+ entry = entry.strip()
+ logger.put(5, 'entry=%s' % entry)
+ logger.put(3, 'Getting a log object from entry "%s"' % entry)
+ try:
+ log = logtracker.getlog(entry)
+ except epylog.AccessError, e:
+ ##
+ # Do not die, but disable this module and complain loudly
+ #
+ logger.put(0, 'Could not init logfile for entry "%s"' % entry)
+ self.enabled = 0
+ logger.put(0, 'Module "%s" disabled' % self.name)
+ return
+ logger.put(5, 'Appending the log object to self.logs[]')
+ self.logs.append(log)
+ logger.put(5, 'Exiting Module.__init__')
def is_python(self):
if self.python:
- self.logger.put(2, 'This module is python')
+ self.logger.put(5, 'This module is python')
return 1
else:
- self.logger.put(2, 'This module is not python')
+ self.logger.put(5, 'This module is not python')
return 0
def invoke_external_module(self, tmpprefix, cfgdir):
@@ -142,14 +137,17 @@ class Module:
logger.put(2, 'Checking if executable "%s" is sane' % self.executable)
if not os.access(self.executable, os.F_OK):
raise epylog.ModuleSanityError(('Executable "%s" for module "%s" '
- + 'does not exist')
- % (self.executable, self.name),
- logger)
+ + 'does not exist')
+ % (self.executable, self.name),
+ logger)
+ if self.is_python():
+ logger.put(5, 'Module is python, not checking for exec bit')
+ return
if not os.access(self.executable, os.X_OK):
raise epylog.ModuleSanityError(('Executable "%s" for module "%s" '
- + 'is not set to execute')
- % (self.executable, self.name),
- logger)
+ + 'is not set to execute')
+ % (self.executable, self.name),
+ logger)
def get_html_report(self):
logger = self.logger
diff --git a/py/epylog/report.py b/py/epylog/report.py
index 8d3e435..d355e94 100644
--- a/py/epylog/report.py
+++ b/py/epylog/report.py
@@ -6,87 +6,93 @@ import time
from publishers import *
class Report:
- section = 'report'
-
def __init__(self, config, logger):
+ logger.put(5, 'Entering Report.__init__')
logger.put(2, 'Starting Report object intialization')
- logger.put(3, 'Storing logger in object')
self.logger = logger
-
- logger.put(3, 'Setting some defaults')
+ ##
+ # publishers: a tuple of publisher objects
+ # filt_fh: where the filtered strings from modules will go
+ # useful: tells epylog if the report is of any use or not.
+ #
self.publishers = []
- self.module_reports = {}
self.filt_fh = None
- self.start_stamp = None
- self.end_stamp = None
self.useful = 0
self.tmpprefix = config.tmpprefix
- sec = self.section
- logger.put(2, 'Section name is %s' % sec)
self.runtime = time.localtime()
- logger.put(2, 'Timestamp is "%s"' % time.strftime('%c', self.runtime))
+ sec = 'report'
try:
title = config.get(sec, 'title')
except:
title = '%HOSTNAME% system events: %LOCALTIME%'
- logger.put(2, 'Title as defined in config is: "%s"' % title)
-
- import socket
- title = re.sub(re.compile('%HOSTNAME%', re.M),
- socket.gethostname(), title)
- title = re.sub(re.compile('%LOCALTIME%', re.M),
- time.strftime('%c', self.runtime), title)
- self.title = title
- logger.put(2, 'Title regexed into: "%s"' % self.title)
-
try:
- template = config.get(sec, 'template').strip()
+ self.template = config.get(sec, 'template').strip()
except:
raise epylog.ConfigError('Report template not specified', logger)
- if not os.access(template, os.R_OK):
- raise epylog.AccessError(('Report template "%s" does not exist or'
- + ' is not readable') % template, logger)
- self.template = template
- logger.put(3, 'template=%s' % self.template)
-
+ if not os.access(self.template, os.R_OK):
+ msg = 'Report template "%s" is not readable' % self.template
+ raise epylog.AccessError(msg, logger)
try:
self.unparsed = config.getboolean(sec, 'include_unparsed')
except:
self.unparsed = 1
- logger.put(3, 'unparsed=%d' % self.unparsed)
-
- logger.put(2, 'Looking at defined publishers')
try:
publishers = config.get(sec, 'publishers')
except:
- raise epylog.ConfigError('No publishers defined in the "%s" section'
- % sec, logger)
+ msg = 'No publishers defined in "%s"' % sec
+ raise epylog.ConfigError(msg, logger)
+
+ logger.put(3, 'Title as defined in config is: "%s"' % title)
+ hregex = re.compile('@@HOSTNAME@@')
+ tregex = re.compile('@@LOCALTIME@@')
+ if hregex.search(title):
+ import socket
+ hostname = socket.gethostname()
+ logger.put(3, 'Regexing @@HOSTNAME@@ into "%s"' % hostname)
+ title = re.sub(hregex, hostname, title)
+ if tregex.search(title):
+ timestr = time.strftime('%c', self.runtime)
+ logger.put(3, 'Regexing @@LOCALTIME@@ into "%s"' % timestr)
+ title = re.sub(tregex, timestr, title)
+ self.title = title
+ logger.put(3, 'Final title is: "%s"' % self.title)
+
+ logger.put(3, 'template=%s' % self.template)
+ logger.put(3, 'unparsed=%d' % self.unparsed)
+ if self.unparsed:
+ logger.put(3, 'Creating a temporary file for filtered strings')
+ import epylog.mytempfile as tempfile
+ tempfile.tmpdir = self.tmpprefix
+ filen = tempfile.mktemp('FILT')
+ self.filt_fh = open(filen, 'w+')
+ logger.put(3, 'Filtered strings file created in "%s"' % filen)
- logger.put(2, 'Publishers: "%s"' % publishers)
- logger.put(2, 'Initializing publishers')
+ logger.put(3, 'Publishers: "%s"' % publishers)
+ logger.put(3, 'Initializing publishers')
for sec in publishers.split(','):
sec = sec.strip()
- logger.put(2, 'Looking for section definition "%s"' % sec)
+ logger.put(3, 'Looking for section definition "%s"' % sec)
if sec not in config.sections():
- raise epylog.ConfigError('Could not find publisher section "%s"'
- % sec, logger)
- logger.put(2, 'Looking for method declaration')
+ message = 'Required publisher section "%s" not found' % sec
+ raise epylog.ConfigError(message, logger)
+ logger.put(3, 'Looking for method declaration')
try:
method = config.get(sec, 'method')
except:
- raise epylog.ConfigError('Publishing method not found in "%s"'
- % sec, logger)
- logger.put(2, 'Found method "%s"' % method)
+ msg = 'Publishing method not found in "%s"' % sec
+ raise epylog.ConfigError(msg, logger)
+ logger.put(3, 'Found method "%s"' % method)
if method == 'file':
publisher = FilePublisher(sec, config, logger)
elif method == 'mail':
publisher = MailPublisher(sec, config, logger)
else:
- raise epylog.ConfigError('Publishing method "%s" not supported'
- % method, logger)
+ msg = 'Publishing method "%s" not supported' % method
+ raise epylog.ConfigError(msg, logger)
self.publishers.append(publisher)
- logger.put(2, 'Finished with Report object initialization')
+ logger.put(3, 'Finished with Report object initialization')
+ logger.put(5, 'Exiting Report.__init__')
def append_module_report(self, module_name, module_report):
if len(module_report) > 0:
