gfs2-utils: master - gfs2_trace: Added a script called gfs2_trace for kernel tracing debugging.

[shane bradley]

Gitweb:       
http://git.fedorahosted.org/git/?p=gfs2-utils.git;a=commitdiff;h=41991d08...
Commit:        41991d08c42ff779d26143598d0835e88ab24f65
Parent:        b3d765d9e7ec7c53bc7b88c7eef35f5a249ebcae
Author:        Shane Bradley <sbradley(a)redhat.com&gt;
AuthorDate:    Wed Dec 19 09:38:48 2012 -0500
Committer:     Shane Bradley <sbradley(a)redhat.com&gt;
CommitterDate: Wed Dec 19 09:38:48 2012 -0500

gfs2_trace: Added a script called gfs2_trace for kernel tracing debugging.

The script gfs2_trace is a tool used for debugging kernel tracing events. The
script can enabled or disable all or selected GFS2 kernel tracing events. The
script can capture the trace events and write them to a specific file which will
be tarred up after the file is written.

Signed-off-by: Shane Bradley <sbradley(a)redhat.com&gt;
---
 gfs2/man/Makefile.am     |    3 +-
 gfs2/man/gfs2_trace.8    |   45 +++
 gfs2/scripts/Makefile.am |    2 +-
 gfs2/scripts/gfs2_trace  |  790 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 838 insertions(+), 2 deletions(-)

diff --git a/gfs2/man/Makefile.am b/gfs2/man/Makefile.am
index 8655a76..82eab9a 100644
--- a/gfs2/man/Makefile.am
+++ b/gfs2/man/Makefile.am
@@ -8,4 +8,5 @@ dist_man_MANS		= fsck.gfs2.8 \
 			  gfs2_jadd.8 \
 			  mkfs.gfs2.8 \
 			  tunegfs2.8 \
-			  gfs2_lockcapture.8 
+			  gfs2_lockcapture.8 \
+			  gfs2_trace.8
diff --git a/gfs2/man/gfs2_trace.8 b/gfs2/man/gfs2_trace.8
new file mode 100644
index 0000000..dd98072
--- /dev/null
+++ b/gfs2/man/gfs2_trace.8
@@ -0,0 +1,45 @@
+.TH gfs2_trace 8
+
+.SH NAME
+gfs2_trace \- can enable trace events, disable trace events, and capture data from GFS2
trace events.
+
+.SH SYNOPSIS
+.B gfs2_trace \fR[-dqEN]  [-e \fItrace event name]\fR [-n \fItrace event name]\fR [-o
\fIoutput filename]\fR
+.PP
+
+.SH DESCRIPTION
+\fIgfs2_trace\fR can enabled and disable trace events on all trace events or selected
trace events. \fIgfs2_trace\fR can
+capture the output of the trace events and write the output to a file. When capturing
trace events, the script will exit
+when control-c is pressed. The trace events will be then written to the selected file.
+.PP
+
+.SH OPTIONS
+.TP
+\fB-h,  --help\fP
+Prints out a short usage message and exits.
+.TP
+\fB-d,  --debug\fP
+enables debug logging.
+.TP
+\fB-q,  --quiet\fP
+disables logging to console.
+.TP
+\fB-l,  --list\fP
+lists the enabled state and filters for the GFS2 trace events
+.TP
+\fB-E,  --enable_all_trace_events\fP
+enables all trace_events for GFS2
+.TP
+\fB-e \fI<trace event name>, \fB----enable_trace_event\fR=\fI<trace event
name>\fP
+selected trace_events that will be enabled for GFS2
+.TP
+\fB-N,  --disable_all_trace_events\fP
+disables all trace_events for GFS2
+.TP
+\fB-n \fI<trace event name>, \fB----disable_trace_event\fR=\fI<trace event
name>\fP
+selected trace_events that will be enabled for GFS2
+.TP
+\fB-c \fI<output filename>, \fB--capture\fR=\fI<output filename>\fP
+enables capturing of trace events and will save the data to a file
+.
+.SH SEE ALSO
diff --git a/gfs2/scripts/Makefile.am b/gfs2/scripts/Makefile.am
index b88580e..2c51222 100644
--- a/gfs2/scripts/Makefile.am
+++ b/gfs2/scripts/Makefile.am
@@ -9,4 +9,4 @@ sbindir := $(shell rpl=0; test '$(exec_prefix):$(sbindir)' =
/usr:/usr/sbin \
                                      test $$rpl = 1 && echo /sbin || echo
'$(exec_prefix)/sbin')
 
 
-dist_sbin_SCRIPTS           = gfs2_lockcapture
+dist_sbin_SCRIPTS           = gfs2_lockcapture gfs2_trace
diff --git a/gfs2/scripts/gfs2_trace b/gfs2/scripts/gfs2_trace
new file mode 100644
index 0000000..38b3d18
--- /dev/null
+++ b/gfs2/scripts/gfs2_trace
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+"""
+This script will enable or disable trace events for GFS2. The script can capture
+trace events and write the trace events captured to a file.
+
+When capturing events, hit "control-c" to exit and then the capture events will
be
+written to a file. A file will be created by reading this pipe:
+/sys/kernel/debug/tracing/trace_pipe
+
+The debug directory is required to be mounted which will be mounted if not
+mounted. The trace events are located in this directory
+/sys/kernel/debug/tracing/events/gfs2.
+
+The file that can be used to validate what fields are valid for "filters" is
+described in the format file. For example:
+/sys/kernel/debug/tracing/events/gfs2/*/format
+
+@author    :  Shane Bradley
+@contact   :  sbradley(a)redhat.com
+@version   :  0.9
+@copyright :  GPLv2
+"""
+import sys
+import os
+import os.path
+import logging
+import platform
+import fileinput
+import tarfile
+import subprocess
+from optparse import OptionParser, Option
+
+# #####################################################################
+# Global Vars:
+# #####################################################################
+"""
+@cvar VERSION_NUMBER: The version number of this script.
+@type VERSION_NUMBER: String
+@cvar MAIN_LOGGER_NAME: The name of the logger.
+@type MAIN_LOGGER_NAME: String
+@cvar PATH_TO_DEBUG_DIR: The path to the debug directory for the linux kernel.
+@type PATH_TO_DEBUG_DIR: String
+@cvar PATH_TO_PID_FILENAME: The path to the pid file that will be used to make
+sure only 1 instance of this script is running at any time.
+@type PATH_TO_PID_FILENAME: String
+@cvar PATH_TO_GFS2_TRACE_EVENTS_DIR: The path to the directory that contains
+all the GFS2 trace events.
+@type PATH_TO_GFS2_TRACE_EVENTS_DIR: String
+@cvar PATH_TO_TRACE_PIPE: The path to the tracing pipe.
+@type PATH_TO_TRACE_PIPE: String
+"""
+VERSION_NUMBER = "0.9-1"
+MAIN_LOGGER_NAME = "gfs2trace"
+PATH_TO_DEBUG_DIR="/sys/kernel/debug"
+PATH_TO_PID_FILENAME = "/var/run/%s.pid" %(os.path.basename(sys.argv[0]))
+PATH_TO_GFS2_TRACE_EVENTS_DIR="%s/tracing/events/gfs2" %(PATH_TO_DEBUG_DIR)
+PATH_TO_TRACE_PIPE="%s/tracing/trace_pipe" %(PATH_TO_DEBUG_DIR)
+
+class FileUtils:
+    """
+    A class that provides static functions for files such as reading and
+    writing.
+    """
+    def getDataFromFile(pathToSrcFile) :
+        """
+        This function will return the data in an array. Where each newline in file
+        is a seperate item in the array. This should really just be used on
+        relatively small files.
+
+        None is returned if no file is found.
+
+        @return: Returns an array of Strings, where each newline in file is an item
+        in the array.
+        @rtype: Array
+
+        @param pathToSrcFile: The path to the file which will be read.
+        @type pathToSrcFile: String
+        """
+        if (len(pathToSrcFile) > 0) :
+            try:
+                fin = open(pathToSrcFile, "r")
+                data = fin.readlines()
+                fin.close()
+                return data
+            except (IOError, os.error):
+                message = "An error occured reading the file: %s."
%(pathToSrcFile)
+                logging.getLogger(MAIN_LOGGER_NAME).error(message)
+        return None
+    getDataFromFile = staticmethod(getDataFromFile)
+
+    def writeToFile(pathToFilename, data, appendToFile=True, createFile=False):
+        """
+        This function will write a string to a file.
+
+        @return: Returns True if the string was successfully written to the file,
+        otherwise False is returned.
+        @rtype: Boolean
+
+        @param pathToFilename: The path to the file that will have a string written
+        to it.
+        @type pathToFilename: String
+        @param data: The string that will be written to the file.
+        @type data: String
+        @param appendToFile: If True then the data will be appened to the file, if
+        False then the data will overwrite the contents of the file.
+        @type appendToFile: Boolean
+        @param createFile: If True then the file will be created if it does not
+        exists, if False then file will not be created if it does not exist
+        resulting in no data being written to the file.
+        @type createFile: Boolean
+        """
+        [parentDir, filename] = os.path.split(pathToFilename)
+        if (os.path.isfile(pathToFilename) or (os.path.isdir(parentDir) and
createFile)):
+            try:
+                filemode = "w"
+                if (appendToFile):
+                    filemode = "a"
+                fout = open(pathToFilename, filemode)
+                fout.write(data + "\n")
+                fout.close()
+                return True
+            except UnicodeEncodeError, e:
+                message = "There was a unicode encode error writing to the file:
%s." %(pathToFilename)
+                logging.getLogger(MAIN_LOGGER_NAME).error(message)
+                return False
+            except IOError:
+                message = "There was an error writing to the file: %s."
%(pathToFilename)
+                logging.getLogger(MAIN_LOGGER_NAME).error(message)
+                return False
+        return False
+    writeToFile = staticmethod(writeToFile)
+
+class TraceEvent:
+    """
+    A class that reprensents a trace event.
+    """
+    def __init__(self, pathToTraceEvent):
+        """
+        @param pathToTraceEvent: The path to the trace event directory.
+        @type pathToTraceEvent: String
+        """
+        self.__pathToTraceEvent = pathToTraceEvent
+
+    def __str__(self):
+        """
+        Returns a string representation of a TraceEvent.
+
+        @return: Returns a string representation of a TraceEvent.
+        @rtype: String
+        """
+        return "%s: %s" %(self.getName(), self.getEnable())
+
+    def __getEventPathItem(self, pathToFilename):
+        """
+        Returns the data contain in the file. If file does not exist then empty
+        array is returned.
+
+        @return: Returns the data contained in the file. If file does not exist
+        then empty array is returned.
+        @rtype: Array
+
+        @param pathToFilename: The path to the filename of the file in the trace
+        event's directory.
+        @type pathToFilename: String
+        """
+        output = FileUtils.getDataFromFile(pathToFilename)
+        if (output == None):
+            return []
+        return output
+
+    def __setEventPathItem(self, pathToFilename, contentsOfEventItem,
appendToFile=True):
+        """
+        This function will write data to a file in the trace event's directory.
+
+        @return: Returns True is data was successfully written.
+        @rtype: Boolean
+        """
+        return FileUtils.writeToFile(pathToFilename, contentsOfEventItem, appendToFile)
+
+    def getPathToTraceEvent(self):
+        """
+        Returns the path to the trace event directory.
+
+        @return: Returns the path to the trace event directory.
+        @rtype: String
+        """
+        return self.__pathToTraceEvent
+
+    def getName(self):
+        """
+        Returns the shortname of the trace event.
+
+        @return: Returns the shortname of the trace event.
+        @rtype: String
+        """
+        return os.path.basename(self.getPathToTraceEvent())
+
+    def getEnable(self):
+        """
+        Returns the contents of the file "enable" in the trace event
directory.
+
+        @return: Returns the contents of the file "enable" in the trace event
+        directory.
+        @rtype: String
+        """
+        fileContents = self.__getEventPathItem(os.path.join(self.getPathToTraceEvent(),
"enable"))
+        if (len(fileContents) > 0):
+            return fileContents[0].rstrip()
+        else:
+            return ""
+
+    def getFilter(self):
+        """
+        Returns the contents of the file "filter" in the trace event
+        directory. Each line in the file is appened to an array that will be
+        returned.
+
+        @return: Returns the contents of the file "filter" in the trace event
+        directory.
+        @rtype: Array
+        """
+        return self.__getEventPathItem(os.path.join(self.getPathToTraceEvent(),
"filter"))
+
+    def getFormat(self):
+        """
+        Returns the contents of the file "format" in the trace event
+        directory. Each line in the file is appened to an array that will be
+        returned.
+
+        @return: Returns the contents of the file "format" in the trace event
+        directory.
+        @rtype: Array
+        """
+        return self.__getEventPathItem(os.path.join(self.getPathToTraceEvent(),
"format"))
+
+    def getID(self):
+        """
+        Returns the contents of the file "id" in the trace event directory.
+
+        @return: Returns the contents of the file "id" in the trace event
+        directory.
+        @rtype: String
+        """
+        fileContents = self.__getEventPathItem(os.path.join(self.getPathToTraceEvent(),
"id"))
+        if (len(fileContents) > 0):
+            return fileContents[0].rstrip()
+        else:
+            return ""
+
+    def setEventEnable(self, eventEnableString):
+        """
+        Set the trace event to either enabled(1) or disabled(0) by writing to
+        the trace event's "enable" file.
+
+        @param eventEnableString: The value of the string should be 1 for
+        enabled or 0 for disabled.
+        @param eventEnableString: String
+        """
+        if ((eventEnableString == "0") or (eventEnableString ==
"1")):
+            self.__setEventPathItem(os.path.join(self.getPathToTraceEvent(),
"enable"), eventEnableString, appendToFile=False)
+        else:
+            message = "The trace event \"enable\" file only accepts the
values of 0 or 1. The value %s will not be written: %s." %(eventEnableString)
+            logging.getLogger(MAIN_LOGGER_NAME).error(message)
+
+class TraceEvents:
+    """
+    A class that is a container for multiple trace events that are located in a
+    directory.
+    """
+    def __init__(self, pathToTraceEvents):
+        """
+        @param pathToTraceEvents: The path to the directory that contains trace
+        events.
+        @type pathToTraceEvents: String
+        """
+        self.__pathToTraceEvents = pathToTraceEvents
+        self.__traceEventsMap = self.__generateTraceEvents()
+
+    def __generateTraceEvents(self):
+        """
+        Generates a map of all the trace events.
+
+        @return: Returns a map of all the TraceEvent found.
+        @rtype: Dict
+        """
+        traceEventsMap = {}
+        if (not os.path.exists(self.__pathToTraceEvents)):
+            message = "The path does not exist: %s"
%(self.__pathToTraceEvents)
+            logging.getLogger(MAIN_LOGGER_NAME).error(message)
+            return traceEventsMap
+        elif (os.path.isdir(self.__pathToTraceEvents)):
+            dirlist = []
+            try:
+                dirlist = os.listdir(self.__pathToTraceEvents)
+            except OSError:
+                message = "There was error listing contents of the directory:
%s" %(self.__pathToTraceEvents)
+                logging.getLogger(MAIN_LOGGER_NAME).error(message)
+            for item in dirlist:
+                pathToItem = os.path.join(PATH_TO_GFS2_TRACE_EVENTS_DIR, item)
+                if (os.path.isdir(pathToItem)):
+                    traceEvent = TraceEvent(pathToItem)
+                    traceEventsMap[traceEvent.getName()] = traceEvent
+        return traceEventsMap
+
+    def getPathToTraceEvents(self):
+        """
+        Returns the path to the directory that contains all the trace events.
+
+        @return: Return the path to the directory that contains all the trace
+        events.
+        @rtype: String
+        """
+        return self.__pathToTraceEvents
+
+    def getTraceEventNames(self):
+        """
+        Returns a list of all the trace event names found.
+
+        @return: Returns a list of all the trace event names found.
+        @rtype: Array
+        """
+        return self.__traceEventsMap.keys()
+
+    def getTraceEvent(self, traceEventName):
+        """
+        Returns a TraceEvent that matches the traceEventName. If no match is
+        found then None is returned.
+
+        @return: Returns a TraceEvent that matches the traceEventName. If no
+        match is found then None is returned.
+        @rtype: TraceEvent
+        """
+        if (self.__traceEventsMap.has_key(traceEventName)):
+            return self.__traceEventsMap.get(traceEventName)
+        return None
+
+# #####################################################################
+# Helper functions.
+# #####################################################################
+def runCommand(command, listOfCommandOptions, standardOut=subprocess.PIPE,
standardError=subprocess.PIPE):
+    """
+    This function will execute a command. It will return True if the return code
+    was zero, otherwise False is returned.
+
+    @return: Returns True if the return code was zero, otherwise False is
+    returned.
+    @rtype: Boolean
+
+    @param command: The command that will be executed.
+    @type command: String
+    @param listOfCommandOptions: The list of options for the command that will
+    be executed.
+    @type listOfCommandOptions: Array
+    @param standardOut: The pipe that will be used to write standard output. By
+    default the pipe that is used is subprocess.PIPE.
+    @type standardOut: Pipe
+    @param standardError: The pipe that will be used to write standard error. By
+    default the pipe that is used is subprocess.PIPE.
+    @type standardError: Pipe
+    """
+    stdout = ""
+    stderr = ""
+    try:
+        commandList = [command]
+        commandList += listOfCommandOptions
+        task = subprocess.Popen(commandList, stdout=standardOut, stderr=standardError)
+        task.wait()
+        (stdout, stderr) = task.communicate()
+        return (task.returncode == 0)
+    except OSError:
+        commandOptionString = ""
+        for option in listOfCommandOptions:
+            commandOptionString += "%s " %(option)
+        message = "An error occurred running the command: $ %s %s\n" %(command,
commandOptionString)
+        if (len(stdout) > 0):
+            message += stdout
+        message += "\n"
+        if (len(stderr) > 0):
+            message += stderr
+        logging.getLogger(MAIN_LOGGER_NAME).error(message)
+    return False
+
+def mountFilesystem(filesystemType, pathToDevice, pathToMountPoint):
+    """
+    This function will attempt to mount a filesystem. If the filesystem is
+    already mounted or the filesystem was successfully mounted then True is
+    returned, otherwise False is returned.
+
+    @return: If the filesystem is already mounted or the filesystem was
+    successfully mounted then True is returned, otherwise False is returned.
+    @rtype: Boolean
+
+    @param filesystemType: The type of filesystem that will be mounted.
+    @type filesystemType: String
+    @param pathToDevice: The path to the device that will be mounted.
+    @type pathToDevice: String
+    @param pathToMountPoint: The path to the directory that will be used as the
+    mount point for the device.
+    @type pathToMountPoint: String
+    """
+    if (os.path.ismount(PATH_TO_DEBUG_DIR)):
+        return True
+    listOfCommandOptions = ["-t", filesystemType, pathToDevice,
pathToMountPoint]
+    if (not runCommand("mount", listOfCommandOptions)):
+        message = "There was an error mounting the filesystem type %s for the device
%s to the mount point %s." %(filesystemType, pathToDevice, pathToMountPoint)
+        logging.getLogger(MAIN_LOGGER_NAME).error(message)
+    return  os.path.ismount(PATH_TO_DEBUG_DIR)
+
+def exitScript(removePidFile=True, errorCode=0):
+    """
+    This function will cause the script to exit or quit. It will return an error
+    code and will remove the pid file that was created.
+
+    @param removePidFile: If True(default) then the pid file will be remove
+    before the script exits.
+    @type removePidFile: Boolean
+    @param errorCode: The exit code that will be returned. The default value is 0.
+    @type errorCode: Int
+    """
+    if (removePidFile):
+        message = "Removing the pid file: %s" %(PATH_TO_PID_FILENAME)
+        logging.getLogger(MAIN_LOGGER_NAME).debug(message)
+        if (os.path.exists(PATH_TO_PID_FILENAME)):
+            try:
+                os.remove(PATH_TO_PID_FILENAME)
+            except IOError:
+                message = "There was an error removing the file: %s."
%(PATH_TO_PID_FILENAME)
+                logging.getLogger(MAIN_LOGGER_NAME).error(message)
+    message = "The script will exit."
+    logging.getLogger(MAIN_LOGGER_NAME).info(message)
+    sys.exit(errorCode)
+
+def getMountedGFS2Filesystems():
+    """
+    This function returns a list of all the mounted GFS2 filesystems.
+
+    @return: Returns a list of all the mounted GFS2 filesystems.
+    @rtype: Array
+    """
+    fsType = "gfs2"
+    listOfMountedFilesystems = []
+    dataOutput = FileUtils.getDataFromFile("/proc/mounts")
+    if (not dataOutput == None):
+        for line in dataOutput:
+            splitLine = line.split()
+            if (len(splitLine) > 0):
+                if (splitLine[2] == fsType):
+                    listOfMountedFilesystems.append(line)
+    return listOfMountedFilesystems
+# ##############################################################################
+# Get user selected options
+# ##############################################################################
+def __getOptions(version) :
+    """
+    This function creates the OptionParser and returns commandline
+    a tuple of the selected commandline options and commandline args.
+
+    The cmdlineOpts which is the options user selected and cmdLineArgs
+    is value passed and  not associated with an option.
+
+    @return: A tuple of the selected commandline options and commandline args.
+    @rtype: Tuple
+
+    @param version: The version of the this script.
+    @type version: String
+    """
+    cmdParser = OptionParserExtended(version)
+    cmdParser.add_option("-d", "--debug",
+                         action="store_true",
+                         dest="enableDebugLogging",
+                         help="enables debug logging",
+                         default=False)
+    cmdParser.add_option("-q", "--quiet",
+                         action="store_true",
+                         dest="disableLoggingToConsole",
+                         help="disables logging to console",
+                         default=False)
+    cmdParser.add_option("-l", "--list",
+                         action="store_true",
+                         dest="listTraceEvents",
+                         help="lists the enabled state and filters for the GFS2
trace events",
+                         default=False)
+    cmdParser.add_option("-E", "--enable_all_trace_events",
+                         action="store_true",
+                         dest="enableAllTraceEvents",
+                         help="enables all trace_events for GFS2",
+                         default=False)
+    cmdParser.add_option("-e", "--enable_trace_event",
+                         action="extend",
+                         dest="enableTraceEventsList",
+                         help="selected trace_events that will be enabled for
GFS2",
+                         type="string",
+                         metavar="<trace event name>",
+                         default=[])
+    cmdParser.add_option("-N", "--disable_all_trace_events",
+                         action="store_true",
+                         dest="disableAllTraceEvents",
+                         help="disables all trace_events for GFS2",
+                         default=False)
+    cmdParser.add_option("-n", "--disable_trace_event",
+                         action="extend",
+                         dest="disableTraceEventsList",
+                         help="selected trace_events that will be disabled for
GFS2",
+                         type="string",
+                         metavar="<trace event name>",
+                         default=[])
+    cmdParser.add_option("-c", "--capture",
+                         action="store",
+                         dest="pathToOutputFilename",
+                         help="enables capturing of trace events and will save the
data to a file",
+                         type="string",
+                         metavar="<output filename>",
+                         default="")
+
+    (cmdLineOpts, cmdLineArgs) = cmdParser.parse_args()
+    return (cmdLineOpts, cmdLineArgs)
+
+# ##############################################################################
+# OptParse classes for commandline options
+# ##############################################################################
+class OptionParserExtended(OptionParser):
+    """
+    This is the class that gets the command line options the end user
+    selects.
+    """
+    def __init__(self, version) :
+        """
+        @param version: The version of the this script.
+        @type version: String
+        """
+        self.__commandName = os.path.basename(sys.argv[0])
+        versionMessage = "%s %s\n" %(self.__commandName, version)
+
+        commandDescription  ="%s can enable trace events, disable trace events, and
capture data from GFS2 trace events.\n"%(self.__commandName)
+        OptionParser.__init__(self, option_class=ExtendOption,
+                              version=versionMessage,
+                              description=commandDescription)
+
+    def print_help(self):
+        """
+        Print examples at the bottom of the help message.
+        """
+        exampleMessage = "\nExamples:\n"
+        exampleMessage += "To list the enable status and filter for each trace
event.\n"
+        exampleMessage += "# %s -l\n\n" %(self.__commandName)
+        exampleMessage += "To disable all trace events.\n"
+        exampleMessage += "# %s -N\n\n" %(self.__commandName)
+        exampleMessage += "To enable all trace events.\n"
+        exampleMessage += "# %s -E\n\n" %(self.__commandName)
+        exampleMessage += "To disable all trace events and then enable a couple
trace events.\n"
+        exampleMessage += "# %s -N -e
gfs2_demote_rq,gfs2_glock_state_change,gfs2_promote\n\n" %(self.__commandName)
+        exampleMessage += "To capture all the trace events and write to the file
/tmp/gfs2_trace.log.\n"
+        exampleMessage += "# %s -c /tmp/gfs2_trace.log\n\n"
%(self.__commandName)
+        exampleMessage += "To disable all trace events and then enable a couple
trace events and capture the output to a file.\n"
+        exampleMessage += "# %s -N -e
gfs2_demote_rq,gfs2_glock_state_change,gfs2_promote -c /tmp/gfs2_trace.log\n"
%(self.__commandName)
+        self.print_version()
+        OptionParser.print_help(self)
+        print exampleMessage
+
+class ExtendOption (Option):
+        """
+        Allow to specify comma delimited list of entries for arrays
+        and dictionaries.
+        """
+        ACTIONS = Option.ACTIONS + ("extend",)
+        STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",)
+        TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",)
+
+        def take_action(self, action, dest, opt, value, values, parser):
+            """
+            This function is a wrapper to take certain options passed on command
+            prompt and wrap them into an Array.
+
+            @param action: The type of action that will be taken. For example:
+            "store_true", "store_false", "extend".
+            @type action: String
+            @param dest: The name of the variable that will be used to store the
+            option.
+            @type dest: String/Boolean/Array
+            @param opt: The option string that triggered the action.
+            @type opt: String
+            @param value: The value of opt(option) if it takes a
+            value, if not then None.
+            @type value:
+            @param values: All the opt(options) in a dictionary.
+            @type values: Dictionary
+            @param parser: The option parser that was orginally called.
+            @type parser: OptionParser
+            """
+            if (action == "extend") :
+                valueList=[]
+                try:
+                    for v in value.split(","):
+                        # Need to add code for dealing with paths if there is option for
paths.
+                        valueList.append(v)
+                except:
+                    pass
+                else:
+                    values.ensure_value(dest, []).extend(valueList)
+            else:
+                Option.take_action(self, action, dest, opt, value, values, parser)
+
+# ###############################################################################
+# Main Function
+# ###############################################################################
+if __name__ == "__main__":
+    try:
+        # #######################################################################
+        # Get the options from the commandline.
+        # #######################################################################
+        (cmdLineOpts, cmdLineArgs) = __getOptions(VERSION_NUMBER)
+
+        # #######################################################################
+        # Setup the logger
+        # #######################################################################
+        # Create the logger
+        logLevel = logging.INFO
+        logger = logging.getLogger(MAIN_LOGGER_NAME)
+        logger.setLevel(logLevel)
+        # Create a new status function and level.
+        logging.STATUS = logging.INFO + 2
+        logging.addLevelName(logging.STATUS, "STATUS")
+        # Create a function for the STATUS_LEVEL since not defined by python. This
+        # means you can call it like the other predefined message
+        # functions. Example: logging.getLogger("loggerName").status(message)
+        setattr(logger, "status", lambda *args: logger.log(logging.STATUS,
*args))
+        streamHandler = logging.StreamHandler()
+        streamHandler.setFormatter(logging.Formatter("%(levelname)s
%(message)s"))
+        logger.addHandler(streamHandler)
+
+        # Set options on logger for debugging or no logging.
+        if (cmdLineOpts.disableLoggingToConsole):
+            logging.disable(logging.CRITICAL)
+        elif (cmdLineOpts.enableDebugLogging) :
+            logging.getLogger(MAIN_LOGGER_NAME).setLevel(logging.DEBUG)
+            message = "Debugging has been enabled."
+            logging.getLogger(MAIN_LOGGER_NAME).debug(message)
+
+        # #######################################################################
+        # Check to see if pid file exists and error if it does.
+        # #######################################################################
+        if (os.path.exists(PATH_TO_PID_FILENAME)):
+            message = "The PID file %s already exists and this script cannot run
till it does not exist." %(PATH_TO_PID_FILENAME)
+            logging.getLogger(MAIN_LOGGER_NAME).error(message)
+            message = "Verify that there are no other existing processes running. If
there are running processes those need to be stopped first and the file removed."
+            logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            exitScript(removePidFile=False, errorCode=1)
+        else:
+            message = "Creating the pid file: %s" %(PATH_TO_PID_FILENAME)
+            logging.getLogger(MAIN_LOGGER_NAME).debug(message)
+            # Creata the pid file so we dont have more than 1 process of this
+            # script running.
+            FileUtils.writeToFile(PATH_TO_PID_FILENAME, str(os.getpid()),
createFile=True)
+
+        # #######################################################################
+        # Check to see if there any GFS2 filesystems mounted, if not then exit.
+        # #######################################################################
+        if (not len(getMountedGFS2Filesystems()) > 0):
+            message = "There was no GFS2 file-systems mounted."
+            logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            exitScript(errorCode=1)
+
+        # #######################################################################
+        # Check to see if the debug directory is mounted. If not then
+        # log an error.
+        # #######################################################################
+        if(mountFilesystem("debugfs", "none", PATH_TO_DEBUG_DIR)):
+            message = "The debug filesystem %s is mounted."
%(PATH_TO_DEBUG_DIR)
+            logging.getLogger(MAIN_LOGGER_NAME).debug(message)
+        else:
+            message = "There was a problem mounting the debug filesystem: %s"
%(PATH_TO_DEBUG_DIR)
+            logging.getLogger(MAIN_LOGGER_NAME).error(message)
+            message = "The debug filesystem is required to be mounted for this
script to run."
+            logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            exitScript(errorCode=1)
+
+        # #######################################################################
+        # List of the enable state and filters for each trace event
+        # #######################################################################
+        traceEvents = TraceEvents(PATH_TO_GFS2_TRACE_EVENTS_DIR)
+        listOfTraceEventNames = traceEvents.getTraceEventNames()
+
+        if (cmdLineOpts.listTraceEvents):
+            listOfTraceEventNames.sort()
+            maxTraceEventNameSize = len(max(listOfTraceEventNames, key=len))
+            traceEventsString = ""
+            for traceEventName in listOfTraceEventNames:
+                traceEvent = traceEvents.getTraceEvent(traceEventName)
+                if (not traceEventName == None):
+                    traceEventEnableStatus = "UNKNOWN"
+                    if (traceEvent.getEnable() == "0"):
+                        traceEventEnableStatus = "DISABLED"
+                    elif (traceEvent.getEnable() == "1"):
+                        traceEventEnableStatus = "ENABLED"
+                    whitespaces = ""
+                    for i in range(0, (maxTraceEventNameSize - len(traceEventName))):
+                        whitespaces += " "
+                    traceEventsString += "%s    %s%s\n" %(traceEventName,
whitespaces, traceEventEnableStatus)
+            # Disable logging to console except for debug when we print into information
to console.
+            logging.disable(logging.CRITICAL)
+            if (len(traceEventsString) > 0):
+                print "trace event name           trace event status"
+                print "----------------           ------------------"
+                print traceEventsString.rstrip()
+            exitScript()
+        # #######################################################################
+        # Enable or Disable Trace Events
+        # #######################################################################
+        if (cmdLineOpts.disableAllTraceEvents):
+            message = "Disabling all trace events."
+            logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            for traceEventName in listOfTraceEventNames:
+                traceEvent = traceEvents.getTraceEvent(traceEventName)
+                if (not traceEvent == None):
+                    traceEvent.setEventEnable("0")
+        elif (cmdLineOpts.enableAllTraceEvents):
+            message = "Enabling all trace events."
+            logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            for traceEventName in listOfTraceEventNames:
+                traceEvent = traceEvents.getTraceEvent(traceEventName)
+                if (not traceEvent == None):
+                    traceEvent.setEventEnable("1")
+
+        if (len(cmdLineOpts.disableTraceEventsList) > 0):
+            for traceEventName in cmdLineOpts.disableTraceEventsList:
+                traceEvent = traceEvents.getTraceEvent(traceEventName)
+                if (not traceEvent == None):
+                    message = "Disabling the selected trace event: %s"
%(traceEvent.getName())
+                    logging.getLogger(MAIN_LOGGER_NAME).info(message)
+                    traceEvent.setEventEnable("0")
+        if (len(cmdLineOpts.enableTraceEventsList) > 0):
+            for traceEventName in cmdLineOpts.enableTraceEventsList:
+                traceEvent = traceEvents.getTraceEvent(traceEventName)
+                if (not traceEvent == None):
+                    message = "Enabling the selected trace event: %s"
%(traceEvent.getName())
+                    logging.getLogger(MAIN_LOGGER_NAME).info(message)
+                    traceEvent.setEventEnable("1")
+
+        # #######################################################################
+        # Capture the data generate by the trace events.
+        # #######################################################################
+        if (len(cmdLineOpts.pathToOutputFilename) > 0):
+            # Read from tracing pipe and write the output to a file.
+            message = "The capturing of the trace events that were enabled to a file
will be started by reading the the trace pipe: %s." %(PATH_TO_TRACE_PIPE)
+            logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            message = "Leave this script running until you have capture all the
data, then hit control-c to exit."
+            logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            try:
+                fout = open(cmdLineOpts.pathToOutputFilename, "w")
+                for line in fileinput.input(PATH_TO_TRACE_PIPE):
+                    fout.write(line)
+                fout.close()
+                message = "The data was written to this file: %s"
%(cmdLineOpts.pathToOutputFilename)
+                logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            except KeyboardInterrupt:
+                message = "A control-c was detected and the capturing of trace
events data will stop."
+                logging.getLogger(MAIN_LOGGER_NAME).info(message)
+                fout.close()
+                message = "The data was written to this file: %s"
%(cmdLineOpts.pathToOutputFilename)
+                logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            except UnicodeEncodeError, e:
+                message = "There was a unicode encode error writing to the file:
%s." %(cmdLineOpts.pathToOutputFilename)
+                logging.getLogger(MAIN_LOGGER_NAME).error(message)
+            except IOError:
+                message = "There was an error writing to the file: %s."
%(cmdLineOpts.pathToOutputFilename)
+                logging.getLogger(MAIN_LOGGER_NAME).error(message)
+            message = "The capturing of the trace event data has completed."
+            logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            # Compress the file so that it will have a smaller file size.
+            pathToTarFilename = "%s.tar.bz2"
%(os.path.splitext(cmdLineOpts.pathToOutputFilename)[0])
+            message = "Creating a compressed archvied file: %s"
%(pathToTarFilename)
+            logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            try:
+                tar = tarfile.open(pathToTarFilename, "w:bz2")
+                tar.add(cmdLineOpts.pathToOutputFilename,
arcname=os.path.basename(cmdLineOpts.pathToOutputFilename))
+                tar.close()
+                message = "The compressed archvied file was created: %s"
%(pathToTarFilename)
+                logging.getLogger(MAIN_LOGGER_NAME).info(message)
+            except tarfile.TarError:
+                message = "There was an error creating the tarfile: %s."
%(pathToTarFilename)
+                logging.getLogger(MAIN_LOGGER_NAME).error(message)
+    except KeyboardInterrupt:
+        print ""
+        message =  "This script will exit since control-c was executed by end
user."
+        logging.getLogger(MAIN_LOGGER_NAME).error(message)
+        exitScript()
+    # #######################################################################
+    # Exit the application with zero exit code since we cleanly exited.
+    # #######################################################################
+    exitScript()