r4475 - in branches/1.3.2/cumin: bin etc python/cumin

Author: tmckay
Date: 2011-01-18 20:59:25 +0000 (Tue, 18 Jan 2011)
New Revision: 4475

Added:
   branches/1.3.2/cumin/bin/cumin
Removed:
   branches/1.3.2/cumin/bin/cumin
Modified:
   branches/1.3.2/cumin/bin/cumin-admin
   branches/1.3.2/cumin/bin/cumin-bench
   branches/1.3.2/cumin/bin/cumin-data
   branches/1.3.2/cumin/bin/cumin-smoke-test
   branches/1.3.2/cumin/bin/cumin-web
   branches/1.3.2/cumin/etc/sysvinit-cumin
   branches/1.3.2/cumin/python/cumin/config.py
Log:
Merge in configuration changes from multiples branch.

Revisions 4467 through 4474


Deleted: branches/1.3.2/cumin/bin/cumin
===================================================================
--- branches/1.3.2/cumin/bin/cumin	2011-01-18 20:51:52 UTC (rev 4474)
+++ branches/1.3.2/cumin/bin/cumin	2011-01-18 20:59:25 UTC (rev 4475)
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-function die {
-    kill "$data" "$web"
-}
-
-trap die EXIT
-
-function start_data {
-    cumin-data &
-    data="$!"
-}
-
-function start_web {
-    cumin-web &
-    web="$!"
-}
-
-start_data
-start_web
-
-while [ 0 ]; do
-    sleep 30
-
-    if [ ! -d "/proc/$data" ]; then
-        start_data
-    fi
-
-    if [ ! -d "/proc/$web" ]; then
-        start_web
-    fi
-done

Copied: branches/1.3.2/cumin/bin/cumin (from rev 4474,
branches/multiples/cumin/bin/cumin)
===================================================================
--- branches/1.3.2/cumin/bin/cumin	                        (rev 0)
+++ branches/1.3.2/cumin/bin/cumin	2011-01-18 20:59:25 UTC (rev 4475)
@@ -0,0 +1,74 @@
+#!/usr/bin/python
+
+from optparse import OptionParser
+from time import sleep
+import subprocess
+import os
+import sys
+import signal
+
+def call_sys_exit(sig,frame):
+    sys.exit()
+
+def set_exit_handler(func):
+    if os.name == "nt":
+        try:
+            import win32api
+            win32api.SetConsoleCtrlHandler(func, True)
+        except ImportError:
+            version = ".".join(map(str, sys.version_info[:2]))
+            raise Exception("pywin32 not installed for Python " + version)
+    else:
+        signal.signal(signal.SIGTERM, func)
+
+def main():
+
+    # tuple indices, for clarity
+    PROCESS = 0
+    SECTION = 1
+    PROGRAM = 2
+
+    parser = OptionParser()
+
+    parser.add_option("--webs", dest="webs",
default="web",
+                      help="Configuration section names for cumin-web
instances."\
+                           "\nEach value implies a separate cumin-web
instance.")
+
+    parser.add_option("--datas", dest="datas",
default="data",
+                      help="Configuration section names for cumin-data
instances."\
+                           "\nEach value implies a separate cumin-data
instance.")
+
+    (options, args) = parser.parse_args()
+
+    # Get our list cumin-web and data instances
+    # create list elements to hold the process object, section arg, and app
+    apps = []
+    for app in options.webs.split(','):
+        apps.append([None, app, "cumin-web"])
+
+    for app in options.datas.split(','):
+        apps.append([None, app, "cumin-data"])
+
+    # Launch and babysit
+    try:
+        while True:
+            for app in apps:
+                if not app[PROCESS] or app[PROCESS].poll():
+                    arg = "--section=" + app[SECTION]
+                    app[PROCESS] = subprocess.Popen([app[PROGRAM], arg]) 
+            sleep(30)
+
+    finally:
+        for app in apps:
+            if app[PROCESS]:
+	        os.kill(app[PROCESS].pid, signal.SIGTERM)
+    
+if __name__ == "__main__":
+    # This is necessary so that on a SIGTERM we can call sys.exit()
+    # and cause the finally block to be executed.  Ctrl-C will
+    # run the finally block anyway.
+    set_exit_handler(call_sys_exit)
+    try:
+        main()
+    except KeyboardInterrupt:
+        pass

Modified: branches/1.3.2/cumin/bin/cumin-admin
===================================================================
--- branches/1.3.2/cumin/bin/cumin-admin	2011-01-18 20:51:52 UTC (rev 4474)
+++ branches/1.3.2/cumin/bin/cumin-admin	2011-01-18 20:59:25 UTC (rev 4475)
@@ -24,14 +24,10 @@
     if uid == 0:
         os.setuid(file_uid)
 
-    config = CuminConfig()
-    values = config.parse()
-
-    parser = CuminOptionParser(values.data)
+    parser = CuminDataOptionParser()
     parser.set_usage(generate_usage())
+    config, values, opts, args = get_configuration(parser, CuminDataConfig)
 
-    opts, args = parser.parse_args()
-
     try:
         name = args[0]
     except IndexError:
@@ -50,7 +46,7 @@
 
     broker_uris = [x.strip() for x in opts.brokers.split(",")]
 
-    app = Cumin(config.home, broker_uris, opts.database)
+    app = Cumin(config.get_home(), broker_uris, opts.database)
 
     app.check()
     app.init()

Modified: branches/1.3.2/cumin/bin/cumin-bench
===================================================================
--- branches/1.3.2/cumin/bin/cumin-bench	2011-01-18 20:51:52 UTC (rev 4474)
+++ branches/1.3.2/cumin/bin/cumin-bench	2011-01-18 20:59:25 UTC (rev 4475)
@@ -8,22 +8,17 @@
 from cumin.config import *
 
 def do_main():
-    config = CuminConfig()
-    values = config.parse()
+    config, values, opts, args = get_configuration(CuminWebOptionParser(),
CuminWebConfig)
 
-    parser = CuminOptionParser(values.web)
-
-    opts, args = parser.parse_args()
-
     setup_logging(opts)
 
     broker_uris = [x.strip() for x in opts.brokers.split(",")]
 
-    cumin = Cumin(config.home, broker_uris, opts.database)
+    cumin = Cumin(config.get_home(), broker_uris, opts.database)
 
     cumin.debug = True
-    cumin.user = values.web.user
-    cumin.update_interval = values.web.update_interval
+    cumin.user = values.user
+    cumin.update_interval = values.update_interval
 
     cumin.check()
     cumin.init()

Modified: branches/1.3.2/cumin/bin/cumin-data
===================================================================
--- branches/1.3.2/cumin/bin/cumin-data	2011-01-18 20:51:52 UTC (rev 4474)
+++ branches/1.3.2/cumin/bin/cumin-data	2011-01-18 20:59:25 UTC (rev 4475)
@@ -11,15 +11,8 @@
 from mint import *
 
 def main():
-    config = CuminConfig()
-    values = config.parse().data
+    config, values, opts, args = get_configuration(CuminDataOptionParser(),
CuminDataConfig)
 
-    parser = CuminOptionParser(values)
-    parser.add_option("--print-stats", action="store_true")
-    parser.add_option("--print-events", type="int", default=0,
metavar="LEVEL")
-
-    opts, args = parser.parse_args()
-
     setup_logging(opts)
 
     model_dir = os.path.join(config.home, "model")

Modified: branches/1.3.2/cumin/bin/cumin-smoke-test
===================================================================
--- branches/1.3.2/cumin/bin/cumin-smoke-test	2011-01-18 20:51:52 UTC (rev 4474)
+++ branches/1.3.2/cumin/bin/cumin-smoke-test	2011-01-18 20:59:25 UTC (rev 4475)
@@ -13,22 +13,16 @@
 from cumin.util import *
 
 def main():
-    config = CuminConfig()
-    values = config.parse()
+    config, values, opts, args = get_configuration(CuminWebOptionParser(),
CuminWebConfig)
 
-    parser = CuminOptionParser(values.web)
-
-    parser.add_option("--host", default=values.web.host)
-    parser.add_option("--port", default=values.web.port)
-
-    opts, args = parser.parse_args()
-
     setup_logging(opts)
 
-    cumin = Cumin(config.home, opts.brokers, opts.database,
+    broker_uris = [x.strip() for x in opts.brokers.split(",")]
+
+    cumin = Cumin(config.get_home(), broker_uris, opts.database,
                   opts.host, opts.port)
 
-    cumin.user = values.web.user
+    cumin.user = values.user
 
     cumin.check()
     cumin.init()

Modified: branches/1.3.2/cumin/bin/cumin-web
===================================================================
--- branches/1.3.2/cumin/bin/cumin-web	2011-01-18 20:51:52 UTC (rev 4474)
+++ branches/1.3.2/cumin/bin/cumin-web	2011-01-18 20:59:25 UTC (rev 4475)
@@ -11,32 +11,24 @@
 from cumin.util import *
 
 def main():
-    config = CuminConfig()
-    values = config.parse()
+    config, values, opts, args = get_configuration(CuminWebOptionParser(),
CuminWebConfig)
 
-    parser = CuminOptionParser(values.web)
-
-    parser.add_option("--host", default=values.web.host)
-    parser.add_option("--port", default=values.web.port, type=int)
-
-    opts, args = parser.parse_args()
-
     setup_logging(opts)
 
     broker_uris = [x.strip() for x in opts.brokers.split(",")]
 
-    cumin = Cumin(config.home, broker_uris, opts.database,
+    cumin = Cumin(config.get_home(), broker_uris, opts.database, 
                   opts.host, opts.port)
 
     cumin.debug = opts.debug
-    cumin.user = values.web.user
-    cumin.update_interval = values.web.update_interval
+    cumin.user = values.user
+    cumin.update_interval = values.update_interval
 
     # set default values for form inputs
-    cumin.set_form_defaults(values.web.request_memory,
-                            values.web.request_memory_vm,
-                            values.web.request_disk,
-                            values.web.request_disk_vm)
+    cumin.set_form_defaults(values.request_memory,
+                            values.request_memory_vm,
+                            values.request_disk,
+                            values.request_disk_vm)
 
     cumin.check()
     cumin.init()

Modified: branches/1.3.2/cumin/etc/sysvinit-cumin
===================================================================
--- branches/1.3.2/cumin/etc/sysvinit-cumin	2011-01-18 20:51:52 UTC (rev 4474)
+++ branches/1.3.2/cumin/etc/sysvinit-cumin	2011-01-18 20:59:25 UTC (rev 4475)
@@ -11,6 +11,9 @@
 test -x /usr/bin/cumin || exit 1
 test -x /usr/bin/cumin-database || exit 1
 
+SYSCONFIG="/etc/sysconfig/cumin"
+[ -r "$SYSCONFIG" ] && source "$SYSCONFIG"
+
 cumin-database check-started &> /dev/null || {
     echo "Cumin's database server is not running"
     echo "Run 'cumin-database start' as root"
@@ -34,7 +37,7 @@
 
 start() {
     echo -n $"Starting cumin: "
-    daemon --user cumin --check $servicename $processname \&
+    daemon --user cumin --check $servicename $processname $CUMIN_OPTIONS \&
     RETVAL=$?
     echo
     [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$servicename

Modified: branches/1.3.2/cumin/python/cumin/config.py
===================================================================
--- branches/1.3.2/cumin/python/cumin/config.py	2011-01-18 20:51:52 UTC (rev 4474)
+++ branches/1.3.2/cumin/python/cumin/config.py	2011-01-18 20:59:25 UTC (rev 4475)
@@ -17,9 +17,28 @@
         if not os.path.isdir(self.home):
             raise Exception("Home path '%s' is not a directory")
 
-        web = CuminConfigSection(self, "web")
-        web.log_file.default = os.path.join(self.home, "log",
"web.log")
+    def get_home(self):
+        return self.home
 
+    def parse(self):
+        paths = list()
+
+        paths.append(os.path.join(os.sep, "etc", "cumin",
"cumin.conf"))
+        paths.append(os.path.join(self.home, "etc", "cumin.conf"))
+        paths.append(os.path.join(os.path.expanduser("~"),
".cumin.conf"))
+
+        return self.parse_files(paths)
+
+class CuminWebConfig(CuminConfig):
+    def __init__(self, section_name):
+        super(CuminWebConfig, self).__init__()
+
+        self.create_section(section_name)
+
+    def create_section(self, name):
+        web = CuminConfigSection(self, name)
+        web.log_file.default = os.path.join(self.home, "log", name +
".log")
+
         param = ConfigParameter(web, "update-interval", int)
         param.default = 10
 
@@ -42,8 +61,15 @@
         param = ConfigParameter(web, "request-disk-vm",   int)
         param.default = 5 * 1024 #MB
 
-        data = CuminConfigSection(self, "data")
-        data.log_file.default = os.path.join(self.home, "log",
"data.log")
+class CuminDataConfig(CuminConfig):
+    def __init__(self, section_name):
+        super(CuminDataConfig, self).__init__()
+
+        self.create_section(section_name)
+
+    def create_section(self, name):
+        data = CuminConfigSection(self, name)
+        data.log_file.default = os.path.join(self.home, "log", name +
".log")
         
         param = ConfigParameter(data, "packages", str)
 
@@ -56,15 +82,6 @@
         param = ConfigParameter(data, "vacuum-interval", int)
         param.default = 60 * 60 # 1 hour
 
-    def parse(self):
-        paths = list()
-
-        paths.append(os.path.join(os.sep, "etc", "cumin",
"cumin.conf"))
-        paths.append(os.path.join(self.home, "etc", "cumin.conf"))
-        paths.append(os.path.join(os.path.expanduser("~"),
".cumin.conf"))
-
-        return self.parse_files(paths)
-
 class CuminConfigSection(ConfigSection):
     def __init__(self, config, name):
         super(CuminConfigSection, self).__init__(config, name)
@@ -83,17 +100,59 @@
         param = ConfigParameter(self, "debug", bool)
         param.default = False
 
-class CuminOptionParser(OptionParser):
-    def __init__(self, section):
+class CuminMultiOptionParser(OptionParser,object):
+    def __init__(self, default_section):
         OptionParser.__init__(self)
 
-        self.add_option("--database", default=section.database)
-        self.add_option("--brokers", default=section.brokers)
-        self.add_option("--log-file", default=section.log_file)
-        self.add_option("--log-level", default=section.log_level)
-        self.add_option("--debug", action="store_true",
default=section.debug)
+        self.add_option("--section", default=default_section)
+
+        # We don't know defaults yet...          
+        self.add_option("--database")
+        self.add_option("--brokers")
+        self.add_option("--log-file")
+        self.add_option("--log-level")
+        self.add_option("--debug", action="store_true")
         self.add_option("--init-only", action="store_true")
 
+    def _apply_default(self, section, name):
+        try:
+            val = getattr(section, name)
+            self.set_default(name, val)
+        except:
+            pass
+
+    def set_defaults_from_section(self, section):
+        # It is possible that a section does not contain an updated
+        # default value for a particular option.  Skip exception.
+        self._apply_default(section, "database")
+        self._apply_default(section, "brokers")
+        self._apply_default(section, "log_file")
+        self._apply_default(section, "log_level")
+        self._apply_default(section, "debug")
+
+class CuminWebOptionParser(CuminMultiOptionParser):
+    def __init__(self, default_section="web"):
+        super(CuminWebOptionParser, self).__init__(default_section)
+
+        # Add additional parameters for web
+        self.add_option("--host")
+        self.add_option("--port", type=int)
+
+    def set_defaults_from_section(self, section):
+        super(CuminWebOptionParser, self).set_defaults_from_section(section)
+
+        # For web, these options also have defaults in config files
+        self._apply_default(section, "host")
+        self._apply_default(section, "port")
+
+class CuminDataOptionParser(CuminMultiOptionParser):
+    def __init__(self, default_section="data"):
+        super(CuminDataOptionParser, self).__init__(default_section)
+
+        # Add additional parameters for data
+        self.add_option("--print-stats", action="store_true")
+        self.add_option("--print-events", type="int", default=0,
metavar="LEVEL")
+        
 def setup_logging(values):
     modules = ("cumin", "mint", "parsley",
"rosemary", "wooly")
 
@@ -103,3 +162,21 @@
     if values.debug:
         for name in modules:
             enable_logging(name, "debug", sys.stderr)
+
+def get_configuration(parser, ConfigType):
+    # First parse should return a "section" value
+    opts, args = parser.parse_args()
+    if hasattr(opts, "section"):
+        # Create config object with the given section and parse config files
+        config = ConfigType(opts.section)
+        values = config.parse()
+        if hasattr(values, opts.section):
+            # Reset defaults and parse again
+            values = getattr(values, opts.section)
+            parser.set_defaults_from_section(values)
+            opts, args = parser.parse_args()
+        else:
+            raise Exception("Section ["+opts.section+"] not present in
configuration files.")
+    else:
+        raise Exception("No '--section' value available as option or
default")
+    return config, values, opts, args