[PATCH 1/8] nettestctl: Integration of config module

[Ondrej Lichtner]

From: Ondrej Lichtner <olichtne(a)redhat.com&gt;

nettestctl now reads the users configuration file ~/.lnst/lnst.conf

As we agreed, the loaded configs will be distributed as function
parameters, therefore I made the necessary changes to functions
process_recipe and get_recipe_result.

Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com&gt;
---
 Common/Config.py |  5 -----
 nettestctl.py    | 16 +++++++++++-----
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/Common/Config.py b/Common/Config.py
index 8a1ad56..8ef2452 100644
--- a/Common/Config.py
+++ b/Common/Config.py
@@ -24,12 +24,7 @@ class Config():
 
     def __init__(self):
         self._parser = ConfigParser(dict_type=dict)
-
-        # defaults.conf should contain all possible sections and options
-        # sections and options not listed there will be undefined which
-        # can cause problems
         self.options = dict()
-        self.load_config("default.conf")
 
     def get_config(self):
         return self.options
diff --git a/nettestctl.py b/nettestctl.py
index de07911..2fab067 100755
--- a/nettestctl.py
+++ b/nettestctl.py
@@ -21,6 +21,7 @@ from NetTest.NetTestResultSerializer import NetTestResultSerializer
 from Common.Logs import Logs
 from Common.LoggingServer import LoggingServer
 import Common.ProcessManager
+from Common.Config import Config
 
 def usage():
     """
@@ -45,7 +46,7 @@ def usage():
     sys.exit()
 
 def process_recipe(action, file_path, remoteexec, cleanup,
-                   res_serializer, packet_capture):
+                   res_serializer, packet_capture, config):
     nettestctl = NetTestController(os.path.realpath(file_path),
                                    remoteexec=remoteexec, cleanup=cleanup,
                                    res_serializer=res_serializer)
@@ -70,14 +71,14 @@ def print_summary(summary):
     logging.info("=====================================================")
 
 def get_recipe_result(args, file_path, remoteexec, cleanup,
-                      res_serializer, packet_capture):
+                      res_serializer, packet_capture, config):
     res_serializer.add_recipe(file_path)
     Logs.set_logging_root_path(file_path)
     loggingServer = LoggingServer(LoggingServer.DEFAULT_PORT,
                                   Logs.root_path, Logs.debug)
     loggingServer.start()
     res = process_recipe(args, file_path, remoteexec, cleanup,
-                         res_serializer, packet_capture)
+                         res_serializer, packet_capture, config)
     loggingServer.stop()
     return ((file_path, res))
 
@@ -97,6 +98,9 @@ def main():
         usage()
         sys.exit()
 
+    config = Config()
+    config.load_config('~/.lnst/lnst.conf')
+
     debug = 0
     recipe_path = None
     remoteexec = False
@@ -144,12 +148,14 @@ def main():
                     summary.append(get_recipe_result(action, recipe_file,
                                                      remoteexec, cleanup,
                                                      res_serializer,
-                                                     packet_capture))
+                                                     packet_capture,
+                                                     config))
                     Logs.set_logging_root_path(clean=False)
         else:
             summary.append(get_recipe_result(action, recipe_path, remoteexec,
                                              cleanup, res_serializer,
-                                             packet_capture))
+                                             packet_capture,
+                                             config))
 
     Logs.set_logging_root_path(clean=False)
 
-- 
1.7.11.4

[PATCH 2/8] Config: bug fix for mac range verification

[Ondrej Lichtner]

From: Ondrej Lichtner <olichtne(a)redhat.com&gt;

I made a small mistake in verification of mac range.
This commit fixes that.

Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com&gt;
---
 Common/Config.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Common/Config.py b/Common/Config.py
index 8ef2452..4b02a54 100644
--- a/Common/Config.py
+++ b/Common/Config.py
@@ -104,7 +104,10 @@ class Config():
             msg = "Option mac_pool_range expects 2"\
                     " values sepparated by whitespaces."
             raise ConfigError(msg)
-        if not verify_mac_address(option):
-            msg = "Invalid MAC address: %s" % option
+        if not verify_mac_address(vals[0]):
+            msg = "Invalid MAC address: %s" % vals[0]
+            raise ConfigError(msg)
+        if not verify_mac_address(vals[1]):
+            msg = "Invalid MAC address: %s" % vals[1]
             raise ConfigError(msg)
         return vals
-- 
1.7.11.4

[PATCH 3/8] Logs: integration with config module

[Ondrej Lichtner]

From: Ondrej Lichtner <olichtne(a)redhat.com&gt;

Logs are now saved to location specified in the config files by option
'path' in section 'log'.
The port on which the logging server is listening on can now be changed
by option 'port' in the same section.

Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com&gt;
---
 Common/Logs.py               | 7 +++++--
 NetTest/NetTestController.py | 5 +++--
 nettestctl.py                | 7 ++++---
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/Common/Logs.py b/Common/Logs.py
index 0772f1f..bf20e80 100644
--- a/Common/Logs.py
+++ b/Common/Logs.py
@@ -187,7 +187,7 @@ class Logs:
     @classmethod
     def __init__(cls,debug=0, waitForNet=False, logger=logging.getLogger(),
                  recipe_path=None, log_root="Logs", to_display=True,
date=None,
-                 nameExtend=None):
+                 nameExtend=None, log_folder=None):
         logging.addLevelName(5, "DEBUG2")
         logging.DEBUG2 = 5
         if nameExtend is None:
@@ -200,7 +200,10 @@ class Logs:
                             ':%(lineno)4.4d| %(levelname)s: '
                             '%(message)s', '%d/%m %H:%M:%S', "
"*4)
         cls.log_root = log_root
-        cls.logFolder = os.path.dirname(sys.argv[0])
+        if log_folder != None:
+            cls.logFolder = log_folder
+        else:
+            cls.logFolder = os.path.dirname(sys.argv[0])
         cls.logger = logger
         cls.debug = debug
         if date is None:
diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py
index 2a6bbd6..cf94a05 100644
--- a/NetTest/NetTestController.py
+++ b/NetTest/NetTestController.py
@@ -38,11 +38,12 @@ def ignore_event(**kwarg):
 
 class NetTestController:
     def __init__(self, recipe_path, remoteexec=False, cleanup=False,
-                 res_serializer=None):
+                 res_serializer=None, config=None):
         self._remoteexec = remoteexec
         self._docleanup = cleanup
         self._res_serializer = res_serializer
         self._remote_capture_files = {}
+        self._config = config
 
         self._machine_pool = MachinePool([])
 
@@ -259,7 +260,7 @@ class NetTestController:
         logging.info("Setting logging server on machine %s", hostname)
         rpc = self._get_machinerpc(machine_id)
         ip_addr = get_corespond_local_ip(hostname)
-        rpc.set_logging(ip_addr, LoggingServer.DEFAULT_PORT)
+        rpc.set_logging(ip_addr, self._config.get_option('log', 'port'))
 
     def _deconfigure_slaves(self):
         for machine_id in self._recipe["machines"]:
diff --git a/nettestctl.py b/nettestctl.py
index 2fab067..bc79398 100755
--- a/nettestctl.py
+++ b/nettestctl.py
@@ -49,7 +49,8 @@ def process_recipe(action, file_path, remoteexec, cleanup,
                    res_serializer, packet_capture, config):
     nettestctl = NetTestController(os.path.realpath(file_path),
                                    remoteexec=remoteexec, cleanup=cleanup,
-                                   res_serializer=res_serializer)
+                                   res_serializer=res_serializer,
+                                   config=config)
     if action == "run":
         return nettestctl.run_recipe(packet_capture)
     elif action == "dump":
@@ -74,7 +75,7 @@ def get_recipe_result(args, file_path, remoteexec, cleanup,
                       res_serializer, packet_capture, config):
     res_serializer.add_recipe(file_path)
     Logs.set_logging_root_path(file_path)
-    loggingServer = LoggingServer(LoggingServer.DEFAULT_PORT,
+    loggingServer = LoggingServer(config.get_option('log', 'port'),
                                   Logs.root_path, Logs.debug)
     loggingServer.start()
     res = process_recipe(args, file_path, remoteexec, cleanup,
@@ -122,7 +123,7 @@ def main():
             packet_capture = True
 
 
-    Logs(debug)
+    Logs(debug, log_folder=config.get_option('log', 'path'))
 
     if not args:
         logging.error("No action command passed")
-- 
1.7.11.4

[PATCH 4/8] MacPool: use pool range from config files

[Ondrej Lichtner]

From: Ondrej Lichtner <olichtne(a)redhat.com&gt;

Ranges of mac pool are now defined in config files, by option
'mac_pool_range' in section 'environment'.
Format of this option is
mac_pool_range = <starting_mac_address> <ending_mac_address>

Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com&gt;
---
 NetTest/NetTestController.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py
index cf94a05..f191b4d 100644
--- a/NetTest/NetTestController.py
+++ b/NetTest/NetTestController.py
@@ -28,8 +28,6 @@ from Common.VirtUtils import VirtNetCtl, VirtDomainCtl, BridgeCtl
 from Common.Utils import wait_for
 from NetTest.MachinePool import MachinePool
 
-MAC_POOL_RANGE = {"start": "52:54:01:00:00:01", "end":
"52:54:01:FF:FF:FF"}
-
 class NetTestError(Exception):
     pass
 
@@ -51,8 +49,10 @@ class NetTestController:
         definitions = {"recipe": self._recipe}
 
         self._recipe["networks"] = {}
-        self._mac_pool = MacPool(MAC_POOL_RANGE["start"],
-                                 MAC_POOL_RANGE["end"])
+
+        mac_pool_range = config.get_option('environment',
'mac_pool_range')
+        self._mac_pool = MacPool(mac_pool_range[0],
+                                 mac_pool_range[1])
 
         ntparse = NetTestParse(recipe_path)
         ntparse.set_recipe(self._recipe)
-- 
1.7.11.4

[PATCH 5/8] MachinePool: get pool dirs from config

[Ondrej Lichtner]

From: Ondrej Lichtner <olichtne(a)redhat.com&gt;

This patch adds support for option 'machine_pool_dirs' in section
'environment' of config files.
This option is a list of directories separated by whitespace.
Whitespaces in names need to be escaped with '\'.
Behaviour for this option is also a bit different, config files with
higher priority don't overwrite the previous value but instead are added
to the list.

This list is later passed to class MachinePool. Currently this class
doesn't check for the existence of these directories. Nonexistent
directories will cause a crash of the application so be sure to create
them before trying to use them.

Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com&gt;
---
 Common/Config.py             | 10 ++++++++++
 NetTest/NetTestController.py |  3 ++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/Common/Config.py b/Common/Config.py
index 4b02a54..6e68918 100644
--- a/Common/Config.py
+++ b/Common/Config.py
@@ -12,6 +12,7 @@ olichtne(a)redhat.com (Ondrej Lichtner)
 
 import os
 import logging
+import re
 from ConfigParser import ConfigParser
 from NetUtils import verify_ip_address, verify_mac_address
 
@@ -77,6 +78,8 @@ class Config():
                 section['mac_pool_range'] = self.optionMacRange(config[option])
             elif option == 'rpcport':
                 section['rpcport'] = self.optionPort(config[option])
+            elif option == 'machine_pool_dirs':
+                section['pool_dirs'] = self.optionPoolDirs(config[option])
             else:
                 msg = "Unknown option: %s in section environment" % option
                 raise ConfigError(msg)
@@ -111,3 +114,10 @@ class Config():
             msg = "Invalid MAC address: %s" % vals[1]
             raise ConfigError(msg)
         return vals
+
+    def optionPoolDirs(self, option):
+        env = self.get_section('environment')
+        if 'pool_dirs' not in env:
+            env['pool_dirs'] = []
+        opts = re.split(r'(?<!\\)\s', option)
+        return env['pool_dirs'] + opts
diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py
index f191b4d..059a65e 100644
--- a/NetTest/NetTestController.py
+++ b/NetTest/NetTestController.py
@@ -43,7 +43,8 @@ class NetTestController:
         self._remote_capture_files = {}
         self._config = config
 
-        self._machine_pool = MachinePool([])
+        self._machine_pool = MachinePool(config.get_option('environment',
+                                                            'pool_dirs'))
 
         self._recipe = {}
         definitions = {"recipe": self._recipe}
-- 
1.7.11.4

[PATCH 6/8] NetTestController: use rpcport from configuration files

[Ondrej Lichtner]

From: Ondrej Lichtner <olichtne(a)redhat.com&gt;

When remote execution is used NetTestController now uses the rpc port
specified in configuration files.
This port is also used for connecting to the rpc servers when no custom
port is specified in the recipe.

Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com&gt;
---
 NetTest/NetTestController.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py
index 059a65e..d9b2124 100644
--- a/NetTest/NetTestController.py
+++ b/NetTest/NetTestController.py
@@ -21,7 +21,6 @@ from Common.XmlRpc import ServerProxy
 from NetTest.NetTestParse import NetTestParse
 from Common.SlaveUtils import prepare_client_session
 from Common.NetUtils import get_corespond_local_ip, MacPool
-from NetTest.NetTestSlave import DefaultRPCPort
 from NetTest.NetTestCommand import NetTestCommand, str_command
 from Common.LoggingServer import LoggingServer
 from Common.VirtUtils import VirtNetCtl, VirtDomainCtl, BridgeCtl
@@ -233,8 +232,9 @@ class NetTestController:
 
         port = "22"
         login = "root"
+        rpc_port = self._config.get_option('environment', 'rpcport')
         session = prepare_client_session(hostname, port, login, passwd,
-                                         "nettestslave.py")
+                                         "nettestslave.py -p %s" % rpc_port)
         session.add_kill_handler(self._session_die)
         info["session"] = session
 
@@ -244,7 +244,7 @@ class NetTestController:
         if "rpcport" in info:
             port = info["rpcport"]
         else:
-            port = DefaultRPCPort
+            port = self._config.get_option('environment', 'rpcport')
         logging.info("Connecting to RPC on machine %s", hostname)
 
         url = "http://%s:%d" % (hostname, port)
-- 
1.7.11.4

[PATCH 7/8] Config: cleanup of class Config

[Ondrej Lichtner]

From: Ondrej Lichtner <olichtne(a)redhat.com&gt;

In this patch I removed the support for option 'local_ip' ip section
'log' because this value is only used in one place as part of an output
string.

I also added proper initialization of the data structure that stores
loaded configuration values. The default values used here are values
that were previously used as global variables. Functionality therefore
shouldn't change.

The parser is now a local object of the function load_config, previously
it would cause problems by using old values when nonexistant files were
to be parsed.

I also added exception handling to functions get_section and get_option.

Finally I modified the way paths are handled- relative paths should be
relative to the location of the config file they are specified in. And
paths using ~ should be properly expanded to the current users home.

Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com&gt;
---
 Common/Config.py | 83 +++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 52 insertions(+), 31 deletions(-)

diff --git a/Common/Config.py b/Common/Config.py
index 6e68918..cf79c6c 100644
--- a/Common/Config.py
+++ b/Common/Config.py
@@ -11,9 +11,11 @@ olichtne(a)redhat.com (Ondrej Lichtner)
 """
 
 import os
+import sys
 import logging
 import re
 from ConfigParser import ConfigParser
+from NetTest.NetTestSlave import DefaultRPCPort
 from NetUtils import verify_ip_address, verify_mac_address
 
 class ConfigError(Exception):
@@ -21,55 +23,68 @@ class ConfigError(Exception):
 
 class Config():
     options = None
-    _parser = None
 
     def __init__(self):
-        self._parser = ConfigParser(dict_type=dict)
         self.options = dict()
 
+        self.options['log'] = dict()
+        self.options['log']['port'] = 9998
+        self.options['log']['path'] = os.path.join(
+                os.path.dirname(sys.argv[0]), './')
+
+        self.options['environment'] = dict()
+        self.options['environment']['mac_pool_range'] = \
+                ['52:54:01:00:00:01', '52:54:01:FF:FF:FF']
+        self.options['environment']['rpcport'] = DefaultRPCPort
+        self.options['environment']['pool_dirs'] = []
+
     def get_config(self):
         return self.options
 
     def get_section(self, section):
+        if section not in self.options:
+            msg = 'Unknow section: %s' % section
+            raise ConfigError(msg)
         return self.options[section]
 
     def get_option(self, section, option):
-        return self.options[section][option]
+        sect = self.get_section(section)
+        if option not in sect:
+            msg = 'Unknown option: %s in section: %s' % (option, section)
+            raise ConfigError(msg)
+        return sect[option]
 
     def load_config(self, path):
         '''Parse and load the config file'''
-        self._parser.read(path)
+        exp_path = os.path.expanduser(path)
+        abs_path = os.path.abspath(exp_path)
+        parser = ConfigParser(dict_type=dict)
+        parser.read(abs_path)
 
-        sections = self._parser._sections
+        sections = parser._sections
         for section in sections:
             if section == "log":
-                self.sectionLogs(sections[section])
+                self.sectionLogs(sections[section], abs_path)
             elif section == "environment":
-                self.sectionEnvironment(sections[section])
+                self.sectionEnvironment(sections[section], abs_path)
             else:
                 msg = "Unknown section: %s" % section
                 raise ConfigError(msg)
 
-    def sectionLogs(self, config):
-        if 'log' not in self.options:
-            self.options['log'] = dict()
+    def sectionLogs(self, config, cfg_path):
         section = self.options['log']
 
         config.pop('__name__', None)
         for option in config:
-            if option == 'local_ip':
-                section['local_ip'] = self.optionLocalIP(config[option])
-            elif option == 'port':
+            if option == 'port':
                 section['port'] = self.optionPort(config[option])
             elif option == 'path':
-                section['path'] = self.optionLogPath(config[option])
+                section['path'] = self.optionLogPath(config[option], cfg_path)
             else:
                 msg = "Unknown option: %s in section log" % option
                 raise ConfigError(msg)
 
-    def sectionEnvironment(self, config):
-        if 'environment' not in self.options:
-            self.options['environment'] = dict()
+    def sectionEnvironment(self, config, cfg_path):
         section = self.options['environment']
 
         config.pop('__name__', None)
@@ -79,17 +94,12 @@ class Config():
             elif option == 'rpcport':
                 section['rpcport'] = self.optionPort(config[option])
             elif option == 'machine_pool_dirs':
-                section['pool_dirs'] = self.optionPoolDirs(config[option])
+                section['pool_dirs'] = self.optionPoolDirs(config[option],
+                                                           cfg_path)
             else:
                 msg = "Unknown option: %s in section environment" % option
                 raise ConfigError(msg)
 
-    def optionLocalIP(self, option):
-        if not verify_ip_address(option):
-            msg = "Invalid IP address: %s" % option
-            raise ConfigError(msg)
-        return option
-
     def optionPort(self, option):
         try:
             int(option)
@@ -98,8 +108,11 @@ class Config():
             raise ConfigError(msg)
         return int(option)
 
-    def optionLogPath(self, option):
-        return option
+    def optionLogPath(self, option, cfg_path):
+        exp_path = os.path.expanduser(option)
+        abs_path = os.path.join(os.path.dirname(cfg_path), exp_path)
+        norm_path = os.path.normpath(abs_path)
+        return norm_path
 
     def optionMacRange(self, option):
         vals = option.split()
@@ -115,9 +128,17 @@ class Config():
             raise ConfigError(msg)
         return vals
 
-    def optionPoolDirs(self, option):
+    def optionPoolDirs(self, option, cfg_path):
         env = self.get_section('environment')
-        if 'pool_dirs' not in env:
-            env['pool_dirs'] = []
-        opts = re.split(r'(?<!\\)\s', option)
-        return env['pool_dirs'] + opts
+        paths = re.split(r'(?<!\\)\s', option)
+
+        pool_dirs = env['pool_dirs']
+        for path in paths:
+            if path == '':
+                continue
+            exp_path = os.path.expanduser(path)
+            abs_path = os.path.join(os.path.dirname(cfg_path), exp_path)
+            norm_path = os.path.normpath(abs_path)
+            pool_dirs.append(norm_path)
+
+        return pool_dirs
-- 
1.7.11.4

[PATCH 8/8] add lnst.conf.example configuration file

[Ondrej Lichtner]

From: Ondrej Lichtner <olichtne(a)redhat.com&gt;

This file is being added as an example of how to create custom
configuration files. For illustration it contains the default values
currently used by lnst.
All that needs to be done is changing the values and place the resulting
file in a proper location. Currently only file ~/.lnst/lnst.conf will be
used.

Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com&gt;
---
 lnst.conf.example | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 lnst.conf.example

diff --git a/lnst.conf.example b/lnst.conf.example
new file mode 100644
index 0000000..af64aa8
--- /dev/null
+++ b/lnst.conf.example
@@ -0,0 +1,7 @@
+[environment]
+mac_pool_range = 52:54:01:00:00:01 52:54:01:FF:FF:FF
+rpcport = 9999
+machine_pool_dirs =
+[log]
+path = ./
+port = 9998
-- 
1.7.11.4