[PATCH 3/5] NetTestController: draft for PyRecipes

[Jiri Prochazka]

Changes:
* multi_match flag added to NTC attributes
* run_mode flag added to NTC attributes
* _prepare_network now uses mreq dict instead of parsed XML recipe
* new method set_machine_requirements() for SlavePool
* removed resource_sync from provision_machines
* new method prepare_test_env() which provisions machines, prints match desc,
  prepares network and initializes HostAPI objects
* get_aliases() method reworked

Signed-off-by: Jiri Prochazka <jprochaz(a)redhat.com&gt;
---
 lnst/Controller/NetTestController.py | 396 ++++++++++-------------------------
 1 file changed, 108 insertions(+), 288 deletions(-)

diff --git a/lnst/Controller/NetTestController.py b/lnst/Controller/NetTestController.py
index 8fdd43e..431b909 100644
--- a/lnst/Controller/NetTestController.py
+++ b/lnst/Controller/NetTestController.py
@@ -24,7 +24,7 @@ from lnst.Common.NetUtils import MacPool
 from lnst.Common.Utils import md5sum, dir_md5sum
 from lnst.Common.Utils import check_process_running, bool_it, get_module_tools
 from lnst.Common.NetTestCommand import str_command, CommandException
-from lnst.Controller.RecipeParser import RecipeParser, RecipeError
+from lnst.Controller.RecipeParser import RecipeError
 from lnst.Controller.SlavePool import SlavePool
 from lnst.Controller.Machine import MachineError, VirtualInterface
 from lnst.Controller.Machine import StaticInterface
@@ -54,18 +54,21 @@ class NetTestController:
                  res_serializer=None, pool_checks=True,
                  packet_capture=False,
                  defined_aliases=None, overriden_aliases=None,
-                 reduce_sync=False, restrict_pools=[]):
+                 reduce_sync=False, restrict_pools=[], multi_match=False):
         self._res_serializer = res_serializer
         self._remote_capture_files = {}
         self._log_ctl = log_ctl
-        self._recipe_path = Path(None, recipe_path).abs_path()
+        self._recipe_path = Path(None, recipe_path)
         self._msg_dispatcher = MessageDispatcher(log_ctl)
         self._packet_capture = packet_capture
         self._reduce_sync = reduce_sync
-        self._parser = RecipeParser(recipe_path)
+        self._defined_aliases = defined_aliases
+        self._multi_match = multi_match
 
         self.remove_saved_machine_config()
 
+        self.run_mode = "run"
+
         self._machines = {}
         self._network_bridges = {}
         self._tasks = []
@@ -73,10 +76,6 @@ class NetTestController:
         mac_pool_range = lnst_config.get_option('environment',
'mac_pool_range')
         self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
 
-        self._parser.set_machines(self._machines)
-        self._parser.set_aliases(defined_aliases, overriden_aliases)
-        self._recipe = self._parser.parse()
-
         conf_pools = lnst_config.get_pools()
         pools = {}
         if len(restrict_pools) > 0:
@@ -93,9 +92,6 @@ class NetTestController:
         sp = SlavePool(pools, pool_checks)
         self._slave_pool = sp
 
-        mreq = self._get_machine_requirements()
-        sp.set_machine_requirements(mreq)
-
         modules_dirs = lnst_config.get_option('environment',
'module_dirs')
         tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
 
@@ -103,6 +99,9 @@ class NetTestController:
         self._resource_table["module"] = self._load_test_modules(modules_dirs)
         self._resource_table["tools"] = self._load_test_tools(tools_dirs)
 
+    def _get_run_mode(self):
+        return self.run_mode
+
     def _get_machineinfo(self, machine_id):
         try:
             info = self._recipe["machines"][machine_id]["params"]
@@ -118,92 +117,22 @@ class NetTestController:
         msg = "SSH session terminated with status %s" % status
         raise NetTestError(msg)
 
-    def _get_machine_requirements(self):
-        recipe = self._recipe
-
-        # There must be some machines specified in the recipe
-        if "machines" not in recipe or \
-          ("machines" in recipe and len(recipe["machines"]) == 0):
-            msg = "No hosts specified in the recipe. At least two " \
-                  "hosts are required to perform a network test."
-            raise RecipeError(msg, recipe)
-
-        # machine requirements
-        mreq = {}
-        for machine in recipe["machines"]:
-            m_id = machine["id"]
-
-            if m_id in mreq:
-                msg = "Machine with id='%s' already exists." % m_id
-                raise RecipeError(msg, machine)
-
-            params = {}
-            if "params" in machine:
-                for p in machine["params"]:
-                    if p["name"] in params:
-                        msg = "Parameter '%s' of host %s was specified
" \
-                              "multiple times. Overriding the previous value."
\
-                              % (p["name"], m_id)
-                        logging.warn(RecipeError(msg, p))
-                    name = p["name"]
-                    val = p["value"]
-                    params[name] = val
-
-            # Each machine must have at least one interface
-            if "interfaces" not in machine or \
-              ("interfaces" in machine and len(machine["interfaces"])
== 0):
-                msg = "Host '%s' has no interfaces specified." % m_id
-                raise RecipeError(msg, machine)
-
-            ifaces = {}
-            for iface in machine["interfaces"]:
-                if_id = iface["id"]
-                if if_id in ifaces:
-                    msg = "Interface with id='%s' already exists on host
" \
-                          "'%s'." % (if_id, m_id)
-
-                iface_type = iface["type"]
-                if iface_type != "eth":
-                    continue
-
-                iface_params = {}
-                if "params" in iface:
-                    for i in iface["params"]:
-                        if i["name"] in iface_params:
-                            msg = "Parameter '%s' of interface %s of "
\
-                                  "host %s was defined multiple times. " \
-                                  "Overriding the previous value." \
-                                  % (i["name"], if_id, m_id)
-                            logging.warn(RecipeError(msg, p))
-                        name = i["name"]
-                        val = i["value"]
-                        iface_params[name] = val
-
-                ifaces[if_id] = {
-                    "network": iface["network"],
-                    "params": iface_params
-                }
-
-            mreq[m_id] = {"params": params, "interfaces": ifaces}
-
-        return mreq
-
     def _prepare_network(self, resource_sync=True):
-        recipe = self._recipe
+        mreq = Task.get_mreq()
 
         machines = self._machines
         for m_id in machines.keys():
             self._prepare_machine(m_id, resource_sync)
 
-        for machine_xml_data in recipe["machines"]:
-            m_id = machine_xml_data["id"]
+        for machine_id, machine_data in mreq.iteritems():
+            m_id = machine_id
             m = machines[m_id]
             namespaces = set()
-            for iface_xml_data in machine_xml_data["interfaces"]:
-                self._prepare_interface(m_id, iface_xml_data)
+            for if_id, iface_data in machine_data["interfaces"].iteritems():
+                self._prepare_interface(m_id, if_id, iface_data)
 
-                if iface_xml_data["netns"] != None:
-                    namespaces.add(iface_xml_data["netns"])
+                if iface_data["netns"] != None:
+                    namespaces.add(iface_data["netns"])
 
             if len(namespaces) > 0:
                 m.disable_nm()
@@ -225,9 +154,15 @@ class NetTestController:
 
             m.wait_interface_init()
 
-    def provision_machines(self):
+
+    def set_machine_requirements(self):
+        mreq = Task.get_mreq()
         sp = self._slave_pool
+        sp.set_machine_requirements(mreq)
+
+    def provision_machines(self):
         machines = self._machines
+        sp = self._slave_pool
         if not sp.provision_machines(machines):
             msg = "This setup cannot be provisioned with the current pool."
             raise NoMatchError(msg)
@@ -257,148 +192,51 @@ class NetTestController:
         machine.set_mac_pool(self._mac_pool)
         machine.set_network_bridges(self._network_bridges)
 
-        recipe_name = os.path.basename(self._recipe_path)
+        recipe_name = os.path.basename(self._recipe_path.abs_path())
         machine.configure(recipe_name)
 
-        sync_table = {'module': {}, 'tools': {}}
-        if resource_sync:
-            for task in self._recipe['tasks']:
-                res_table = copy.deepcopy(self._resource_table)
-                if 'module_dir' in task:
-                    modules = self._load_test_modules([task['module_dir']])
-                    res_table['module'].update(modules)
-                if 'tools_dir' in task:
-                    tools = self._load_test_tools([task['tools_dir']])
-                    res_table['tools'].update(tools)
-
-                if 'commands' not in task:
-                    if not self._reduce_sync:
-                        sync_table = res_table
-                        break
-                    else:
-                        continue
-                for cmd in task['commands']:
-                    if 'host' not in cmd or cmd['host'] != m_id:
-                        continue
-                    if cmd['type'] == 'test':
-                        mod = cmd['module']
-                        if mod in res_table['module']:
-                            sync_table['module'][mod] =
res_table['module'][mod]
-                            # check if test module uses some test tools
-                            mod_path =
res_table['module'][mod]["path"]
-                            mod_tools = get_module_tools(mod_path)
-                            for t in mod_tools:
-                                if t in sync_table['tools']:
-                                    continue
-                                logging.debug("Adding '%s' tool as "\
-                                    "dependency of %s test module" % (t, mod))
-                                sync_table['tools'][t] =
res_table['tools'][t]
-                        else:
-                            msg = "Module '%s' not found on the
controller"\
-                                    % mod
-                            raise RecipeError(msg, cmd)
-                    if cmd['type'] == 'exec' and 'from' in cmd:
-                        tool = cmd['from']
-                        if tool in res_table['tools']:
-                            sync_table['tools'][tool] =
res_table['tools'][tool]
-                        else:
-                            msg = "Tool '%s' not found on the
controller" % tool
-                            raise RecipeError(msg, cmd)
-        machine.sync_resources(sync_table)
-
-    def _prepare_interface(self, m_id, iface_xml_data):
+    def _prepare_interface(self, m_id, if_id, iface_data):
         machine = self._machines[m_id]
-        if_id = iface_xml_data["id"]
-        if_type = iface_xml_data["type"]
-
-        try:
-            iface = machine.get_interface(if_id)
-        except MachineError:
-            if if_type == 'lo':
-                iface = machine.new_loopback_interface(if_id)
-            else:
-                iface = machine.new_soft_interface(if_id, if_type)
-
-        if "slaves" in iface_xml_data:
-            for slave in iface_xml_data["slaves"]:
-                slave_id = slave["id"]
-                iface.add_slave(machine.get_interface(slave_id))
+        #if_type = iface_data["type"]
 
-                # Some soft devices (such as team) use per-slave options
-                if "options" in slave:
-                    for opt in slave["options"]:
-                        iface.set_slave_option(slave_id, opt["name"],
-                                               opt["value"])
+        #try:
+        iface = machine.get_interface(if_id)
+        #except MachineError:
+            #if if_type == 'lo':
+            #    iface = machine.new_loopback_interface(if_id)
+            #else:
+            #    iface = machine.new_soft_interface(if_id, if_type)
 
-        if "addresses" in iface_xml_data:
-            for addr in iface_xml_data["addresses"]:
-                iface.add_address(addr)
+        #if "slaves" in iface_data:
+        #    for slave in iface_data["slaves"]:
+        #        slave_id = slave["id"]
+        #        iface.add_slave(machine.get_interface(slave_id))
 
-        if "options" in iface_xml_data:
-            for opt in iface_xml_data["options"]:
-                iface.set_option(opt["name"], opt["value"])
+        #        # Some soft devices (such as team) use per-slave options
+        #        if "options" in slave:
+        #            for opt in slave["options"]:
+        #                iface.set_slave_option(slave_id, opt["name"],
+        #                                       opt["value"])
 
-        if "netem" in iface_xml_data:
-            iface.set_netem(iface_xml_data["netem"].to_dict())
+        #if "addresses" in iface_data:
+        #    for addr in iface_data["addresses"]:
+        #        iface.add_address(addr)
 
-        if "ovs_conf" in iface_xml_data:
-            iface.set_ovs_conf(iface_xml_data["ovs_conf"].to_dict())
+        #if "options" in iface_data:
+        #    for opt in iface_data["options"]:
+        #        iface.set_option(opt["name"], opt["value"])
 
-        if iface_xml_data["netns"] != None:
-            iface.set_netns(iface_xml_data["netns"])
+        #if "netem" in iface_data:
+        #    iface.set_netem(iface_data["netem"].to_dict())
 
-        if "peer" in iface_xml_data:
-            iface.set_peer(iface_xml_data["peer"])
-
-    def _prepare_tasks(self):
-        self._tasks = []
-        for task_data in self._recipe["tasks"]:
-            task = {}
-            task["quit_on_fail"] = False
-            if "quit_on_fail" in task_data:
-                task["quit_on_fail"] =
bool_it(task_data["quit_on_fail"])
+        #if "ovs_conf" in iface_data:
+        #    iface.set_ovs_conf(iface_data["ovs_conf"].to_dict())
 
-            if "module_dir" in task_data:
-                task["module_dir"] = task_data["module_dir"]
+        if iface_data["netns"] != None:
+            iface.set_netns(iface_data["netns"])
 
-            if "tools_dir" in task_data:
-                task["tools_dir"] = task_data["tools_dir"]
-
-            if "python" in task_data:
-                root = Path(None, self._recipe_path).get_root()
-                path = Path(root, task_data["python"])
-
-                task["python"] = path
-                if not path.exists():
-                    msg = "Task file '%s' not found." % path.to_str()
-                    raise RecipeError(msg, task_data)
-
-                self._tasks.append(task)
-                continue
-
-            task["commands"] = task_data["commands"]
-            task["skeleton"] = []
-            for cmd_data in task["commands"]:
-                cmd = {"type": cmd_data["type"]}
-
-                if "host" in cmd_data:
-                    cmd["host"] = cmd_data["host"]
-                    if cmd["host"] not in self._machines:
-                        msg = "Invalid host id '%s'." %
cmd["host"]
-                        raise RecipeError(msg, cmd_data)
-
-                if cmd["type"] in ["test", "exec"]:
-                    if "bg_id" in cmd_data:
-                        cmd["bg_id"] = cmd_data["bg_id"]
-                elif cmd["type"] in ["wait", "intr",
"kill"]:
-                    cmd["proc_id"] = cmd_data["bg_id"]
-
-                task["skeleton"].append(cmd)
-
-            if self._check_task(task):
-                raise RecipeError("Incorrect command sequence.", task_data)
-
-            self._tasks.append(task)
+        #if "peer" in iface_data:
+        #    iface.set_peer(iface_data["peer"])
 
     def _prepare_command(self, cmd_data):
         cmd = {"type": cmd_data["type"]}
@@ -613,36 +451,23 @@ class NetTestController:
             os.remove("/tmp/.lnst_machine_conf")
 
     def match_setup(self):
+        self.run_mode = "match_setup"
+        res = self._run_python_task()
         return {"passed": True}
 
     def config_only_recipe(self):
+        self.run_mode = "config_only"
         try:
-            self._prepare_network(resource_sync=False)
-        except (KeyboardInterrupt, Exception) as exc:
-            msg = "Exception raised during configuration."
-            logging.error(msg)
-            self._cleanup_slaves()
+            res = self._run_recipe()
+        except Exception as exc:
+            logging.error("Recipe execution terminated by unexpected
exception")
             raise
-
-        self._save_machine_config()
-
-        self._cleanup_slaves(deconfigure=False)
-        return {"passed": True}
-
-    def run_recipe(self):
-        try:
-            self._prepare_tasks()
-            self._prepare_network()
-        except (KeyboardInterrupt, Exception) as exc:
-            msg = "Exception raised during configuration."
-            logging.error(msg)
+        finally:
             self._cleanup_slaves()
-            raise
 
-        if self._packet_capture:
-            self._start_packet_capture()
+        return res
 
-        err = None
+    def run_recipe(self):
         try:
             res = self._run_recipe()
         except Exception as exc:
@@ -656,46 +481,55 @@ class NetTestController:
 
         return res
 
+    def prepare_test_env(self):
+        try:
+            self.provision_machines()
+            self.print_match_description()
+            if self.run_mode == "match_setup":
+                return True
+            self._prepare_network()
+            Task.ctl.init_hosts(self._machines)
+            return True
+        except (NoMatchError) as exc:
+            self._cleanup_slaves()
+            raise exc
+            return False
+        except (KeyboardInterrupt, Exception) as exc:
+            msg = "Exception raised during configuration."
+            logging.error(msg)
+            self._cleanup_slaves()
+            raise
+
     def _run_recipe(self):
         overall_res = {"passed": True}
 
-        for task in self._tasks:
+        try:
             self._res_serializer.add_task()
-            try:
-                res = self._run_task(task)
-            except CommandException as exc:
-                logging.debug(exc)
-                overall_res["passed"] = False
-                overall_res["err_msg"] = "Command exception raised."
-                break
-
-            for machine in self._machines.itervalues():
-                machine.restore_system_config()
-
-            # task failed, check if we should quit_on_fail
-            if not res:
-                overall_res["passed"] = False
-                overall_res["err_msg"] = "At least one command
failed."
-                if task["quit_on_fail"]:
-                    break
+            res = self._run_python_task()
+        except CommandException as exc:
+            logging.debug(exc)
+            overall_res["passed"] = False
+            overall_res["err_msg"] = "Command exception raised."
+
+        for machine in self._machines.itervalues():
+            machine.restore_system_config()
+
+        # task failed
+        if not res:
+            overall_res["passed"] = False
+            overall_res["err_msg"] = "At least one command failed."
 
         return overall_res
 
-    def _run_python_task(self, task):
+    def init_taskapi(self):
+        Task.ctl = Task.ControllerAPI(self)
+
+    def _run_python_task(self):
         #backup of resource table
         res_table_bkp = copy.deepcopy(self._resource_table)
-        if 'module_dir' in task:
-            modules = self._load_test_modules([task['module_dir']])
-            self._resource_table['module'].update(modules)
-        if 'tools_dir' in task:
-            tools = self._load_test_tools([task['tools_dir']])
-            self._resource_table['tools'].update(tools)
-
-        # Initialize the API handle
-        Task.ctl = Task.ControllerAPI(self, self._machines)
 
         cwd = os.getcwd()
-        task_path = task["python"]
+        task_path = self._recipe_path
         name = os.path.basename(task_path.abs_path()).split(".")[0]
         sys.path.append(os.path.dirname(task_path.resolve()))
         os.chdir(os.path.dirname(task_path.resolve()))
@@ -706,20 +540,7 @@ class NetTestController:
         #restore resource table
         self._resource_table = res_table_bkp
 
-        return module.ctl._result
-
-    def _run_task(self, task):
-        if "python" in task:
-            return self._run_python_task(task)
-
-        seq_passed = True
-        for cmd_data in task["commands"]:
-            cmd = self._prepare_command(cmd_data)
-            cmd_res = self._run_command(cmd)
-            if not cmd_res["passed"]:
-                seq_passed = False
-
-        return seq_passed
+        return Task.ctl._result
 
     def _run_command(self, command):
         logging.info("Executing command: [%s]", str_command(command))
@@ -848,12 +669,11 @@ class NetTestController:
         return packages
 
     def _get_alias(self, alias):
-        templates = self._parser._template_proc
-        return templates._find_definition(alias)
+        if alias in self._defined_aliases:
+            return self._defined_aliases[alias]
 
     def _get_aliases(self):
-        templates = self._parser._template_proc
-        return templates._dump_definitions()
+        return self._defined_aliases
 
 class MessageDispatcher(ConnectionHandler):
     def __init__(self, log_ctl):
-- 
2.4.11