[lnst] NetTest: Adding support for the new parser

[Jiří Pírko]

commit 86185c61918b6e22f6c342bb6aaed16f54b70b7b
Author: Radek Pazdera <rpazdera(a)redhat.com&gt;
Date:   Wed Jul 18 17:03:38 2012 +0200

    NetTest: Adding support for the new parser
    
    This commit modifies current NetTestController and NetTestSlave
    implementations, so they can work with the new parser.
    
    Controller used to pass the interface configuration all at once.
    This was modified so the config is now transfered one interface at
    a time.
    
    Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com&gt;

 NetTest/NetTestController.py |  277 +++++++++++++++++++++++-------------------
 NetTest/NetTestSlave.py      |   35 ++++--
 2 files changed, 175 insertions(+), 137 deletions(-)
---
diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py
index a33cacb..4daf070 100644
--- a/NetTest/NetTestController.py
+++ b/NetTest/NetTestController.py
@@ -18,146 +18,182 @@ from Common.Logs import Logs
 from Common.SshUtils import scp_from_remote
 from pprint import pprint, pformat
 from Common.XmlRpc import ServerProxy
-from NetTestParse import NetTestParse
+from NetTest.NetTestParse import NetTestParse
 from Common.SlaveUtils import prepare_client_session
 from Common.Utils import get_corespond_local_ip
-from NetTestSlave import DefaultRPCPort
-from NetTestCommand import NetTestCommand, str_command
+from NetTest.NetTestSlave import DefaultRPCPort
+from NetTest.NetTestCommand import NetTestCommand, str_command
 from Common.LoggingServer import LoggingServer
 
+def ignore_event(**kwarg):
+    pass
+
 class NetTestController:
     def __init__(self, recipe_path, remoteexec=False, cleanup=False,
                  res_serializer=None):
-        ntparse = NetTestParse(recipe_path)
-        ntparse.parse_recipe()
-        self._recipe = ntparse.get_recipe()
-        self._ntparse = ntparse
         self._remoteexec = remoteexec
         self._docleanup = cleanup
         self._res_serializer = res_serializer
         self._remote_capture_files = {}
 
+        self._recipe = {}
+        definitions = {"recipe": self._recipe}
+
+        ntparse = NetTestParse(recipe_path)
+        ntparse.set_recipe(self._recipe)
+        ntparse.set_definitions(definitions)
+
+        ntparse.register_event_handler("netdevice_ready", ignore_event)
+        ntparse.register_event_handler("machine_info_ready",
+                                        self._prepare_slave)
+        ntparse.register_event_handler("interface_config_ready",
+                                        self._prepare_interface)
+
+        self._ntparse = ntparse
+
     def _get_machineinfo(self, machine_id):
-        return self._recipe["machines"][machine_id]["info"]
+        try:
+            info = self._recipe["machines"][machine_id]["info"]
+        except KeyError:
+            msg = "Machine info is required, but not yet available"
+            raise Exception(msg)
+
+        return info
 
     def _get_machinerpc(self, machine_id):
-        return self._get_machineinfo(machine_id)["rpc"]
+        try:
+            rpc = self._get_machineinfo(machine_id)["rpc"]
+        except KeyError:
+            msg = "XMLRPC connection required, but not yet available"
+            raise Exception(msg)
 
-    def _session_die(self, session, status):
-        logging.error("Session started with cmd %s die with status %s." %
(session.command, status))
+        return rpc
+
+    @staticmethod
+    def _session_die(session, status):
+        logging.error("Session started with cmd %s die with status %s.",
+                                        session.command, status)
         raise Exception("Session Die.")
 
-    def _prepare_slaves(self):
-        for machine_id in self._recipe["machines"]:
-            info = self._get_machineinfo(machine_id)
-            hostname = info["hostname"]
-            logging.info("Remote app exec on machine %s" % hostname)
-            port = "22"
-            login = "root"
-            if "rootpass" in info:
-                passwd = info["rootpass"]
-            else:
-                passwd = None
-            session = prepare_client_session(hostname, port, login, passwd,
-                                             "nettestslave.py")
-            session.add_kill_handler(self._session_die)
-            info["session"] = session
+    def _prepare_interface(self, machine_id, netdev_config_id):
+        rpc = self._get_machinerpc(machine_id)
+        info = self._get_machineinfo(machine_id)
+        logging.info("Configuring interface #%d on %s", netdev_config_id,
+                                                        info["hostname"])
 
-    def _cleanup_slaves(self):
-        for machine_id in self._recipe["machines"]:
-            info = self._get_machineinfo(machine_id)
-            if "session" in info:
-                info["session"].kill()
-                info["session"].wait()
+        self._configure_interface(machine_id, netdev_config_id)
 
-    def _rpc_connect(self):
-        for machine_id in self._recipe["machines"]:
-            info = self._get_machineinfo(machine_id)
-            hostname = info["hostname"]
-            logging.info("Connecting to RPC on machine \"%s\"" %
hostname)
-            if "rpcport" in info:
-                port = info["rpcport"]
-            else:
-                port = DefaultRPCPort
-            url = "http://%s:%d" % (hostname, port)
-            rpc = ServerProxy(url, allow_none = True)
-            if rpc.hello() != "hello":
-                logging.error("Handshake error with machine id %d" %
machine_id)
-                raise Exception("Hanshake error")
-            if self._docleanup:
-                rpc.machine_cleanup()
-            info["rpc"] = rpc
-
-    def _logging_connect(self):
-        for machine_id in self._recipe["machines"]:
-            info = self._get_machineinfo(machine_id)
-            logging.info("Setting logging server on machine \"%s\"" %
info["hostname"])
-            rpc = self._get_machinerpc(machine_id)
-            ip = get_corespond_local_ip(info["hostname"])
-            rpc.set_logging(ip, LoggingServer.DEFAULT_PORT)
-
-    def _netconfigs_set(self):
-        machines = self._recipe["machines"]
-        for machine_id in machines:
-            machine = machines[machine_id]
-            logging.info("Setting netconfigs on machine \"%s\"" %
machine["info"]["hostname"])
-            rpc = self._get_machinerpc(machine_id)
-            rpc.netconfig_set(machine["netmachineconfig_xml"],
-                              machine["netconfig_xml"])
-            '''
-            Finally get processed netconfig from slave back to us.
-            Will be handy later on.
-            '''
-            machine["netconfig"] = dict(rpc.netconfig_dump())
-            del machine["netmachineconfig_xml"]
-            del machine["netconfig_xml"]
-
-    def _netconfigs_clear(self):
-        machines = self._recipe["machines"]
-        for machine_id in machines:
-            machine = machines[machine_id]
-            logging.info("Clearing netconfigs on machine \"%s\"" %
machine["info"]["hostname"])
+        if_info = rpc.get_interface_info(netdev_config_id)
+        machine = self._recipe["machines"][machine_id]
+        if "name" in if_info:
+            machine["netconfig"][netdev_config_id]["name"] =
if_info["name"]
+
+        info["configured_interfaces"].append(netdev_config_id)
+
+    def _configure_interface(self, machine_id, netdev_config_id):
+        rpc = self._get_machinerpc(machine_id)
+        netconfig =
self._recipe["machines"][machine_id]["netconfig"]
+        dev_config = netconfig[netdev_config_id]
+
+        rpc.configure_interface(netdev_config_id, dev_config)
+
+    def _deconfigure_interface(self, machine_id, netdev_config_id):
+        rpc = self._get_machinerpc(machine_id)
+        rpc.deconfigure_interface(netdev_config_id)
+
+    def _prepare_slave(self, machine_id):
+        logging.info("Preparing machine #%d", machine_id)
+        info = self._get_machineinfo(machine_id)
+
+        if self._remoteexec and not "session" in info:
+            self._init_slave_session(machine_id)
+
+        if not "rpc" in info:
+            self._init_slave_rpc(machine_id)
+            self._init_slave_logging(machine_id)
+
+            info["configured_interfaces"] = []
+
+    def _init_slave_session(self, machine_id):
+        info = self._get_machineinfo(machine_id)
+        hostname = info["hostname"]
+        if "rootpass" in info:
+            passwd = info["rootpass"]
+        else:
+            passwd = None
+        logging.info("Remote app exec on machine %s", hostname)
+
+        port = "22"
+        login = "root"
+        session = prepare_client_session(hostname, port, login, passwd,
+                                         "nettestslave.py")
+        session.add_kill_handler(self._session_die)
+        info["session"] = session
+
+    def _init_slave_rpc(self, machine_id):
+        info = self._get_machineinfo(machine_id)
+        hostname = info["hostname"]
+        if "rpcport" in info:
+            port = info["rpcport"]
+        else:
+            port = DefaultRPCPort
+        logging.info("Connecting to RPC on machine %s", hostname)
+
+        url = "http://%s:%d" % (hostname, port)
+        rpc = ServerProxy(url, allow_none = True)
+        if rpc.hello() != "hello":
+            logging.error("Handshake error with machine %s", hostname)
+            raise Exception("Hanshake error")
+
+        if self._docleanup:
+            rpc.machine_cleanup()
+
+        info["rpc"] = rpc
+
+    def _init_slave_logging(self, machine_id):
+        info = self._get_machineinfo(machine_id)
+        hostname = info["hostname"]
+        logging.info("Setting logging server on machine %s", hostname)
+        rpc = self._get_machinerpc(machine_id)
+        ip_addr = get_corespond_local_ip(hostname)
+        rpc.set_logging(ip_addr, LoggingServer.DEFAULT_PORT)
+
+    def _cleanup_slave(self, machine_id):
+        info = self._get_machineinfo(machine_id)
+
+        if self._docleanup:
             rpc = self._get_machinerpc(machine_id)
-            rpc.netconfig_clear()
+            for if_id in reversed(info["configured_interfaces"]):
+                rpc.deconfigure_interface(if_id)
+
+        if self._remoteexec and "session" in info:
+            info["session"].kill()
+            info["session"].wait()
 
     def _prepare(self):
-        if self._remoteexec:
-            self._prepare_slaves()
-        self._rpc_connect()
-        self._logging_connect()
-        self._netconfigs_set()
-
-        '''
-        Now as we have all netconfigs processed by slaves back, it's time to
-        parse recipe command sequence
-        '''
-        self._ntparse.parse_recipe_command_sequence()
-        self._recipe = self._ntparse.get_recipe()
+        # All the perparations are made within the recipe parsing
+        # This is achieved by handling parser events (by registering
+        self._ntparse.parse_recipe()
 
     def _cleanup(self):
-        self._netconfigs_clear()
-        if self._remoteexec:
-            self._cleanup_slaves()
+        for machine_id in self._recipe["machines"]:
+            self._cleanup_slave(machine_id)
 
     def _run_command(self, command):
-        cmd_type = command["type"]
         machine_id = command["machine_id"]
-
         try:
             desc = command["desc"]
-            logging.info("Cmd description: %s" % desc)
+            logging.info("Cmd description: %s", desc)
         except KeyError:
             pass
 
         if machine_id == 0:
             cmd_res = NetTestCommand(command).run()
         else:
-            info = self._get_machineinfo(machine_id)
-            hostname = info["hostname"]
             rpc = self._get_machinerpc(machine_id)
             if "timeout" in command:
                 timeout = command["timeout"]
-                logging.debug("Setting socket timeout to \"%d\"" %
timeout)
+                logging.debug("Setting socket timeout to \"%d\"",
timeout)
                 socket.setdefaulttimeout(timeout)
             try:
                 cmd_res = rpc.run_command(command)
@@ -182,36 +218,30 @@ class NetTestController:
     def _run_command_sequence(self, sequence):
         seq_passed = True
         for command in sequence:
-            logging.info("Executing command: [%s]" % str_command(command))
+            logging.info("Executing command: [%s]", str_command(command))
             cmd_res = self._run_command(command)
             if self._res_serializer:
                 self._res_serializer.add_cmd_result(command, cmd_res)
-            logging.debug("Result: %s" % str(cmd_res))
+            logging.debug("Result: %s", str(cmd_res))
             if "res_data" in cmd_res:
                 res_data = pformat(cmd_res["res_data"])
-                logging.info("Result data: %s" % (res_data))
+                logging.info("Result data: %s", (res_data))
             if not cmd_res["passed"]:
                 logging.error("Command failed - command: [%s], "
-                              "Error message: \"%s\""
-                                % (str_command(command), cmd_res["err_msg"]))
+                              "Error message: \"%s\"",
+                              str_command(command), cmd_res["err_msg"])
                 seq_passed = False
         return seq_passed
 
     def dump_recipe(self):
-        pprint(self._recipe)
-        return True
-
-    def all_dump_recipe(self):
         self._prepare()
         pprint(self._recipe)
-        if self._docleanup:
-            self._cleanup()
+        self._cleanup()
         return True
 
     def config_only_recipe(self):
         self._prepare()
-        if self._docleanup:
-            self._cleanup()
+        self._cleanup()
         return True
 
     def run_recipe(self, packet_capture=False):
@@ -223,15 +253,14 @@ class NetTestController:
         err = None
         try:
             res = self._run_recipe()
-        except e:
-            err = e
+        except Exception, exc:
+            err = exc
 
         if packet_capture:
             self._stop_packet_capture()
             self._gather_capture_files()
 
-        if self._docleanup:
-            self._cleanup()
+        self._cleanup()
 
         if not err:
             return res
@@ -275,8 +304,8 @@ class NetTestController:
             slave_logging_dir = os.path.join(logging_root, hostname)
             try:
                 os.mkdir(slave_logging_dir)
-            except OSError, e:
-                if e.errno != 17:
+            except OSError, err:
+                if err.errno != 17:
                     raise
 
             capture_files = self._remote_capture_files[machine_id]
@@ -286,11 +315,6 @@ class NetTestController:
                 scp_from_remote(hostname, "22", "root", rootpass,
                                     remote_path, local_path)
 
-    def eval_expression_recipe(self, expr):
-        self._prepare()
-        value = eval("self._recipe%s" % expr)
-        return True
-
     def _update_system_config(self, machine_id, res_data, persistent=False):
         info = self._get_machineinfo(machine_id)
         system_config = info["system_config"]
@@ -300,7 +324,8 @@ class NetTestController:
                     del system_config[option]
             else:
                 if not option in system_config:
-                    system_config[option] = {"initial_val":
values["previous_val"]}
+                    initial_val = {"initial_val":
values["previous_val"]}
+                    system_config[option] = initial_val
                 system_config[option]["current_val"] =
values["current_val"]
 
 
diff --git a/NetTest/NetTestSlave.py b/NetTest/NetTestSlave.py
index 955c28e..272066d 100644
--- a/NetTest/NetTestSlave.py
+++ b/NetTest/NetTestSlave.py
@@ -20,7 +20,7 @@ from Common.XmlRpc import Server
 from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
 from NetConfig.NetConfig import NetConfig
 from NetConfig.NetConfigDevice import NetConfigDeviceAllCleanup
-from NetTestCommand import NetTestCommand, CommandException
+from NetTest.NetTestCommand import NetTestCommand, CommandException
 from Common.Utils import die_when_parent_die
 
 DefaultRPCPort = 9999
@@ -32,6 +32,7 @@ class NetTestSlaveXMLRPC:
     def __init__(self):
         self._netconfig = None
         self._packet_captures = {}
+        self._netconfig = NetConfig()
 
     def hello(self):
         return "hello"
@@ -45,19 +46,30 @@ class NetTestSlaveXMLRPC:
         Logs.append_network_hadler(logger_address, port)
         return True
 
-    def netconfig_set(self, machine_xml_string, config_xml_string):
-        self._netconfig = NetConfig(machine_xml_string, config_xml_string)
-        self._netconfig.configure_all()
+    def get_interface_info(self, if_id):
+        if_config = self._netconfig.get_interface_config(if_id)
+        info = {}
+
+        if "name" in if_config:
+            info["name"] = if_config["name"]
+
+        if "hwaddr" in if_config:
+            info["hwaddr"] = if_config["hwaddr"]
+
+        return info
+
+    def configure_interface(self, if_id, config):
+        self._netconfig.add_interface_config(if_id, config)
+        self._netconfig.configure(if_id)
+        return True
+
+    def deconfigure_interface(self, if_id):
+        self._netconfig.deconfigure(if_id)
         return True
 
     def netconfig_dump(self):
         return self._netconfig.dump_config().items()
 
-    def netconfig_clear(self):
-        self._netconfig.deconfigure_all()
-        self.__init__()
-        return True
-
     def start_packet_capture(self, filt):
         logging_dir = Logs.get_logging_root_path()
         logging_dir = os.path.abspath(logging_dir)
@@ -91,8 +103,9 @@ class NetTestSlaveXMLRPC:
             return NetTestCommand(command).run()
         except:
             import sys, traceback
-            type, value, tb = sys.exc_info()
-            logging.error(''.join(traceback.format_exception(type, value, tb)))
+            cmd_type, value, tb = sys.exc_info()
+            exception = traceback.format_exception(cmd_type, value, tb)
+            logging.error(''.join(exception))
             raise CommandException(command)
 
     def machine_cleanup(self):