Tue, Jul 12, 2016 at 11:00:33AM CEST, jprochaz(a)redhat.com wrote:
This patch implements PyRecipes and breaks existing XML recipes.
Python recipes, unlike XML recipes, do not need to be parsed before
execution. They contain both network topology definition and test
commands.
Network topology is defined in the beginning of PyRecipe, example:
import lnst
m1 = lnst.add_host()
m2 = lnst.add_host()
m1_eth1 = m1.add_interface(label="tnet")
m2_eth1 = m2.add_interface(label="tnet")
In the example, test will use 2 different hosts, each with 1 eth device.
To run match and network provision, use lnst.match() method in while
loop, example:
while lnst.match():
m1_eth1.reset(ip="192.168.0.1/24")
m2_eth1.reset(ip="192.168.0.2/24")
Why "reset"? For every match, the config should be cleared.
>
>This piece of code will run the match, prepare the setup and sets IP
>addresses to both eth devices.
>
>More info on how to use Pyrecipes will be in our documentation (github wiki).
>
>After this patch, XML recipes will no longer work. It removes and
>modifies a lot of code in order to not break git bisect.
>
>Signed-off-by: Jiri Prochazka <jprochaz(a)redhat.com>
>---
> lnst-ctl | 13 +-
> lnst/Controller/NetTestController.py | 266 ++++++-----------------------------
> lnst/Controller/Task.py | 24 ++--
> 3 files changed, 62 insertions(+), 241 deletions(-)
>
>diff --git a/lnst-ctl b/lnst-ctl
>index b75b8a0..93b084b 100755
>--- a/lnst-ctl
>+++ b/lnst-ctl
>@@ -178,9 +178,13 @@ def get_recipe_result(action, file_path, log_ctl,
res_serializer,
> res = {}
> if matches == 1:
> try:
>- nettestctl.provision_machines()
>- nettestctl.print_match_description()
>+ # init TaskAPI.Ctl
>+ nettestctl.init_taskapi()
> res = exec_action(action, nettestctl)
>+ except NoMatchError as err:
>+ no_match = True
>+ log_ctl.unset_recipe()
>+ logging.warning("Match %d not possible." % matches)
> except Exception as err:
> no_match = True
> log_exc_traceback()
>@@ -190,12 +194,11 @@ def get_recipe_result(action, file_path, log_ctl,
res_serializer,
> retval = RETVAL_ERR
> elif matches > 1:
> try:
>- nettestctl.provision_machines()
>+ nettestctl.init_taskapi()
> log_ctl.set_recipe(file_path, prepend=True,
expand="match_%d" % matches)
> log_dir = log_ctl.get_recipe_log_path()
> recipe_head_log_entry(file_path, log_dir, matches)
> res_serializer.add_recipe(file_path, matches)
>- nettestctl.print_match_description()
> res = exec_action(action, nettestctl)
> except NoMatchError as err:
> no_match = True
>@@ -400,7 +403,7 @@ def main():
> all_files.sort()
> for f in all_files:
> recipe_file = os.path.join(recipe_path, f)
>- if re.match(r'^.*\.xml$', recipe_file):
>+ if re.match(r'^.*\.py$', recipe_file):
> recipe_files.append(recipe_file)
> else:
> recipe_files.append(recipe_path)
>diff --git a/lnst/Controller/NetTestController.py
b/lnst/Controller/NetTestController.py
>index 0115d2f..135bbf6 100644
>--- a/lnst/Controller/NetTestController.py
>+++ b/lnst/Controller/NetTestController.py
>@@ -61,11 +61,10 @@ class NetTestController:
> self._res_serializer = res_serializer
> self._remote_capture_files = {}
> self._log_ctl = log_ctl
>- self._recipe_path = Path(None, recipe_path).abs_path()
>+ self._recipe_path = Path(None, recipe_path)
> self._msg_dispatcher = MessageDispatcher(log_ctl)
> self._packet_capture = packet_capture
> self._reduce_sync = reduce_sync
>- self._parser = RecipeParser(recipe_path)
> self._defined_aliases = defined_aliases
> self._multi_match = multi_match
>
>@@ -80,10 +79,6 @@ class NetTestController:
> mac_pool_range = lnst_config.get_option('environment',
'mac_pool_range')
> self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
>
>- self._parser.set_machines(self._machines)
>- self._parser.set_aliases(defined_aliases, overriden_aliases)
>- self._recipe = self._parser.parse()
>-
> conf_pools = lnst_config.get_pools()
> pools = {}
> if len(restrict_pools) > 0:
>@@ -100,9 +95,6 @@ class NetTestController:
> sp = SlavePool(pools, pool_checks)
> self._slave_pool = sp
>
>- mreq = self._get_machine_requirements()
>- sp.set_machine_requirements(mreq)
>-
> modules_dirs = lnst_config.get_option('environment',
'module_dirs')
> tools_dirs = lnst_config.get_option('environment',
'tool_dirs')
>
>@@ -196,21 +188,21 @@ class NetTestController:
> return mreq
>
> def _prepare_network(self, resource_sync=True):
>- recipe = self._recipe
>+ mreq = Task.get_mreq()
>
> machines = self._machines
> for m_id in machines.keys():
> self._prepare_machine(m_id, resource_sync)
>
>- for machine_xml_data in recipe["machines"]:
>- m_id = machine_xml_data["id"]
>+ for machine_id, machine_data in mreq.iteritems():
>+ m_id = machine_id
> m = machines[m_id]
> namespaces = set()
>- for iface_xml_data in machine_xml_data["interfaces"]:
>- self._prepare_interface(m_id, iface_xml_data)
>+ for if_id, iface_data in
machine_data["interfaces"].iteritems():
>+ self._prepare_interface(m_id, if_id, iface_data)
>
>- if iface_xml_data["netns"] != None:
>- namespaces.add(iface_xml_data["netns"])
>+ if iface_data["netns"] != None:
>+ namespaces.add(iface_data["netns"])
>
> if len(namespaces) > 0:
> m.disable_nm()
>@@ -269,148 +261,16 @@ class NetTestController:
> machine.set_mac_pool(self._mac_pool)
> machine.set_network_bridges(self._network_bridges)
>
>- recipe_name = os.path.basename(self._recipe_path)
>+ recipe_name = os.path.basename(self._recipe_path.abs_path())
> machine.init_connection(recipe_name)
>
>- sync_table = {'module': {}, 'tools': {}}
>- if resource_sync:
>- res_table = copy.deepcopy(self._resource_table)
>- for task in self._recipe['tasks']:
>- if 'module_dir' in task:
>- modules = self._load_test_modules([task['module_dir']])
>- res_table['module'].update(modules)
>- if 'tools_dir' in task:
>- tools = self._load_test_tools([task['tools_dir']])
>- res_table['tools'].update(tools)
>-
>- if 'commands' not in task:
>- if not self._reduce_sync:
>- sync_table = res_table
>- break
>- else:
>- continue
>- for cmd in task['commands']:
>- if 'host' not in cmd or cmd['host'] != m_id:
>- continue
>- if cmd['type'] == 'test':
>- mod = cmd['module']
>- if mod in res_table['module']:
>- sync_table['module'][mod] =
res_table['module'][mod]
>- # check if test module uses some test tools
>- mod_path =
res_table['module'][mod]["path"]
>- mod_tools = get_module_tools(mod_path)
>- for t in mod_tools:
>- if t in sync_table['tools']:
>- continue
>- logging.debug("Adding '%s' tool as
"\
>- "dependency of %s test module" % (t,
mod))
>- sync_table['tools'][t] =
res_table['tools'][t]
>- else:
>- msg = "Module '%s' not found on the
controller"\
>- % mod
>- raise RecipeError(msg, cmd)
>- if cmd['type'] == 'exec' and 'from' in
cmd:
>- tool = cmd['from']
>- if tool in res_table['tools']:
>- sync_table['tools'][tool] =
res_table['tools'][tool]
>- else:
>- msg = "Tool '%s' not found on the
controller" % tool
>- raise RecipeError(msg, cmd)
>- machine.sync_resources(sync_table)
>-
>- def _prepare_interface(self, m_id, iface_xml_data):
>+ def _prepare_interface(self, m_id, if_id, iface_data):
> machine = self._machines[m_id]
>- if_id = iface_xml_data["id"]
>- if_type = iface_xml_data["type"]
>-
>- try:
>- iface = machine.get_interface(if_id)
>- except MachineError:
>- if if_type == 'lo':
>- iface = machine.new_loopback_interface(if_id)
>- else:
>- iface = machine.new_soft_interface(if_id, if_type)
>-
>- if "slaves" in iface_xml_data:
>- for slave in iface_xml_data["slaves"]:
>- slave_id = slave["id"]
>- iface.add_slave(machine.get_interface(slave_id))
>-
>- # Some soft devices (such as team) use per-slave options
>- if "options" in slave:
>- for opt in slave["options"]:
>- iface.set_slave_option(slave_id, opt["name"],
>- opt["value"])
>-
>- if "addresses" in iface_xml_data:
>- for addr in iface_xml_data["addresses"]:
>- iface.add_address(addr)
>-
>- if "options" in iface_xml_data:
>- for opt in iface_xml_data["options"]:
>- iface.set_option(opt["name"], opt["value"])
>-
>- if "netem" in iface_xml_data:
>- iface.set_netem(iface_xml_data["netem"].to_dict())
>-
>- if "ovs_conf" in iface_xml_data:
>- iface.set_ovs_conf(iface_xml_data["ovs_conf"].to_dict())
>-
>- if iface_xml_data["netns"] != None:
>- iface.set_netns(iface_xml_data["netns"])
>-
>- if "peer" in iface_xml_data:
>- iface.set_peer(iface_xml_data["peer"])
>-
>- def _prepare_tasks(self):
>- self._tasks = []
>- for task_data in self._recipe["tasks"]:
>- task = {}
>- task["quit_on_fail"] = False
>- if "quit_on_fail" in task_data:
>- task["quit_on_fail"] =
bool_it(task_data["quit_on_fail"])
>-
>- if "module_dir" in task_data:
>- task["module_dir"] = task_data["module_dir"]
>-
>- if "tools_dir" in task_data:
>- task["tools_dir"] = task_data["tools_dir"]
>-
>- if "python" in task_data:
>- root = Path(None, self._recipe_path).get_root()
>- path = Path(root, task_data["python"])
>-
>- task["python"] = path
>- if not path.exists():
>- msg = "Task file '%s' not found." %
path.to_str()
>- raise RecipeError(msg, task_data)
>-
>- self._tasks.append(task)
>- continue
>-
>- task["commands"] = task_data["commands"]
>- task["skeleton"] = []
>- for cmd_data in task["commands"]:
>- cmd = {"type": cmd_data["type"]}
>-
>- if "host" in cmd_data:
>- cmd["host"] = cmd_data["host"]
>- if cmd["host"] not in self._machines:
>- msg = "Invalid host id '%s'." %
cmd["host"]
>- raise RecipeError(msg, cmd_data)
>-
>- if cmd["type"] in ["test", "exec"]:
>- if "bg_id" in cmd_data:
>- cmd["bg_id"] = cmd_data["bg_id"]
>- elif cmd["type"] in ["wait", "intr",
"kill"]:
>- cmd["proc_id"] = cmd_data["bg_id"]
>-
>- task["skeleton"].append(cmd)
>
>- if self._check_task(task):
>- raise RecipeError("Incorrect command sequence.",
task_data)
>+ iface = machine.get_interface(if_id)
>
>- self._tasks.append(task)
>+ if iface_data["netns"] != None:
>+ iface.set_netns(iface_data["netns"])
>
> def _prepare_command(self, cmd_data):
> cmd = {"type": cmd_data["type"]}
>@@ -635,36 +495,23 @@ class NetTestController:
> os.remove("/tmp/.lnst_machine_conf")
>
> def match_setup(self):
>+ self.run_mode = "match_setup"
>+ res = self._run_python_task()
> return {"passed": True}
>
> def config_only_recipe(self):
>+ self.run_mode = "config_only"
> try:
>- self._prepare_network(resource_sync=False)
>- except (KeyboardInterrupt, Exception) as exc:
>- msg = "Exception raised during configuration."
>- logging.error(msg)
>- self._cleanup_slaves()
>+ res = self._run_recipe()
>+ except Exception as exc:
>+ logging.error("Recipe execution terminated by unexpected
exception")
> raise
>-
>- self._save_machine_config()
>-
>- self._cleanup_slaves(deconfigure=False)
>- return {"passed": True}
>-
>- def run_recipe(self):
>- try:
>- self._prepare_tasks()
>- self._prepare_network()
>- except (KeyboardInterrupt, Exception) as exc:
>- msg = "Exception raised during configuration."
>- logging.error(msg)
>+ finally:
> self._cleanup_slaves()
>- raise
>
>- if self._packet_capture:
>- self._start_packet_capture()
>+ return res
>
>- err = None
>+ def run_recipe(self):
> try:
> res = self._run_recipe()
> except Exception as exc:
>@@ -699,70 +546,44 @@ class NetTestController:
> def _run_recipe(self):
> overall_res = {"passed": True}
>
>- for task in self._tasks:
>+ try:
> self._res_serializer.add_task()
>- try:
>- res = self._run_task(task)
>- except CommandException as exc:
>- logging.debug(exc)
>- overall_res["passed"] = False
>- overall_res["err_msg"] = "Command exception
raised."
>- break
>-
>- for machine in self._machines.itervalues():
>- machine.restore_system_config()
>-
>- # task failed, check if we should quit_on_fail
>- if not res:
>- overall_res["passed"] = False
>- overall_res["err_msg"] = "At least one command
failed."
>- if task["quit_on_fail"]:
>- break
>+ res = self._run_python_task()
>+ except CommandException as exc:
>+ logging.debug(exc)
>+ overall_res["passed"] = False
>+ overall_res["err_msg"] = "Command exception raised."
>+
>+ for machine in self._machines.itervalues():
>+ machine.restore_system_config()
>+
>+ # task failed
>+ if not res:
>+ overall_res["passed"] = False
>+ overall_res["err_msg"] = "At least one command
failed."
>
> return overall_res
>
> def init_taskapi(self):
> Task.ctl = Task.ControllerAPI(self)
>
>- def _run_python_task(self, task):
>+ def _run_python_task(self):
> #backup of resource table
> res_table_bkp = copy.deepcopy(self._resource_table)
>- if 'module_dir' in task:
>- modules = self._load_test_modules([task['module_dir']])
>- self._resource_table['module'].update(modules)
>- if 'tools_dir' in task:
>- tools = self._load_test_tools([task['tools_dir']])
>- self._resource_table['tools'].update(tools)
>-
>- # Initialize the API handle
>- Task.ctl = Task.ControllerAPI(self, self._machines)
>
> cwd = os.getcwd()
>- task_path = task["python"]
>+ task_path = self._recipe_path
> name = os.path.basename(task_path.abs_path()).split(".")[0]
> sys.path.append(os.path.dirname(task_path.resolve()))
> os.chdir(os.path.dirname(task_path.resolve()))
>- module = imp.load_source(name, task_path.resolve())
>+ imp.load_source(name, task_path.resolve())
> os.chdir(cwd)
> sys.path.remove(os.path.dirname(task_path.resolve()))
>
> #restore resource table
> self._resource_table = res_table_bkp
>
>- return module.ctl._result
>-
>- def _run_task(self, task):
>- if "python" in task:
>- return self._run_python_task(task)
>-
>- seq_passed = True
>- for cmd_data in task["commands"]:
>- cmd = self._prepare_command(cmd_data)
>- cmd_res = self._run_command(cmd)
>- if not cmd_res["passed"]:
>- seq_passed = False
>-
>- return seq_passed
>+ return Task.ctl._result
>
> def _run_command(self, command):
> logging.info("Executing command: [%s]", str_command(command))
>@@ -891,12 +712,11 @@ class NetTestController:
> return packages
>
> def _get_alias(self, alias):
>- templates = self._parser._template_proc
>- return templates._find_definition(alias)
>+ if alias in self._defined_aliases:
>+ return self._defined_aliases[alias]
>
> def _get_aliases(self):
>- templates = self._parser._template_proc
>- return templates._dump_definitions()
>+ return self._defined_aliases
>
> class MessageDispatcher(ConnectionHandler):
> def __init__(self, log_ctl):
>diff --git a/lnst/Controller/Task.py b/lnst/Controller/Task.py
>index e36b79f..9f40846 100644
>--- a/lnst/Controller/Task.py
>+++ b/lnst/Controller/Task.py
>@@ -88,7 +88,7 @@ class TaskError(Exception): pass
> class ControllerAPI(object):
> """ An API class representing the controller. """
>
>- def __init__(self, ctl, hosts):
>+ def __init__(self, ctl):
> self._ctl = ctl
> self.run_mode = ctl.run_mode
> self._result = True
>@@ -99,8 +99,6 @@ class ControllerAPI(object):
> self._perf_repo_api = PerfRepoAPI()
>
> self._hosts = {}
>- for host_id, host in hosts.iteritems():
>- self._hosts[host_id] = HostAPI(self, host_id, host)
>
> def get_mreq(self):
> return self.mreq
>@@ -188,7 +186,7 @@ class ControllerAPI(object):
> cmd = {"type": "ctl_wait", "seconds":
int(seconds)}
> return self._ctl._run_command(cmd)
>
>- def get_alias(self, alias):
>+ def get_alias(self, alias, default=None):
> """
> Get the value of user defined alias.
>
>@@ -199,9 +197,13 @@ class ControllerAPI(object):
> :rtype: string
> """
> try:
>- return self._ctl._get_alias(alias)
>+ val = self._ctl._get_alias(alias)
>+ if val is None:
>+ return default
>+ else:
>+ return val
> except XmlTemplateError:
>- return None
>+ return default
>
> def get_aliases(self):
> """
>@@ -269,16 +271,12 @@ class ControllerAPI(object):
> class HostAPI(object):
> """ An API class representing a host machine. """
>
>- def __init__(self, ctl, host_id, host):
>+ def __init__(self, ctl, host_id):
> self._ctl = ctl
> self._id = host_id
>- self._m = host
>+ self._m = None
>
> self._ifaces = {}
>- for interface in self._m.get_interfaces():
>- if interface.get_id() is None:
>- continue
>- self._ifaces[interface.get_id()] = InterfaceAPI(interface, self)
> self._if_id_seq = 0
> self._bg_id_seq = 0
>
>@@ -313,7 +311,7 @@ class HostAPI(object):
> return self._ifaces[if_id]
>
> def get_id(self):
>- return self._m.get_id()
>+ return self._id
>
> def get_configuration(self):
> return self._m.get_configuration()
>--
>2.4.11
>_______________________________________________
>LNST-developers mailing list
>lnst-developers(a)lists.fedorahosted.org
>https://lists.fedorahosted.org/admin/lists/lnst-developers@lists.fedorahosted.org