4 a.m.
This patch implements PyRecipes and breaks existing XML recipes.
Python recipes, unlike XML recipes, do not need to be parsed before
execution. They contain both network topology definition and test
commands.
Network topology is defined in the beginning of PyRecipe, example:
import lnst
m1 = lnst.add_host()
m2 = lnst.add_host()
m1_eth1 = m1.add_interface(label="tnet")
m2_eth1 = m2.add_interface(label="tnet")
In the example, test will use 2 different hosts, each with 1 eth device.
To run match and network provision, use lnst.match() method in while
loop, example:
while lnst.match():
m1_eth1.reset(ip="192.168.0.1/24")
m2_eth1.reset(ip="192.168.0.2/24")
This piece of code will run the match, prepare the setup and sets IP
addresses to both eth devices.
More info on how to use Pyrecipes will be in our documentation (github wiki).
After this patch, XML recipes will no longer work. It removes and
modifies a lot of code in order to not break git bisect.
Signed-off-by: Jiri Prochazka <jprochaz(a)redhat.com>
---
lnst-ctl | 13 +-
lnst/Controller/NetTestController.py | 266 ++++++-----------------------------
lnst/Controller/Task.py | 24 ++--
3 files changed, 62 insertions(+), 241 deletions(-)
diff --git a/lnst-ctl b/lnst-ctl
index b75b8a0..93b084b 100755
--- a/lnst-ctl
+++ b/lnst-ctl
@@ -178,9 +178,13 @@ def get_recipe_result(action, file_path, log_ctl, res_serializer,
res = {}
if matches == 1:
try:
- nettestctl.provision_machines()
- nettestctl.print_match_description()
+ # init TaskAPI.Ctl
+ nettestctl.init_taskapi()
res = exec_action(action, nettestctl)
+ except NoMatchError as err:
+ no_match = True
+ log_ctl.unset_recipe()
+ logging.warning("Match %d not possible." % matches)
except Exception as err:
no_match = True
log_exc_traceback()
@@ -190,12 +194,11 @@ def get_recipe_result(action, file_path, log_ctl, res_serializer,
retval = RETVAL_ERR
elif matches > 1:
try:
- nettestctl.provision_machines()
+ nettestctl.init_taskapi()
log_ctl.set_recipe(file_path, prepend=True, expand="match_%d" %
matches)
log_dir = log_ctl.get_recipe_log_path()
recipe_head_log_entry(file_path, log_dir, matches)
res_serializer.add_recipe(file_path, matches)
- nettestctl.print_match_description()
res = exec_action(action, nettestctl)
except NoMatchError as err:
no_match = True
@@ -400,7 +403,7 @@ def main():
all_files.sort()
for f in all_files:
recipe_file = os.path.join(recipe_path, f)
- if re.match(r'^.*\.xml$', recipe_file):
+ if re.match(r'^.*\.py$', recipe_file):
recipe_files.append(recipe_file)
else:
recipe_files.append(recipe_path)
diff --git a/lnst/Controller/NetTestController.py b/lnst/Controller/NetTestController.py
index 0115d2f..135bbf6 100644
--- a/lnst/Controller/NetTestController.py
+++ b/lnst/Controller/NetTestController.py
@@ -61,11 +61,10 @@ class NetTestController:
self._res_serializer = res_serializer
self._remote_capture_files = {}
self._log_ctl = log_ctl
- self._recipe_path = Path(None, recipe_path).abs_path()
+ self._recipe_path = Path(None, recipe_path)
self._msg_dispatcher = MessageDispatcher(log_ctl)
self._packet_capture = packet_capture
self._reduce_sync = reduce_sync
- self._parser = RecipeParser(recipe_path)
self._defined_aliases = defined_aliases
self._multi_match = multi_match
@@ -80,10 +79,6 @@ class NetTestController:
mac_pool_range = lnst_config.get_option('environment',
'mac_pool_range')
self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
- self._parser.set_machines(self._machines)
- self._parser.set_aliases(defined_aliases, overriden_aliases)
- self._recipe = self._parser.parse()
-
conf_pools = lnst_config.get_pools()
pools = {}
if len(restrict_pools) > 0:
@@ -100,9 +95,6 @@ class NetTestController:
sp = SlavePool(pools, pool_checks)
self._slave_pool = sp
- mreq = self._get_machine_requirements()
- sp.set_machine_requirements(mreq)
-
modules_dirs = lnst_config.get_option('environment',
'module_dirs')
tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
@@ -196,21 +188,21 @@ class NetTestController:
return mreq
def _prepare_network(self, resource_sync=True):
- recipe = self._recipe
+ mreq = Task.get_mreq()
machines = self._machines
for m_id in machines.keys():
self._prepare_machine(m_id, resource_sync)
- for machine_xml_data in recipe["machines"]:
- m_id = machine_xml_data["id"]
+ for machine_id, machine_data in mreq.iteritems():
+ m_id = machine_id
m = machines[m_id]
namespaces = set()
- for iface_xml_data in machine_xml_data["interfaces"]:
- self._prepare_interface(m_id, iface_xml_data)
+ for if_id, iface_data in machine_data["interfaces"].iteritems():
+ self._prepare_interface(m_id, if_id, iface_data)
- if iface_xml_data["netns"] != None:
- namespaces.add(iface_xml_data["netns"])
+ if iface_data["netns"] != None:
+ namespaces.add(iface_data["netns"])
if len(namespaces) > 0:
m.disable_nm()
@@ -269,148 +261,16 @@ class NetTestController:
machine.set_mac_pool(self._mac_pool)
machine.set_network_bridges(self._network_bridges)
- recipe_name = os.path.basename(self._recipe_path)
+ recipe_name = os.path.basename(self._recipe_path.abs_path())
machine.init_connection(recipe_name)
- sync_table = {'module': {}, 'tools': {}}
- if resource_sync:
- res_table = copy.deepcopy(self._resource_table)
- for task in self._recipe['tasks']:
- if 'module_dir' in task:
- modules = self._load_test_modules([task['module_dir']])
- res_table['module'].update(modules)
- if 'tools_dir' in task:
- tools = self._load_test_tools([task['tools_dir']])
- res_table['tools'].update(tools)
-
- if 'commands' not in task:
- if not self._reduce_sync:
- sync_table = res_table
- break
- else:
- continue
- for cmd in task['commands']:
- if 'host' not in cmd or cmd['host'] != m_id:
- continue
- if cmd['type'] == 'test':
- mod = cmd['module']
- if mod in res_table['module']:
- sync_table['module'][mod] =
res_table['module'][mod]
- # check if test module uses some test tools
- mod_path =
res_table['module'][mod]["path"]
- mod_tools = get_module_tools(mod_path)
- for t in mod_tools:
- if t in sync_table['tools']:
- continue
- logging.debug("Adding '%s' tool as "\
- "dependency of %s test module" % (t, mod))
- sync_table['tools'][t] =
res_table['tools'][t]
- else:
- msg = "Module '%s' not found on the
controller"\
- % mod
- raise RecipeError(msg, cmd)
- if cmd['type'] == 'exec' and 'from' in cmd:
- tool = cmd['from']
- if tool in res_table['tools']:
- sync_table['tools'][tool] =
res_table['tools'][tool]
- else:
- msg = "Tool '%s' not found on the
controller" % tool
- raise RecipeError(msg, cmd)
- machine.sync_resources(sync_table)
-
- def _prepare_interface(self, m_id, iface_xml_data):
+ def _prepare_interface(self, m_id, if_id, iface_data):
machine = self._machines[m_id]
- if_id = iface_xml_data["id"]
- if_type = iface_xml_data["type"]
-
- try:
- iface = machine.get_interface(if_id)
- except MachineError:
- if if_type == 'lo':
- iface = machine.new_loopback_interface(if_id)
- else:
- iface = machine.new_soft_interface(if_id, if_type)
-
- if "slaves" in iface_xml_data:
- for slave in iface_xml_data["slaves"]:
- slave_id = slave["id"]
- iface.add_slave(machine.get_interface(slave_id))
-
- # Some soft devices (such as team) use per-slave options
- if "options" in slave:
- for opt in slave["options"]:
- iface.set_slave_option(slave_id, opt["name"],
- opt["value"])
-
- if "addresses" in iface_xml_data:
- for addr in iface_xml_data["addresses"]:
- iface.add_address(addr)
-
- if "options" in iface_xml_data:
- for opt in iface_xml_data["options"]:
- iface.set_option(opt["name"], opt["value"])
-
- if "netem" in iface_xml_data:
- iface.set_netem(iface_xml_data["netem"].to_dict())
-
- if "ovs_conf" in iface_xml_data:
- iface.set_ovs_conf(iface_xml_data["ovs_conf"].to_dict())
-
- if iface_xml_data["netns"] != None:
- iface.set_netns(iface_xml_data["netns"])
-
- if "peer" in iface_xml_data:
- iface.set_peer(iface_xml_data["peer"])
-
- def _prepare_tasks(self):
- self._tasks = []
- for task_data in self._recipe["tasks"]:
- task = {}
- task["quit_on_fail"] = False
- if "quit_on_fail" in task_data:
- task["quit_on_fail"] =
bool_it(task_data["quit_on_fail"])
-
- if "module_dir" in task_data:
- task["module_dir"] = task_data["module_dir"]
-
- if "tools_dir" in task_data:
- task["tools_dir"] = task_data["tools_dir"]
-
- if "python" in task_data:
- root = Path(None, self._recipe_path).get_root()
- path = Path(root, task_data["python"])
-
- task["python"] = path
- if not path.exists():
- msg = "Task file '%s' not found." % path.to_str()
- raise RecipeError(msg, task_data)
-
- self._tasks.append(task)
- continue
-
- task["commands"] = task_data["commands"]
- task["skeleton"] = []
- for cmd_data in task["commands"]:
- cmd = {"type": cmd_data["type"]}
-
- if "host" in cmd_data:
- cmd["host"] = cmd_data["host"]
- if cmd["host"] not in self._machines:
- msg = "Invalid host id '%s'." %
cmd["host"]
- raise RecipeError(msg, cmd_data)
-
- if cmd["type"] in ["test", "exec"]:
- if "bg_id" in cmd_data:
- cmd["bg_id"] = cmd_data["bg_id"]
- elif cmd["type"] in ["wait", "intr",
"kill"]:
- cmd["proc_id"] = cmd_data["bg_id"]
-
- task["skeleton"].append(cmd)
- if self._check_task(task):
- raise RecipeError("Incorrect command sequence.", task_data)
+ iface = machine.get_interface(if_id)
- self._tasks.append(task)
+ if iface_data["netns"] != None:
+ iface.set_netns(iface_data["netns"])
def _prepare_command(self, cmd_data):
cmd = {"type": cmd_data["type"]}
@@ -635,36 +495,23 @@ class NetTestController:
os.remove("/tmp/.lnst_machine_conf")
def match_setup(self):
+ self.run_mode = "match_setup"
+ res = self._run_python_task()
return {"passed": True}
def config_only_recipe(self):
+ self.run_mode = "config_only"
try:
- self._prepare_network(resource_sync=False)
- except (KeyboardInterrupt, Exception) as exc:
- msg = "Exception raised during configuration."
- logging.error(msg)
- self._cleanup_slaves()
+ res = self._run_recipe()
+ except Exception as exc:
+ logging.error("Recipe execution terminated by unexpected
exception")
raise
-
- self._save_machine_config()
-
- self._cleanup_slaves(deconfigure=False)
- return {"passed": True}
-
- def run_recipe(self):
- try:
- self._prepare_tasks()
- self._prepare_network()
- except (KeyboardInterrupt, Exception) as exc:
- msg = "Exception raised during configuration."
- logging.error(msg)
+ finally:
self._cleanup_slaves()
- raise
- if self._packet_capture:
- self._start_packet_capture()
+ return res
- err = None
+ def run_recipe(self):
try:
res = self._run_recipe()
except Exception as exc:
@@ -699,70 +546,44 @@ class NetTestController:
def _run_recipe(self):
overall_res = {"passed": True}
- for task in self._tasks:
+ try:
self._res_serializer.add_task()
- try:
- res = self._run_task(task)
- except CommandException as exc:
- logging.debug(exc)
- overall_res["passed"] = False
- overall_res["err_msg"] = "Command exception raised."
- break
-
- for machine in self._machines.itervalues():
- machine.restore_system_config()
-
- # task failed, check if we should quit_on_fail
- if not res:
- overall_res["passed"] = False
- overall_res["err_msg"] = "At least one command
failed."
- if task["quit_on_fail"]:
- break
+ res = self._run_python_task()
+ except CommandException as exc:
+ logging.debug(exc)
+ overall_res["passed"] = False
+ overall_res["err_msg"] = "Command exception raised."
+
+ for machine in self._machines.itervalues():
+ machine.restore_system_config()
+
+ # task failed
+ if not res:
+ overall_res["passed"] = False
+ overall_res["err_msg"] = "At least one command failed."
return overall_res
def init_taskapi(self):
Task.ctl = Task.ControllerAPI(self)
- def _run_python_task(self, task):
+ def _run_python_task(self):
#backup of resource table
res_table_bkp = copy.deepcopy(self._resource_table)
- if 'module_dir' in task:
- modules = self._load_test_modules([task['module_dir']])
- self._resource_table['module'].update(modules)
- if 'tools_dir' in task:
- tools = self._load_test_tools([task['tools_dir']])
- self._resource_table['tools'].update(tools)
-
- # Initialize the API handle
- Task.ctl = Task.ControllerAPI(self, self._machines)
cwd = os.getcwd()
- task_path = task["python"]
+ task_path = self._recipe_path
name = os.path.basename(task_path.abs_path()).split(".")[0]
sys.path.append(os.path.dirname(task_path.resolve()))
os.chdir(os.path.dirname(task_path.resolve()))
- module = imp.load_source(name, task_path.resolve())
+ imp.load_source(name, task_path.resolve())
os.chdir(cwd)
sys.path.remove(os.path.dirname(task_path.resolve()))
#restore resource table
self._resource_table = res_table_bkp
- return module.ctl._result
-
- def _run_task(self, task):
- if "python" in task:
- return self._run_python_task(task)
-
- seq_passed = True
- for cmd_data in task["commands"]:
- cmd = self._prepare_command(cmd_data)
- cmd_res = self._run_command(cmd)
- if not cmd_res["passed"]:
- seq_passed = False
-
- return seq_passed
+ return Task.ctl._result
def _run_command(self, command):
logging.info("Executing command: [%s]", str_command(command))
@@ -891,12 +712,11 @@ class NetTestController:
return packages
def _get_alias(self, alias):
- templates = self._parser._template_proc
- return templates._find_definition(alias)
+ if alias in self._defined_aliases:
+ return self._defined_aliases[alias]
def _get_aliases(self):
- templates = self._parser._template_proc
- return templates._dump_definitions()
+ return self._defined_aliases
class MessageDispatcher(ConnectionHandler):
def __init__(self, log_ctl):
diff --git a/lnst/Controller/Task.py b/lnst/Controller/Task.py
index e36b79f..9f40846 100644
--- a/lnst/Controller/Task.py
+++ b/lnst/Controller/Task.py
@@ -88,7 +88,7 @@ class TaskError(Exception): pass
class ControllerAPI(object):
""" An API class representing the controller. """
- def __init__(self, ctl, hosts):
+ def __init__(self, ctl):
self._ctl = ctl
self.run_mode = ctl.run_mode
self._result = True
@@ -99,8 +99,6 @@ class ControllerAPI(object):
self._perf_repo_api = PerfRepoAPI()
self._hosts = {}
- for host_id, host in hosts.iteritems():
- self._hosts[host_id] = HostAPI(self, host_id, host)
def get_mreq(self):
return self.mreq
@@ -188,7 +186,7 @@ class ControllerAPI(object):
cmd = {"type": "ctl_wait", "seconds":
int(seconds)}
return self._ctl._run_command(cmd)
- def get_alias(self, alias):
+ def get_alias(self, alias, default=None):
"""
Get the value of user defined alias.
@@ -199,9 +197,13 @@ class ControllerAPI(object):
:rtype: string
"""
try:
- return self._ctl._get_alias(alias)
+ val = self._ctl._get_alias(alias)
+ if val is None:
+ return default
+ else:
+ return val
except XmlTemplateError:
- return None
+ return default
def get_aliases(self):
"""
@@ -269,16 +271,12 @@ class ControllerAPI(object):
class HostAPI(object):
""" An API class representing a host machine. """
- def __init__(self, ctl, host_id, host):
+ def __init__(self, ctl, host_id):
self._ctl = ctl
self._id = host_id
- self._m = host
+ self._m = None
self._ifaces = {}
- for interface in self._m.get_interfaces():
- if interface.get_id() is None:
- continue
- self._ifaces[interface.get_id()] = InterfaceAPI(interface, self)
self._if_id_seq = 0
self._bg_id_seq = 0
@@ -313,7 +311,7 @@ class HostAPI(object):
return self._ifaces[if_id]
def get_id(self):
- return self._m.get_id()
+ return self._id
def get_configuration(self):
return self._m.get_configuration()
--
2.4.11