[lnst] NetTest: Adding support for the new parser
by Jiří Pírko
commit 86185c61918b6e22f6c342bb6aaed16f54b70b7b
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Wed Jul 18 17:03:38 2012 +0200
NetTest: Adding support for the new parser
This commit modifies current NetTestController and NetTestSlave
implementations, so they can work with the new parser.
Controller used to pass the interface configuration all at once.
This was modified so the config is now transfered one interface at
a time.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
NetTest/NetTestController.py | 277 +++++++++++++++++++++++-------------------
NetTest/NetTestSlave.py | 35 ++++--
2 files changed, 175 insertions(+), 137 deletions(-)
---
diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py
index a33cacb..4daf070 100644
--- a/NetTest/NetTestController.py
+++ b/NetTest/NetTestController.py
@@ -18,146 +18,182 @@ from Common.Logs import Logs
from Common.SshUtils import scp_from_remote
from pprint import pprint, pformat
from Common.XmlRpc import ServerProxy
-from NetTestParse import NetTestParse
+from NetTest.NetTestParse import NetTestParse
from Common.SlaveUtils import prepare_client_session
from Common.Utils import get_corespond_local_ip
-from NetTestSlave import DefaultRPCPort
-from NetTestCommand import NetTestCommand, str_command
+from NetTest.NetTestSlave import DefaultRPCPort
+from NetTest.NetTestCommand import NetTestCommand, str_command
from Common.LoggingServer import LoggingServer
+def ignore_event(**kwarg):
+ pass
+
class NetTestController:
def __init__(self, recipe_path, remoteexec=False, cleanup=False,
res_serializer=None):
- ntparse = NetTestParse(recipe_path)
- ntparse.parse_recipe()
- self._recipe = ntparse.get_recipe()
- self._ntparse = ntparse
self._remoteexec = remoteexec
self._docleanup = cleanup
self._res_serializer = res_serializer
self._remote_capture_files = {}
+ self._recipe = {}
+ definitions = {"recipe": self._recipe}
+
+ ntparse = NetTestParse(recipe_path)
+ ntparse.set_recipe(self._recipe)
+ ntparse.set_definitions(definitions)
+
+ ntparse.register_event_handler("netdevice_ready", ignore_event)
+ ntparse.register_event_handler("machine_info_ready",
+ self._prepare_slave)
+ ntparse.register_event_handler("interface_config_ready",
+ self._prepare_interface)
+
+ self._ntparse = ntparse
+
def _get_machineinfo(self, machine_id):
- return self._recipe["machines"][machine_id]["info"]
+ try:
+ info = self._recipe["machines"][machine_id]["info"]
+ except KeyError:
+ msg = "Machine info is required, but not yet available"
+ raise Exception(msg)
+
+ return info
def _get_machinerpc(self, machine_id):
- return self._get_machineinfo(machine_id)["rpc"]
+ try:
+ rpc = self._get_machineinfo(machine_id)["rpc"]
+ except KeyError:
+ msg = "XMLRPC connection required, but not yet available"
+ raise Exception(msg)
- def _session_die(self, session, status):
- logging.error("Session started with cmd %s die with status %s." % (session.command, status))
+ return rpc
+
+ @staticmethod
+ def _session_die(session, status):
+ logging.error("Session started with cmd %s die with status %s.",
+ session.command, status)
raise Exception("Session Die.")
- def _prepare_slaves(self):
- for machine_id in self._recipe["machines"]:
- info = self._get_machineinfo(machine_id)
- hostname = info["hostname"]
- logging.info("Remote app exec on machine %s" % hostname)
- port = "22"
- login = "root"
- if "rootpass" in info:
- passwd = info["rootpass"]
- else:
- passwd = None
- session = prepare_client_session(hostname, port, login, passwd,
- "nettestslave.py")
- session.add_kill_handler(self._session_die)
- info["session"] = session
+ def _prepare_interface(self, machine_id, netdev_config_id):
+ rpc = self._get_machinerpc(machine_id)
+ info = self._get_machineinfo(machine_id)
+ logging.info("Configuring interface #%d on %s", netdev_config_id,
+ info["hostname"])
- def _cleanup_slaves(self):
- for machine_id in self._recipe["machines"]:
- info = self._get_machineinfo(machine_id)
- if "session" in info:
- info["session"].kill()
- info["session"].wait()
+ self._configure_interface(machine_id, netdev_config_id)
- def _rpc_connect(self):
- for machine_id in self._recipe["machines"]:
- info = self._get_machineinfo(machine_id)
- hostname = info["hostname"]
- logging.info("Connecting to RPC on machine \"%s\"" % hostname)
- if "rpcport" in info:
- port = info["rpcport"]
- else:
- port = DefaultRPCPort
- url = "http://%s:%d" % (hostname, port)
- rpc = ServerProxy(url, allow_none = True)
- if rpc.hello() != "hello":
- logging.error("Handshake error with machine id %d" % machine_id)
- raise Exception("Hanshake error")
- if self._docleanup:
- rpc.machine_cleanup()
- info["rpc"] = rpc
-
- def _logging_connect(self):
- for machine_id in self._recipe["machines"]:
- info = self._get_machineinfo(machine_id)
- logging.info("Setting logging server on machine \"%s\"" % info["hostname"])
- rpc = self._get_machinerpc(machine_id)
- ip = get_corespond_local_ip(info["hostname"])
- rpc.set_logging(ip, LoggingServer.DEFAULT_PORT)
-
- def _netconfigs_set(self):
- machines = self._recipe["machines"]
- for machine_id in machines:
- machine = machines[machine_id]
- logging.info("Setting netconfigs on machine \"%s\"" % machine["info"]["hostname"])
- rpc = self._get_machinerpc(machine_id)
- rpc.netconfig_set(machine["netmachineconfig_xml"],
- machine["netconfig_xml"])
- '''
- Finally get processed netconfig from slave back to us.
- Will be handy later on.
- '''
- machine["netconfig"] = dict(rpc.netconfig_dump())
- del machine["netmachineconfig_xml"]
- del machine["netconfig_xml"]
-
- def _netconfigs_clear(self):
- machines = self._recipe["machines"]
- for machine_id in machines:
- machine = machines[machine_id]
- logging.info("Clearing netconfigs on machine \"%s\"" % machine["info"]["hostname"])
+ if_info = rpc.get_interface_info(netdev_config_id)
+ machine = self._recipe["machines"][machine_id]
+ if "name" in if_info:
+ machine["netconfig"][netdev_config_id]["name"] = if_info["name"]
+
+ info["configured_interfaces"].append(netdev_config_id)
+
+ def _configure_interface(self, machine_id, netdev_config_id):
+ rpc = self._get_machinerpc(machine_id)
+ netconfig = self._recipe["machines"][machine_id]["netconfig"]
+ dev_config = netconfig[netdev_config_id]
+
+ rpc.configure_interface(netdev_config_id, dev_config)
+
+ def _deconfigure_interface(self, machine_id, netdev_config_id):
+ rpc = self._get_machinerpc(machine_id)
+ rpc.deconfigure_interface(netdev_config_id)
+
+ def _prepare_slave(self, machine_id):
+ logging.info("Preparing machine #%d", machine_id)
+ info = self._get_machineinfo(machine_id)
+
+ if self._remoteexec and not "session" in info:
+ self._init_slave_session(machine_id)
+
+ if not "rpc" in info:
+ self._init_slave_rpc(machine_id)
+ self._init_slave_logging(machine_id)
+
+ info["configured_interfaces"] = []
+
+ def _init_slave_session(self, machine_id):
+ info = self._get_machineinfo(machine_id)
+ hostname = info["hostname"]
+ if "rootpass" in info:
+ passwd = info["rootpass"]
+ else:
+ passwd = None
+ logging.info("Remote app exec on machine %s", hostname)
+
+ port = "22"
+ login = "root"
+ session = prepare_client_session(hostname, port, login, passwd,
+ "nettestslave.py")
+ session.add_kill_handler(self._session_die)
+ info["session"] = session
+
+ def _init_slave_rpc(self, machine_id):
+ info = self._get_machineinfo(machine_id)
+ hostname = info["hostname"]
+ if "rpcport" in info:
+ port = info["rpcport"]
+ else:
+ port = DefaultRPCPort
+ logging.info("Connecting to RPC on machine %s", hostname)
+
+ url = "http://%s:%d" % (hostname, port)
+ rpc = ServerProxy(url, allow_none = True)
+ if rpc.hello() != "hello":
+ logging.error("Handshake error with machine %s", hostname)
+ raise Exception("Hanshake error")
+
+ if self._docleanup:
+ rpc.machine_cleanup()
+
+ info["rpc"] = rpc
+
+ def _init_slave_logging(self, machine_id):
+ info = self._get_machineinfo(machine_id)
+ hostname = info["hostname"]
+ logging.info("Setting logging server on machine %s", hostname)
+ rpc = self._get_machinerpc(machine_id)
+ ip_addr = get_corespond_local_ip(hostname)
+ rpc.set_logging(ip_addr, LoggingServer.DEFAULT_PORT)
+
+ def _cleanup_slave(self, machine_id):
+ info = self._get_machineinfo(machine_id)
+
+ if self._docleanup:
rpc = self._get_machinerpc(machine_id)
- rpc.netconfig_clear()
+ for if_id in reversed(info["configured_interfaces"]):
+ rpc.deconfigure_interface(if_id)
+
+ if self._remoteexec and "session" in info:
+ info["session"].kill()
+ info["session"].wait()
def _prepare(self):
- if self._remoteexec:
- self._prepare_slaves()
- self._rpc_connect()
- self._logging_connect()
- self._netconfigs_set()
-
- '''
- Now as we have all netconfigs processed by slaves back, it's time to
- parse recipe command sequence
- '''
- self._ntparse.parse_recipe_command_sequence()
- self._recipe = self._ntparse.get_recipe()
+ # All the perparations are made within the recipe parsing
+ # This is achieved by handling parser events (by registering
+ self._ntparse.parse_recipe()
def _cleanup(self):
- self._netconfigs_clear()
- if self._remoteexec:
- self._cleanup_slaves()
+ for machine_id in self._recipe["machines"]:
+ self._cleanup_slave(machine_id)
def _run_command(self, command):
- cmd_type = command["type"]
machine_id = command["machine_id"]
-
try:
desc = command["desc"]
- logging.info("Cmd description: %s" % desc)
+ logging.info("Cmd description: %s", desc)
except KeyError:
pass
if machine_id == 0:
cmd_res = NetTestCommand(command).run()
else:
- info = self._get_machineinfo(machine_id)
- hostname = info["hostname"]
rpc = self._get_machinerpc(machine_id)
if "timeout" in command:
timeout = command["timeout"]
- logging.debug("Setting socket timeout to \"%d\"" % timeout)
+ logging.debug("Setting socket timeout to \"%d\"", timeout)
socket.setdefaulttimeout(timeout)
try:
cmd_res = rpc.run_command(command)
@@ -182,36 +218,30 @@ class NetTestController:
def _run_command_sequence(self, sequence):
seq_passed = True
for command in sequence:
- logging.info("Executing command: [%s]" % str_command(command))
+ logging.info("Executing command: [%s]", str_command(command))
cmd_res = self._run_command(command)
if self._res_serializer:
self._res_serializer.add_cmd_result(command, cmd_res)
- logging.debug("Result: %s" % str(cmd_res))
+ logging.debug("Result: %s", str(cmd_res))
if "res_data" in cmd_res:
res_data = pformat(cmd_res["res_data"])
- logging.info("Result data: %s" % (res_data))
+ logging.info("Result data: %s", (res_data))
if not cmd_res["passed"]:
logging.error("Command failed - command: [%s], "
- "Error message: \"%s\""
- % (str_command(command), cmd_res["err_msg"]))
+ "Error message: \"%s\"",
+ str_command(command), cmd_res["err_msg"])
seq_passed = False
return seq_passed
def dump_recipe(self):
- pprint(self._recipe)
- return True
-
- def all_dump_recipe(self):
self._prepare()
pprint(self._recipe)
- if self._docleanup:
- self._cleanup()
+ self._cleanup()
return True
def config_only_recipe(self):
self._prepare()
- if self._docleanup:
- self._cleanup()
+ self._cleanup()
return True
def run_recipe(self, packet_capture=False):
@@ -223,15 +253,14 @@ class NetTestController:
err = None
try:
res = self._run_recipe()
- except e:
- err = e
+ except Exception, exc:
+ err = exc
if packet_capture:
self._stop_packet_capture()
self._gather_capture_files()
- if self._docleanup:
- self._cleanup()
+ self._cleanup()
if not err:
return res
@@ -275,8 +304,8 @@ class NetTestController:
slave_logging_dir = os.path.join(logging_root, hostname)
try:
os.mkdir(slave_logging_dir)
- except OSError, e:
- if e.errno != 17:
+ except OSError, err:
+ if err.errno != 17:
raise
capture_files = self._remote_capture_files[machine_id]
@@ -286,11 +315,6 @@ class NetTestController:
scp_from_remote(hostname, "22", "root", rootpass,
remote_path, local_path)
- def eval_expression_recipe(self, expr):
- self._prepare()
- value = eval("self._recipe%s" % expr)
- return True
-
def _update_system_config(self, machine_id, res_data, persistent=False):
info = self._get_machineinfo(machine_id)
system_config = info["system_config"]
@@ -300,7 +324,8 @@ class NetTestController:
del system_config[option]
else:
if not option in system_config:
- system_config[option] = {"initial_val": values["previous_val"]}
+ initial_val = {"initial_val": values["previous_val"]}
+ system_config[option] = initial_val
system_config[option]["current_val"] = values["current_val"]
diff --git a/NetTest/NetTestSlave.py b/NetTest/NetTestSlave.py
index 955c28e..272066d 100644
--- a/NetTest/NetTestSlave.py
+++ b/NetTest/NetTestSlave.py
@@ -20,7 +20,7 @@ from Common.XmlRpc import Server
from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
from NetConfig.NetConfig import NetConfig
from NetConfig.NetConfigDevice import NetConfigDeviceAllCleanup
-from NetTestCommand import NetTestCommand, CommandException
+from NetTest.NetTestCommand import NetTestCommand, CommandException
from Common.Utils import die_when_parent_die
DefaultRPCPort = 9999
@@ -32,6 +32,7 @@ class NetTestSlaveXMLRPC:
def __init__(self):
self._netconfig = None
self._packet_captures = {}
+ self._netconfig = NetConfig()
def hello(self):
return "hello"
@@ -45,19 +46,30 @@ class NetTestSlaveXMLRPC:
Logs.append_network_hadler(logger_address, port)
return True
- def netconfig_set(self, machine_xml_string, config_xml_string):
- self._netconfig = NetConfig(machine_xml_string, config_xml_string)
- self._netconfig.configure_all()
+ def get_interface_info(self, if_id):
+ if_config = self._netconfig.get_interface_config(if_id)
+ info = {}
+
+ if "name" in if_config:
+ info["name"] = if_config["name"]
+
+ if "hwaddr" in if_config:
+ info["hwaddr"] = if_config["hwaddr"]
+
+ return info
+
+ def configure_interface(self, if_id, config):
+ self._netconfig.add_interface_config(if_id, config)
+ self._netconfig.configure(if_id)
+ return True
+
+ def deconfigure_interface(self, if_id):
+ self._netconfig.deconfigure(if_id)
return True
def netconfig_dump(self):
return self._netconfig.dump_config().items()
- def netconfig_clear(self):
- self._netconfig.deconfigure_all()
- self.__init__()
- return True
-
def start_packet_capture(self, filt):
logging_dir = Logs.get_logging_root_path()
logging_dir = os.path.abspath(logging_dir)
@@ -91,8 +103,9 @@ class NetTestSlaveXMLRPC:
return NetTestCommand(command).run()
except:
import sys, traceback
- type, value, tb = sys.exc_info()
- logging.error(''.join(traceback.format_exception(type, value, tb)))
+ cmd_type, value, tb = sys.exc_info()
+ exception = traceback.format_exception(cmd_type, value, tb)
+ logging.error(''.join(exception))
raise CommandException(command)
def machine_cleanup(self):
11 years, 9 months
- 1
- 0
[lnst] NetTestParse: New parser implementation
by Jiří Pírko
commit f8f642679f7131c5615d8be0863ff97b1d46b3b3
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Wed Jul 18 17:03:37 2012 +0200
NetTestParse: New parser implementation
This patch replaces the original parser with another one that operates
on a different principle. It parses tags in order of their occurence
in the source XML file.
Some other features were added to the parser as well. For instance,
support of advanced error reporting that now includes location of
where the error occured (the exact file name and line number).
The parser is able to export some predefined events, that can be
used to signal the parent code. Parent can register handlers for
these events.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
NetTest/NetTestParse.py | 614 +++++++++++++++++++++++++++++-----------------
1 files changed, 387 insertions(+), 227 deletions(-)
---
diff --git a/NetTest/NetTestParse.py b/NetTest/NetTestParse.py
index 68c0058..7869cd0 100644
--- a/NetTest/NetTestParse.py
+++ b/NetTest/NetTestParse.py
@@ -10,224 +10,296 @@ __author__ = """
jpirko(a)redhat.com (Jiri Pirko)
"""
-from xml.dom.minidom import parseString
import logging
import os
import re
-from NetConfig.NetConfigParse import NetConfigParse
-from NetTest.NetTestCommand import str_command
-from Common.XmlPreprocessor import XmlPreprocessor
-
-def load_file(filename):
- handle = open(filename, "r")
- data = handle.read()
- handle.close()
- return data
-
-class WrongCommandSequenceException(Exception):
- pass
-
-class WrongIncludeSource(Exception):
- pass
-
-class NetTestParse:
- def __init__(self, recipe_path):
- recipe_path = os.path.expanduser(recipe_path)
- self._recipe_xml_string = load_file(recipe_path)
- self._dirpath = os.path.dirname(recipe_path)
- self._recipe = None
-
- self._xml_prep = XmlPreprocessor()
-
- def _get_referenced_xml_path(self, filename):
- return os.path.join(self._dirpath, os.path.expanduser(filename))
-
- def _parse_machine(self, dom_machine):
- machine = {}
-
- dom_netmachineconfig = dom_machine.getElementsByTagName("netmachineconfig")[0]
- netmachineconfig_xml = dom_netmachineconfig.toxml()
-
- dom_netconfig = dom_machine.getElementsByTagName("netconfig")[0]
- netconfig_xml = dom_netconfig.toxml()
-
- ncparse = NetConfigParse(netmachineconfig_xml)
- machine["info"] = ncparse.get_machine_info()
- machine["netmachineconfig_xml"] = netmachineconfig_xml
- machine["netconfig_xml"] = netconfig_xml
- return machine
-
- def _parse_machines(self, dom_machines_grp):
- machines = {}
- for dom_machines_item in dom_machines_grp:
- dom_machines = dom_machines_item.getElementsByTagName("machine")
- for dom_machine in dom_machines:
- machine_id = int(dom_machine.getAttribute("id"))
- machines[machine_id] = self._parse_machine(dom_machine)
- return machines
-
- def _parse_definitions(self, dom_definitions_grp):
- xml_prep = self._xml_prep
- definitions = {}
- for dom_definitions_item in dom_definitions_grp:
- dom_aliases = dom_definitions_item.getElementsByTagName("alias")
- for dom_alias in dom_aliases:
- alias_name = str(dom_alias.getAttribute("name"))
- alias_value = str(dom_alias.getAttribute("value"))
- xml_prep.define_alias(alias_name, alias_value)
+from Common.XmlProcessing import RecipeParser
+from Common.XmlProcessing import XmlDomTreeInit
+from Common.XmlProcessing import XmlProcessingError
+from NetConfig.NetConfigDevNames import normalize_hwaddr
- def parse_recipe(self):
- recipe = {}
- dom = parseString(self._recipe_xml_string)
- xml_prep = self._xml_prep
-
- xml_prep.remove_comments(dom)
-
- self._load_included_parts(dom)
- dom_nettestrecipe = dom.getElementsByTagName("nettestrecipe")[0]
-
- dom_definitions_grp = dom_nettestrecipe.getElementsByTagName("define")
- self._parse_definitions(dom_definitions_grp)
- for define_tag in dom_definitions_grp:
- parent = define_tag.parentNode
- parent.removeChild(define_tag)
-
- dom_machines_grp = dom_nettestrecipe.getElementsByTagName("machines")
- xml_prep.expand_group(dom_machines_grp)
- recipe["machines"] = self._parse_machines(dom_machines_grp)
-
- dom_switches_grp = dom_nettestrecipe.getElementsByTagName("switches")
- xml_prep.expand_group(dom_switches_grp)
- recipe["switches"] = self._parse_machines(dom_switches_grp)
-
- self._recipe = recipe
- self._dom_nettestrecipe = dom_nettestrecipe
- xml_prep.define_alias("recipe", recipe, skip_reserved_check=True)
-
- def get_recipe(self):
- return self._recipe
-
- def _load_included_parts(self, dom_node):
- if dom_node.nodeType == dom_node.ELEMENT_NODE:
- source = str(dom_node.getAttribute("source"))
- if source:
- file_path = self._get_referenced_xml_path(source)
- xml_data = load_file(file_path)
-
- dom = parseString(xml_data)
- loaded_node = None
- try:
- loaded_node = dom.getElementsByTagName(dom_node.nodeName)[0]
- except Exception:
- err = ("No '%s' node present in included file '%s'."
- % (dom_node.nodeName, file_path))
- raise WrongIncludeSource(err)
-
- parent = dom_node.parentNode
- parent.replaceChild(loaded_node, dom_node)
- self._load_included_parts(loaded_node)
- return
-
- for child in dom_node.childNodes:
- self._load_included_parts(child)
-
- def _recipe_eval(self, eval_data):
- try:
- return str(eval("self._recipe%s" % eval_data))
- except (KeyError, IndexError):
- logging.error("Wrong recipe_eval value \"%s\" passed"
- % eval_data)
- raise Exception
- @classmethod
- def _int_it(cls, val):
- try:
- num = int(val)
- except ValueError:
- num = 0
- return num
+class NetTestParse(RecipeParser):
+ def __init__(self, recipe_filepath):
+ super(NetTestParse, self).__init__()
- @classmethod
- def _bool_it(cls, val):
- if isinstance(val, str):
- if re.match("^\s*(?i)(true)", val):
- return True
- elif re.match("^\s*(?i)(false)", val):
- return False
- return True if cls._int_it(val) else False
+ self._filepath = recipe_filepath
+ self._include_root = os.path.dirname(recipe_filepath)
- def _parse_command_option(self, dom_option, options):
- logging.debug("Parsing command option")
- option_type = str(dom_option.getAttribute("type"))
- orig_value = None
- if not option_type:
- name = str(dom_option.getAttribute("name"))
- value = str(dom_option.getAttribute("value"))
- elif option_type == "recipe_eval":
- name = str(dom_option.getAttribute("name"))
- orig_value = str(dom_option.getAttribute("value"))
- value = str(self._recipe_eval(orig_value))
+ self._recipe = {}
+ self._template_proc.set_definitions({"recipe": self._recipe})
+
+ def parse_recipe(self):
+ dom_init = XmlDomTreeInit()
+ xml_dom = dom_init.parse_file(self._filepath)
+ self._parse(xml_dom)
+
+ def _parse(self, xml_dom):
+ if xml_dom.nodeType == xml_dom.DOCUMENT_NODE:
+ scheme = {"nettestrecipe": self._nettestrecipe}
+ self._process_child_nodes(xml_dom, scheme)
else:
- logging.error("Unknown option type \"%s\"" % option_type)
- raise Exception("Unknown option type")
-
- logging.debug("Command option name \"%s\", value \"%s\""
- % (name, value))
- option = {"value": value}
- if option_type:
- option["type"] = option_type
- if orig_value:
- option["orig_value"] = orig_value
- if not name in options:
- options[name] = []
- options[name].append(option)
+ raise XmlProcessingError("Passed object is not a XML document")
+
+ def _nettestrecipe(self, node, params):
+ scheme = {"machines": self._machines,
+ "switches": self._switches,
+ "command_sequence": self._command_sequence}
+ self._process_child_nodes(node, scheme)
+
+ def _machines(self, node, params):
+ self._recipe["machines"] = {}
+ scheme = {"machine": self._machine}
+ self._process_child_nodes(node, scheme)
+
+ def _machine(self, node, params):
+ subparser = MachineParse(self)
+ subparser.set_type("host")
+ subparser.parse(node)
+
+ def _switches(self, node, params):
+ self._recipe["switches"] = {}
+ scheme = {"switch": self._switch}
+ self._process_child_nodes(node, scheme)
+
+ def _switch(self, node, params):
+ subparser = MachineParse(self)
+ subparser.set_type("switch")
+ subparser.parse(node)
+
+ def _command_sequence(self, node, params):
+ if not "sequences" in self._recipe:
+ self._recipe["sequences"] = []
+
+ subparser = CommandSequenceParse(self)
+ subparser.parse(node)
+
+
+class MachineParse(RecipeParser):
+ _target = "machines"
+
+ def set_type(self, machine_type):
+ if machine_type == "host":
+ self._target = "machines"
+ elif machine_type == "switch":
+ self._target = "switches"
+ else:
+ raise XmlProcessingError("Unknown machine type")
- def _parse_command(self, dom_command):
- logging.debug("Parsing command")
- recipe = self._recipe
- tmp = dom_command.getAttribute("machine_id")
- if tmp:
- machine_id = int(tmp)
- if machine_id and not machine_id in recipe["machines"]:
- logging.error("Invalid machine id")
- raise Exception("Invalid machine id")
+ def parse(self, node):
+ self._id = self._get_attribute(node, "id", int)
+ self._machine = {}
+ self._recipe[self._target][self._id] = self._machine
+
+ self._machine["info"] = {}
+ self._machine["netdevices"] = {}
+ self._machine["netconfig"] = {}
+
+ scheme = {"netmachineconfig": self._netmachineconfig,
+ "netconfig": self._netconfig }
+ self._process_child_nodes(node, scheme)
+
+ def _netmachineconfig(self, node, params):
+ subparser = NetMachineConfigParse(self)
+ subparser.set_machine(self._id, self._machine)
+ subparser.parse(node)
+
+ def _netconfig(self, node, params):
+ subparser = NetConfigParse(self)
+ subparser.set_machine(self._id, self._machine)
+ subparser.parse(node)
+
+class NetMachineConfigParse(RecipeParser):
+ _machine_id = None
+ _machine = None
+
+ def set_machine(self, machine_id, machine):
+ self._machine_id = machine_id
+ self._machine = machine
+
+ def parse(self, node):
+ scheme = {"info": self._info,
+ "netdevice": self._phys_netdevice}
+ self._process_child_nodes(node, scheme)
+
+ def _info(self, node, params):
+ machine = self._machine
+ info = machine["info"]
+
+ info["hostname"] = self._get_attribute(node, "hostname")
+
+ if self._has_attribute(node, "rootpass"):
+ info["rootpass"] = self._get_attribute(node, "rootpass")
+
+ if self._has_attribute(node, "rpcport"):
+ info["rpcport"] = self._get_attribute(node, "rpcport", int)
+
+ info["system_config"] = {}
+
+ self._trigger_event("machine_info_ready",
+ {"machine_id": self._machine_id})
+
+ def _phys_netdevice(self, node, params):
+ machine = self._machine
+ phys_id = self._get_attribute(node, "phys_id", int)
+
+ dev = machine["netdevices"][phys_id] = {}
+ dev["type"] = self._get_attribute(node, "type")
+ dev["hwaddr"] = normalize_hwaddr(self._get_attribute(node, "hwaddr"))
+
+ if self._has_attribute(node, "name"):
+ dev["name"] = self._get_attribute(node, "name")
+
+ self._trigger_event("netdevice_ready", {"machine_id": self._machine_id,
+ "dev_id": phys_id})
+
+
+class NetConfigParse(RecipeParser):
+ _machine_id = None
+ _machine = None
+
+ def set_machine(self, machine_id, machine):
+ self._machine_id = machine_id
+ self._machine = machine
+
+ def parse(self, node):
+ netconfig = self._machine["netconfig"]
+ self._netconfig = netconfig
+
+ devices = self._machine["netdevices"]
+ self._devices = devices
+
+ scheme = {"netdevice": self._netdevice}
+ self._process_child_nodes(node, scheme)
+
+ def _netdevice(self, node, params):
+ netconfig = self._netconfig
+ devices = self._devices
+
+ dev_id = self._get_attribute(node, "id", int)
+ if not dev_id in netconfig:
+ netconfig[dev_id] = {}
else:
- machine_id = 0 # controller id
- cmd_type = str(dom_command.getAttribute("type"))
- value = str(dom_command.getAttribute("value"))
-
- command = {"type": cmd_type, "value": value, "machine_id": machine_id}
- tmp = dom_command.getAttribute("timeout")
- if tmp:
- command["timeout"] = int(tmp)
- tmp = dom_command.getAttribute("bg_id")
- if tmp:
- command["bg_id"] = int(tmp)
- tmp = dom_command.getAttribute("desc")
- if tmp:
- command["desc"] = str(tmp)
- logging.debug("Parsed command: [%s]" % str_command(command))
-
- if cmd_type == "system_config":
- tmp = dom_command.getAttribute("option")
- if tmp:
- command["option"] = str(tmp)
-
- tmp = dom_command.getAttribute("persistent")
- if tmp:
- command["persistent"] = self._bool_it(tmp)
+ msg = "Netdevice 'id' used more than once"
+ raise XmlProcessingError(msg, node)
+
+ dev = netconfig[dev_id]
+ dev["type"] = self._get_attribute(node, "type")
+
+ if self._has_attribute(node, "phys_id"):
+ self._process_phys_id_attr(node, dev)
+
+ params = {"dev_id": dev_id}
+ scheme = {"addresses": self._addresses}
+ if dev["type"] == "eth":
+ pass
+ elif dev["type"] in ["bond", "bridge", "vlan", "macvlan", "team"]:
+ scheme["options"] = self._options
+ scheme["slaves"] = self._slaves
+ else:
+ logging.warn("unknown type \"%s\"" % dev["type"])
+
+ self._process_child_nodes(node, scheme, params)
+
+ self._trigger_event("interface_config_ready",
+ {"machine_id": self._machine_id,
+ "netdev_config_id": dev_id})
+
+ def _process_phys_id_attr(self, node, dev):
+ netconfig = self._netconfig
+ devices = self._devices
+
+ if dev["type"] == "eth":
+ phys_id = self._get_attribute(node, "phys_id", int)
+ self._check_phys_id(node, phys_id, netconfig)
+ dev["phys_id"] = phys_id
+
+ if phys_id in devices:
+ phys_dev = devices[phys_id]
+ if phys_dev["type"] == dev["type"]:
+ dev["hwaddr"] = phys_dev["hwaddr"]
+ if "name" in phys_dev:
+ dev["name"] = phys_dev["name"]
else:
- command["persistent"] = False
+ msg = "phys_id passed but does not match any device on machine"
+ raise XmlProcessingError(msg, node)
+ else:
+ logging.warn("phys_id found on non-eth netdev, ignoring")
+
+
+ @staticmethod
+ def _check_phys_id(node, dev_pid, config):
+ for key in config:
+ if not "phys_id" in config[key]:
+ continue
+ if config[key]["phys_id"] == dev_pid:
+ msg = "same phys_id \"%d\" used more than once" % dev_pid
+ raise XmlProcessingError(msg, node)
+
+ def _addresses(self, node, params):
+ self._list_init(node, params, "addresses", {"address": self._address})
+
+ def _address(self, node, params):
+ if self._has_attribute(node, "value"):
+ addr = self._get_attribute(node, "value")
+ else:
+ addr = self._get_text_content(node)
+
+ dev_id = params["dev_id"]
+ self._netconfig[dev_id]["addresses"].append(addr)
+
+ def _options(self, node, params):
+ self._list_init(node, params, "options", {"option": self._option})
- dom_options_grp = dom_command.getElementsByTagName("options")
- options = {}
- for dom_options_item in dom_options_grp:
- dom_options = dom_options_item.getElementsByTagName("option")
- for dom_option in dom_options:
- self._parse_command_option(dom_option, options)
- if options:
- command["options"] = options
- return command
+ def _option(self, node, params):
+ name = self._get_attribute(node, "name")
+
+ if self._has_attribute(node, "value"):
+ value = self._get_attribute(node, "value")
+ else:
+ value = self._get_text_content(node)
+
+ dev_id = params["dev_id"]
+ self._netconfig[dev_id]["options"].append((name, value))
+
+ def _slaves(self, node, params):
+ self._list_init(node, params, "slaves", {"slave": self._slave})
+
+ def _slave(self, node, params):
+ if self._has_attribute(node, "id"):
+ slave_id = self._get_attribute(node, "id", int)
+ else:
+ slave_id = self._get_text_content(node, int)
+
+ dev_id = params["dev_id"]
+ self._netconfig[dev_id]["slaves"].append(slave_id)
+
+ def _list_init(self, node, params, node_name, scheme):
+ dev_id = params["dev_id"]
+ dev = self._netconfig[dev_id]
+ dev[node_name] = []
+
+ self._process_child_nodes(node, scheme, params)
+
+
+class CommandSequenceParse(RecipeParser):
+ def parse(self, node):
+ sequences = self._recipe["sequences"]
+ sequences.append([])
+ seq_num = len(sequences) - 1
+
+ self._seq_num = seq_num
+ self._seq_node = node
+
+ scheme = {"command": self._command}
+ self._process_child_nodes(node, scheme)
+
+ self._check_sequence(sequences[seq_num])
+
+ def _command(self, node, params):
+ subparser = CommandParse(self)
+ subparser.set_seq_num(self._seq_num)
+ subparser.parse(node)
def _check_sequence(self, sequence):
err = False
@@ -245,7 +317,7 @@ class NetTestParse:
else:
logging.error("Found command \"%s\" for bg_id \"%s\" on "
"machine \"%d\" which was not previously "
- "defined" % (cmd_type, bg_id, machine_id))
+ "defined", cmd_type, bg_id, machine_id)
err = True
if "bg_id" in command:
@@ -254,29 +326,117 @@ class NetTestParse:
bg_ids[machine_id].add(bg_id)
else:
logging.error("Command \"%d\" uses bg_id \"%d\" on machine "
- "\"%d\" which is already used"
- % (i, bg_id, machine_id))
+ "\"%d\" which is already used",
+ i, bg_id, machine_id)
err = True
for machine_id in bg_ids:
for bg_id in bg_ids[machine_id]:
logging.error("bg_id \"%d\" on machine \"%d\" has no kill/wait "
- "command to it" % (bg_id, machine_id))
- err= True
+ "command to it", bg_id, machine_id)
+ err = True
if err:
- raise WrongCommandSequenceException
-
- def parse_recipe_command_sequence(self):
- dom_sequences = self._dom_nettestrecipe.getElementsByTagName("command_sequence")
- xml_prep = self._xml_prep
- xml_prep.expand_group(dom_sequences)
-
- self._recipe["sequences"] = []
- for dom_sequence in dom_sequences:
- sequence = []
- dom_commands = dom_sequence.getElementsByTagName("command")
- for dom_command in dom_commands:
- sequence.append(self._parse_command(dom_command))
-
- self._check_sequence(sequence)
- self._recipe["sequences"].append(sequence)
+ msg = "Incorrect command sequence"
+ raise XmlProcessingError(msg, self._seq_node)
+
+
+class CommandParse(RecipeParser):
+ _seq_num = None
+ _cmd_num = None
+
+ def set_seq_num(self, num):
+ self._seq_num = num
+
+ def parse(self, node):
+ recipe = self._recipe
+ command = {}
+ recipe["sequences"][self._seq_num].append(command)
+ self._cmd_num = len(recipe["sequences"][self._seq_num]) - 1
+
+ if self._has_attribute(node, "machine_id"):
+ machine_id = self._get_attribute(node, "machine_id", int)
+ if machine_id and not machine_id in recipe["machines"]:
+ raise XmlProcessingError("Invalid machine_id", node)
+ else:
+ machine_id = 0 # controller id
+
+ command["machine_id"] = machine_id
+ command["type"] = self._get_attribute(node, "type")
+ command["value"] = self._get_attribute(node, "value")
+
+ if self._has_attribute(node, "timeout"):
+ command["timeout"] = self._get_attribute(node, "timeout", int)
+
+ if self._has_attribute(node, "bg_id"):
+ command["bg_id"] = self._get_attribute(node, "bg_id", int)
+
+ if self._has_attribute(node, "desc"):
+ command["desc"] = self._get_attribute(node, "desc")
+
+ if command["type"] == "system_config":
+ if self._has_attribute(node, "option"):
+ command["option"] = self._get_attribute(node, "option")
+
+ if self._has_attribute(node, "persistent"):
+ command["persistent"] = self._get_attribute(node, "persistent",
+ self._bool_it)
+ else:
+ command["persistent"] = False
+
+ scheme = {"options": self._options}
+ self._process_child_nodes(node, scheme)
+
+ def _options(self, node, params):
+ seq = self._seq_num
+ cmd = self._cmd_num
+ self._recipe["sequences"][seq][cmd]["options"] = {}
+
+ scheme = {"option": self._option}
+ self._process_child_nodes(node, scheme)
+
+ def _option(self, node, params):
+ seq = self._seq_num
+ cmd = self._cmd_num
+ options = self._recipe["sequences"][seq][cmd]["options"]
+
+ name = self._get_attribute(node, "name")
+ if not name in options:
+ options[name] = []
+
+ option = {}
+ options[name].append(option)
+
+ if self._has_attribute(node, "type"):
+ opt_type = self._get_attribute(node, "type")
+ option["type"] = opt_type
+ else:
+ opt_type = "default"
+
+ if opt_type == "default":
+ if self._has_attribute(node, "value"):
+ value = self._get_attribute(node, "value")
+ else:
+ value = self._get_text_content(node)
+
+ option["value"] = value
+ else:
+ msg = "Unknown option type \"%s\"" % opt_type
+ raise XmlProcessingError(msg, node)
+
+
+ @classmethod
+ def _int_it(cls, val):
+ try:
+ num = int(val)
+ except ValueError:
+ num = 0
+ return num
+
+ @classmethod
+ def _bool_it(cls, val):
+ if isinstance(val, str):
+ if re.match("^\s*(?i)(true)", val):
+ return True
+ elif re.match("^\s*(?i)(false)", val):
+ return False
+ return True if cls._int_it(val) else False
11 years, 9 months
- 1
- 0
[lnst] netconfig.py: Adapting to the new parser
by Jiří Pírko
commit 9f56b4c419439180bf328cd98c6e9486945ffbd3
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Wed Jul 18 17:03:36 2012 +0200
netconfig.py: Adapting to the new parser
The previous parser implementation was replaced by different one, which
broke this configuration tool. This commit adapts it to the new parser.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
netconfig.py | 107 +++++++++++++++++++++++++++++++++++++++++++++++----------
1 files changed, 88 insertions(+), 19 deletions(-)
---
diff --git a/netconfig.py b/netconfig.py
index 4e019db..9551ac6 100755
--- a/netconfig.py
+++ b/netconfig.py
@@ -18,8 +18,11 @@ import re
import os
from pprint import pprint
from NetConfig.NetConfig import NetConfig
-from NetConfig.NetConfigParse import NetConfigParse
from NetConfig.NetConfigDevice import NetConfigDeviceAllCleanup
+from NetConfig.NetConfigDevNames import NetConfigDevNames
+from NetTest.NetTestParse import NetConfigParse
+from NetTest.NetTestParse import NetMachineConfigParse
+from Common.XmlProcessing import XmlDomTreeInit
from Common.Logs import Logs
def usage():
@@ -36,6 +39,76 @@ def usage():
print " -m, --machine-config=FILE use this machine configuration file"
sys.exit()
+def prepare_machine_config(machine_file):
+ tree_init = XmlDomTreeInit()
+ dom = tree_init.parse_file(machine_file)
+ machine_dom = dom.getElementsByTagName("netmachineconfig")[0]
+
+ data = {"info":{}, "netdevices": {}, "netconfig": {}}
+
+ machine_parse = NetMachineConfigParse()
+ machine_parse.disable_events()
+ machine_parse.set_recipe(data)
+ machine_parse.set_machine(0, data)
+ machine_parse.parse(machine_dom)
+
+ return data
+
+def prepare_netconfig(machine_file, config_file):
+ tree_init = XmlDomTreeInit()
+ data = prepare_machine_config(machine_file)
+
+ dom = tree_init.parse_file(config_file)
+ config_dom = dom.getElementsByTagName("netconfig")[0]
+
+ config_parse = NetConfigParse()
+ config_parse.disable_events()
+ config_parse.set_recipe(data)
+ config_parse.set_machine(0, data)
+ config_parse.parse(config_dom)
+
+ netconfig = NetConfig()
+ for key, entry in data["netconfig"].iteritems():
+ netconfig.add_interface_config(key, entry)
+
+ return netconfig
+
+def netmachineconfig_to_xml(machine_data):
+ info = machine_data["info"]
+
+ hostname = ""
+ rootpass = ""
+ rpcport = ""
+
+ if "hostname" in info:
+ hostname = "hostname=\"%s\" " % info["hostname"]
+ if "rootpass" in info:
+ rootpass = "rootpass=\"%s\" " % info["rootpass"]
+ if "rpcport" in info:
+ rpcport = "rpcport=\"%s\" " % info["rpcport"]
+
+ info_tag = " <info %s%s%s/>\n" % (hostname, rootpass, rpcport)
+
+ devices = ""
+ for phys_id, netdev in machine_data["netdevices"].iteritems():
+ pid = "phys_id=\"%d\" " % phys_id
+ dev_type = ""
+ name = ""
+ hwaddr = ""
+
+ if "type" in netdev:
+ dev_type = "type=\"%s\" " % netdev["type"]
+ if "name" in netdev:
+ name = "name=\"%s\" " % netdev["name"]
+ if "hwaddr" in netdev:
+ hwaddr = "hwaddr=\"%s\" " % netdev["hwaddr"]
+
+ device_tag = " <netdevice %s%s%s%s/>\n" % (pid, dev_type,
+ name, hwaddr)
+ devices += device_tag
+
+ return "<netmachineconfig>\n" + info_tag + devices + "</netmachineconfig>"
+
def main():
"""
Main function
@@ -82,14 +155,16 @@ def main():
usage();
machine_config_path = os.path.expanduser(machine_config_path)
- handle = open(machine_config_path, "r")
- machine_config_xml = handle.read()
- handle.close()
-
if action == "refresh":
logging.info("Refreshing machine config")
- net_config_parse = NetConfigParse(machine_config_xml)
- output = net_config_parse.refresh_machine_config()
+ machine_data = prepare_machine_config(machine_config_path)
+ dev_names = NetConfigDevNames()
+ for dev_id, netdev in machine_data["netdevices"].iteritems():
+ if "name" in netdev:
+ del netdev["name"]
+ dev_names.assign_name_by_scan(dev_id, netdev)
+
+ output = netmachineconfig_to_xml(machine_data)
handle = open(machine_config_path, "w")
handle.write(output)
handle.close()
@@ -106,24 +181,18 @@ def main():
'''
for root, dirs, files in os.walk(config_path):
- for f in files:
- config_file = os.path.join(config_path, f)
+ for file_name in files:
+ config_file = os.path.join(config_path, file_name)
if not re.match(r'^.*\.xml$', config_file):
continue
- handle = open(config_file, "r")
- config_xml = handle.read()
- handle.close()
- logging.info("Processing config file \"%s\"" % config_file)
- net_config = NetConfig(machine_config_xml, config_xml)
+ logging.info("Processing config file \"%s\"", config_file)
+ net_config = prepare_netconfig(machine_config_path,
+ config_file)
net_config.configure_all()
net_config.deconfigure_all()
return
- handle = open(config_path, "r")
- config_xml = handle.read()
- handle.close()
-
- net_config = NetConfig(machine_config_xml, config_xml)
+ net_config = prepare_netconfig(machine_config_path, config_path)
if action == "up":
net_config.configure_all()
elif action == "down":
11 years, 9 months
- 1
- 0
[lnst] nettestctl.py: all_dump and eval actions removed
by Jiří Pírko
commit cf6cfb0415e0e9112daa219bed468cdaed2a3219
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Wed Jul 18 17:03:35 2012 +0200
nettestctl.py: all_dump and eval actions removed
'all_dump' now makes no sense. Configuration of slave machines now
occurs at recipe parsing time (because it's needed to be done for
further processing). That means, that you cannot safely parse the
recipe config without connecting to the slaves.
'eval' option is tied to the recipe_eval functionality that was
removed and replaced by aliases and templates.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
nettestctl.py | 9 +--------
1 files changed, 1 insertions(+), 8 deletions(-)
---
diff --git a/nettestctl.py b/nettestctl.py
index f1511df..617202a 100755
--- a/nettestctl.py
+++ b/nettestctl.py
@@ -29,7 +29,7 @@ def usage():
print "Usage:"
print "nettestctl.py [OPTION...] ACTION"
print ""
- print "ACTION = [run | dump | all_dump | config_only | eval EXPR]"
+ print "ACTION = [run | dump | config_only]"
print ""
print " -d, --debug emit debugging messages"
print " -p, --packet_capture capture and log all ongoing\n" \
@@ -54,15 +54,8 @@ def process_recipe(args, file_path, remoteexec, cleanup,
return nettestctl.run_recipe(packet_capture)
elif action == "dump":
return nettestctl.dump_recipe()
- elif action == "all_dump":
- return nettestctl.all_dump_recipe()
elif action == "config_only":
return nettestctl.config_only_recipe()
- elif action == "eval":
- if len(args) < 2:
- logging.error("No expression passed")
- usage();
- return nettestctl.eval_expression_recipe(args[1])
else:
logging.error("Unknown action \"%s\"" % action)
usage();
11 years, 9 months
- 1
- 0
[lnst] Common: Adding XmlProcessing module
by Jiří Pírko
commit 309fbfb8599fdc9a909e95486e7517245293b571
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Wed Jul 18 17:03:34 2012 +0200
Common: Adding XmlProcessing module
This commit introduces new module to the Common package called
XmlProcessing. It's a set of classes to aid XML parsing in LNST.
That includes:
class XmlDomTreeInit: Initialization of DOM tree using sax
parser
class XmlParse: Base class for positional XML parsers
(it parses XML's sequentially,
element-by-element, as they are in the
original XML
class RecipeParse: Extented version of XmlParse prepared
specialy for LNST recipes
exception XmlProcessingError: To indicate error during the parsing
process
This module uses XmlTemplates class to process templates.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Common/XmlProcessing.py | 351 +++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 351 insertions(+), 0 deletions(-)
---
diff --git a/Common/XmlProcessing.py b/Common/XmlProcessing.py
new file mode 100644
index 0000000..a76364e
--- /dev/null
+++ b/Common/XmlProcessing.py
@@ -0,0 +1,351 @@
+"""
+This module contains code code for XML parsing and processing.
+
+Copyright 2012 Red Hat, Inc.
+Licensed under the GNU General Public License, version 2 as
+published by the Free Software Foundation; see COPYING for details.
+"""
+
+__author__ = """
+rpazdera(a)redhat.com (Radek Pazdera)
+"""
+
+import os
+import logging
+from xml.dom.minidom import parseString
+from xml import sax
+from Common.XmlTemplates import XmlTemplates, XmlTemplateError
+
+
+class XmlProcessingError(Exception):
+ """ Exception thrown on parsing errors """
+
+ _filename = None
+ _line = None
+ _col = None
+
+ def __init__(self, msg, node=None):
+ super(XmlProcessingError, self).__init__()
+
+ self._msg = msg
+
+ if node and hasattr(node, "parse_position"):
+ self.set_pos(node.parse_position)
+
+ #logging.error(self.__str__())
+
+ def set_pos(self, pos):
+ self._filename = pos["file"]
+ self._line = pos["line"]
+ #self._col = pos["col"]
+
+ def __str__(self):
+ line = ""
+ col = ""
+ sep = ""
+ pos = ""
+ filename = "<unknown>"
+
+ if self._filename:
+ filename = self._filename
+
+ if self._line:
+ line = "%d" % self._line
+ sep = ":"
+
+ if self._col:
+ col = "%s%d" % (sep, self._col)
+
+ if self._line or self._col:
+ pos = "%s%s:" % (line, col)
+
+ return "XmlProcessingError:%s:%s %s" % (filename, pos, self._msg)
+
+class XmlDomTreeInit:
+ """ Handles creation/initialization of DOM trees
+
+ It allows you to parse XML file or string into a DOM tree.
+ It also adds an extra parameter to each node of the tree
+ called `parse_position' which can be used to determine
+ where exactly was the element placed in the original source
+ XML file. This is useful for error reporting.
+ """
+
+ _sax = None
+ _filename = None
+ __orig_set_content_handler = None
+
+ def __init__(self):
+ self._init_sax()
+
+ def _init_sax(self):
+ parser = sax.make_parser()
+ self.__orig_set_content_handler = parser.setContentHandler
+ parser.setContentHandler = self.__set_content_handler
+ self._sax = parser
+
+ def __set_content_handler(self, dom_handler):
+ def start_element_ns(name, tag_name , attrs):
+ orig_start_cb(name, tag_name, attrs)
+ cur_elem = dom_handler.elementStack[-1]
+ pos = {"file": self._filename,
+ "line": self._sax.getLineNumber(),
+ "col": self._sax.getColumnNumber()}
+ cur_elem.parse_position = pos
+
+ orig_start_cb = dom_handler.startElementNS
+ dom_handler.startElementNS = start_element_ns
+ self.__orig_set_content_handler(dom_handler)
+
+ @staticmethod
+ def _load_file(filename):
+ handle = open(filename, "r")
+ data = handle.read()
+ handle.close()
+ return data
+
+ def parse_file(self, xml_filepath):
+ xml_text = self._load_file(xml_filepath)
+ filename = os.path.basename(xml_filepath)
+ return self.parse_string(xml_text, filename)
+
+ def parse_string(self, xml_text, filename="xml_string"):
+ self._filename = filename
+ try:
+ dom = parseString(xml_text, self._sax)
+ except sax.SAXParseException, err:
+ pos = {"file": filename,
+ "line": err.getLineNumber(),
+ "col": err.getColumnNumber()}
+ exc = XmlProcessingError(err.getMessage())
+ exc.set_pos(pos)
+ raise exc
+
+ return dom
+
+
+class XmlParser(object):
+ """ Parent class for XML processors
+
+ This class handles manipulation of XML DOM objects
+ that are used for processing XML files.
+
+ The standard DOM objects are extended with position data
+ (file name, line number and column number) that can be
+ used in error reporting.
+ """
+
+ def _process_child_nodes(self, parent, scheme, params=None):
+ child_nodes = parent.childNodes
+
+ if not params:
+ params = {}
+
+ for node in child_nodes:
+ if node.nodeType == node.COMMENT_NODE or \
+ node.nodeType == node.TEXT_NODE:
+ continue
+ elif node.nodeType == node.ELEMENT_NODE:
+ node_name = node.nodeName
+ if node_name in scheme:
+ handler = scheme[node_name]
+ self._process_node(node, handler, params)
+ else:
+ msg = "Unexpected '%s' tag under '%s'" % (node_name,
+ parent.nodeName)
+ raise XmlProcessingError(msg, node)
+ else:
+ msg = "Only XML elements are allowed here!"
+ raise XmlProcessingError(msg, node)
+
+ def _process_node(self, node, handler, params):
+ handler(node, params)
+
+ @staticmethod
+ def _convert_string(node, string, conversion_cb):
+ if conversion_cb:
+ try:
+ converted = conversion_cb(string)
+ except ValueError, err:
+ raise XmlProcessingError("Conversion error: " + str(err), node)
+ return converted
+
+ return string
+
+ def _has_attribute(self, node, attr_name):
+ return node.hasAttribute(attr_name)
+
+ def _get_attribute(self, node, attr_name, conversion_cb=None):
+ if not self._has_attribute(node, attr_name):
+ msg = "Expected attribute '%s' missing" % attr_name
+ raise XmlProcessingError(msg, node)
+ attr_val = str(node.getAttribute(attr_name))
+ return self._convert_string(node, attr_val, conversion_cb)
+
+ def _get_text_content(self, node, conversion_cb=None):
+ content = []
+ for child in node.childNodes:
+ if child.nodeType == child.TEXT_NODE:
+ content.append(child.nodeValue)
+
+ text = str(''.join(content).strip())
+ return self._convert_string(node, text, conversion_cb)
+
+
+class RecipeParser(XmlParser):
+ """ Enhanced XmlParser
+
+ This class enhances XmlParser with advanced features that are
+ used in parsing XML recipe files. All recipe (sub)parsers should
+ use this as their base class.
+ """
+
+ _recipe = None
+ _template_proc = None
+ _include_root = None
+ _events_enabled = None
+ _event_handlers = None
+
+ def __init__(self, parent=None):
+ super(RecipeParser, self).__init__()
+
+ if parent:
+ self._recipe = parent._recipe
+ self._template_proc = parent._template_proc
+ self._include_root = parent._include_root
+ self._events_enabled = parent._events_enabled
+ self._event_handlers = parent._event_handlers
+ else:
+ self._recipe = {}
+ self._template_proc = XmlTemplates()
+ self._include_root = os.getcwd()
+ self._events_enabled = True
+ self._event_handlers = {}
+
+ def set_recipe(self, recipe):
+ self._recipe = recipe
+
+ def set_definitions(self, defs):
+ self._template_proc.set_definitions(defs)
+
+ def set_include_root(self, include_root_path):
+ self._include_root = include_root_path
+
+ def enable_events(self):
+ self._events_enabled = True
+
+ def disable_events(self):
+ self._events_enabled = False
+
+ def register_event_handler(self, event_id, handler):
+ self._event_handlers[event_id] = handler
+
+ def _trigger_event(self, event_id, args):
+ if not self._events_enabled:
+ return
+
+ try:
+ handler = self._event_handlers[event_id]
+ except KeyError, err:
+ logging.warn("No handler found for %s event, ignoring", event_id)
+ return
+
+ handler(**args)
+
+ def _process_child_nodes(self, node, scheme, params=None,
+ new_ns_level=True):
+ scheme["define"] = self._define_handler
+
+ if not params:
+ params = {}
+
+ if new_ns_level:
+ self._template_proc.add_namespace_level()
+
+ parent = super(RecipeParser, self)
+ result = parent._process_child_nodes(node, scheme, params)
+
+ if new_ns_level:
+ self._template_proc.drop_namespace_level()
+
+ return result
+
+ def _process_node(self, node, handler, params):
+ old_include_root = None
+ if self._has_attribute(node, "source"):
+ source = self._get_attribute(node, "source")
+ file_path = self._get_referenced_xml_path(source)
+
+ old_include_root = self._include_root
+ self._include_root = os.path.dirname(file_path)
+
+ dom_init = XmlDomTreeInit()
+ try:
+ dom = dom_init.parse_file(file_path)
+ except IOError, err:
+ msg = "Unable to resolve include: %s" % str(err)
+ raise XmlProcessingError(msg, node)
+
+ loaded_node = None
+ try:
+ loaded_node = dom.getElementsByTagName(node.nodeName)[0]
+ except Exception:
+ msg = ("No '%s' element present in included file '%s'."
+ % (node.nodeName, file_path))
+ raise XmlProcessingError(msg, node)
+
+ parent = node.parentNode
+ parent.replaceChild(loaded_node, node)
+ node = loaded_node
+
+ parent = super(RecipeParser, self)
+ parent._process_node(node, handler, params)
+
+ if old_include_root:
+ self._include_root = old_include_root
+
+ def _get_attribute(self, node, attr_name, conversion_cb=None):
+ parent = super(RecipeParser, self)
+ raw_attr_val = parent._get_attribute(node, attr_name)
+
+ try:
+ attr_val = self._template_proc.expand_string(raw_attr_val)
+ except XmlTemplateError, err:
+ raise XmlProcessingError(str(err), node)
+
+ return self._convert_string(node, attr_val, conversion_cb)
+
+ def _get_text_content(self, node, conversion_cb=None):
+ parent = super(RecipeParser, self)
+ raw_content = parent._get_text_content(node)
+
+ try:
+ content = self._template_proc.expand_string(raw_content)
+ except XmlTemplateError, err:
+ raise XmlProcessingError(str(err), node)
+
+ return self._convert_string(node, content, conversion_cb)
+
+ def _define_handler(self, node, params):
+ scheme = {"alias": self._alias_handler}
+ self._process_child_nodes(node, scheme, new_ns_level=False)
+
+ def _alias_handler(self, node, params):
+ if self._has_attribute(node, "name"):
+ name = self._get_attribute(node, "name")
+ else:
+ msg = "Alias tag must have the 'name' attribute"
+ raise XmlProcessingError(msg, node)
+
+ if self._has_attribute(node, "value"):
+ value = self._get_attribute(node, "value")
+ else:
+ value = self._get_text_content(node)
+
+ try:
+ self._template_proc.define_alias(name, value)
+ except XmlTemplateError, err:
+ raise XmlProcessingError(str(err), node)
+
+ def _get_referenced_xml_path(self, filename):
+ return os.path.join(self._include_root, os.path.expanduser(filename))
11 years, 9 months
- 1
- 0
[lnst] XmlPreprocessor: Changing to XmlTemplates
by Jiří Pírko
commit bb82695b5f069d0fdcb16118fb3287f78ca57112
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Wed Jul 18 17:03:33 2012 +0200
XmlPreprocessor: Changing to XmlTemplates
XmlPreprocessor was reworked into a class called XmlTemplates. This
class is responsible for processing XML templates during recipe
parsing.
Various features were removed (especially the preprocessing code and
several new features were added.
XmlTemplates supports hierarchy of namespaces for aliases. In other
words, alias definitions now have only local effect within the tag
of their definition. They will not be accessible from anywhere else.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Common/{XmlPreprocessor.py => XmlTemplates.py} | 135 ++++++++++++++++--------
1 files changed, 89 insertions(+), 46 deletions(-)
---
diff --git a/Common/XmlPreprocessor.py b/Common/XmlTemplates.py
similarity index 62%
rename from Common/XmlPreprocessor.py
rename to Common/XmlTemplates.py
index be2cab2..92ff269 100644
--- a/Common/XmlPreprocessor.py
+++ b/Common/XmlTemplates.py
@@ -1,6 +1,6 @@
"""
-This module contains code for preprocessing templates in XML files/recipes
-before they are parsed.
+This module contains code to aid processing templates in XML files/recipes
+while they're being parsed.
Templates are strings enclosed in curly braces {} and can be present
in all text elements of the XML file (this includes tag values or
@@ -26,46 +26,84 @@ import re
class XmlTemplateError(Exception):
pass
-class XmlPreprocessor:
- """
- This class serves as template processor within a XML DOM tree object.
- """
+class XmlTemplates:
+ """ This class serves as template processor """
_alias_re = "\{\$([a-zA-Z0-9_]+)(\[.+\])*\}"
_func_re = "\{([a-zA-Z0-9_]+)\(([^\(\)]*)\)\}"
- def __init__(self):
- self._definitions = {}
+ def __init__(self, definitions=None):
+ if definitions:
+ self._definitions = [definitions]
+ else:
+ self._definitions = [{}]
+
self._reserved_aliases = ["recipe"]
- def define_alias(self, name, value, skip_reserved_check=False):
+ def set_definitions(self, defs):
+ """ Set alias definitions
+
+ All existing definitions and namespace levels are
+ destroyed and replaced with new definitions.
+ """
+ del self._definitions
+ self._definitions = [defs]
+
+ def get_definitions(self):
+ """ Return definitions dict
+
+ Definitions are returned as a single dictionary of
+ all currently defined aliases, regardless the internal
+ division to namespace levels.
"""
- Associate an alias name with some value. The value can be of
- an atomic type or an array.
+ defs = {}
+ for level in self._definitions:
+ for name, val in level.iteritems():
+ defs[name] = val
+
+ return defs
+
+ def define_alias(self, name, value, skip_reserved_check=False):
+ """ Associate an alias name with some value
+
+ The value can be of an atomic type or an array. The
+ definition is added to the current namespace level.
"""
if not name in self._reserved_aliases \
or skip_reserved_check == True:
- self._definitions[name] = value
+ self._definitions[-1][name] = value
else:
raise XmlTemplateError("Alias name '%s' is reserved" % name)
- def remove_comments(self, node):
+ def add_namespace_level(self):
+ """ Create new namespace level
+
+ This method will create a new level for definitions on
+ the stack. All aliases, that will be defined after this
+ call will be dropped as soon as `drop_namespace_level'
+ is called.
"""
- Remove all comment nodes from the tree.
+ self._definitions.append({})
+
+ def drop_namespace_level(self):
+ """ Remove one namespace level
+
+ This method will erease all defined aliases since the
+ last call of `add_namespace_level' method. All aliases,
+ that were defined beforehand will be kept.
"""
+ self._definitions.pop()
- comments = []
- for child in node.childNodes:
- if child.nodeType == node.COMMENT_NODE:
- comments.append(child)
- else:
- self.remove_comments(child)
+ def _find_definition(self, name):
+ for level in reversed(self._definitions):
+ if name in level:
+ return level[name]
- for comment in comments:
- node.removeChild(comment)
+ err = "'%s' is not defined here" % name
+ raise XmlTemplateError(err)
- def expand(self, node):
+ def expand_dom(self, node):
"""
Traverse DOM tree from `node' down and expand any
templates along the way.
@@ -76,13 +114,13 @@ class XmlPreprocessor:
num_attributes = node.attributes.length
while(i < num_attributes):
attr = node.attributes.item(i)
- attr.value = self._expand_string(str(attr.value))
+ attr.value = self.expand_string(str(attr.value))
i += 1
elif node.nodeType == node.TEXT_NODE:
- node.data = self._expand_string(str(node.data))
+ node.data = self.expand_string(str(node.data))
for child in node.childNodes:
- self.expand(child)
+ self.expand_dom(child)
def expand_group(self, group):
"""
@@ -92,9 +130,14 @@ class XmlPreprocessor:
"""
for node in group:
- self.expand(node)
+ self.expand_dom(node)
+
+ def expand_string(self, string):
+ """ Expand templates in a string
- def _expand_string(self, string):
+ This method will process and expand all templates
+ contained inside a string.
+ """
while True:
alias_match = re.search(self._alias_re, string)
func_match = re.search(self._func_re, string)
@@ -122,20 +165,17 @@ class XmlPreprocessor:
alias_name = alias_match.group(1)
array_subscript = alias_match.group(2)
- if not alias_name in self._definitions:
- err = "Alias '%s' is not defined here" % alias_name
- raise XmlTemplateError(err)
+ alias_obj = self._find_definition(alias_name)
if array_subscript != None:
try:
- result = str(eval("self._definitions['%s']%s" \
- % (alias_name, array_subscript)))
+ result = str(eval("alias_obj%s" % array_subscript))
except (KeyError, IndexError):
- err = "Wrong array subscript in '%s[%s]'" \
+ err = "Wrong array subscript in '%s%s'" \
% (alias_name, array_subscript)
raise XmlTemplateError(err)
else:
- result = self._definitions[alias_name]
+ result = alias_obj
return result
@@ -177,34 +217,34 @@ class XmlPreprocessor:
def _ip_func(self, params):
self._validate_func_params("ip", params, 2, 1)
- self._check_recipe_data("ip")
+ recipe = self._get_recipe_data("ip")
m_id = int(params[0])
if_id = int(params[1])
ip_id = int(params[2]) if len(params) == 3 else 0
- machines = self._definitions["recipe"]["machines"]
+ machines = recipe["machines"]
ip_addr = machines[m_id]['netconfig'][if_id]['addresses'][ip_id]
return ip_addr.split('/')[0]
def _hwaddr_func(self, params):
self._validate_func_params("hwaddr", params, 2, 0)
- self._check_recipe_data("hwaddr")
+ recipe = self._get_recipe_data("hwaddr")
m_id = int(params[0])
if_id = int(params[1])
- machines = self._definitions["recipe"]["machines"]
+ machines = recipe["machines"]
return machines[m_id]['netconfig'][if_id]['hwaddr']
def _devname_func(self, params):
self._validate_func_params("devname", params, 2, 0)
- self._check_recipe_data("devname")
+ recipe = self._get_recipe_data("devname")
m_id = int(params[0])
if_id = int(params[1])
- machines = self._definitions["recipe"]["machines"]
+ machines = recipe["machines"]
return machines[m_id]['netconfig'][if_id]['name']
@staticmethod
@@ -225,8 +265,11 @@ class XmlPreprocessor:
err = "Non-integer parameter passed to '%s'" % name
raise XmlTemplateError(err)
- def _check_recipe_data(self, template_name):
- if not "recipe" in self._definitions:
- err = "Cannot resolve %s() here, recipe data not available yet" \
- % template_name
- raise XmlTemplateError(err)
+ def _get_recipe_data(self, template_name):
+ try:
+ recipe = self._find_definition("recipe")
+ return recipe
+ except XmlTemplateError, err:
+ msg = "Cannot resolve %s(): " % template_name
+ msg += str(err)
+ raise XmlTemplateError(msg)
11 years, 9 months
- 1
- 0
[lnst] XmlPreprocessor: Modify template detection
by Jiří Pírko
commit 2dfa60872151f2d245d589ef362885fcc77b9032
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Fri Jun 8 12:42:04 2012 +0200
XmlPreprocessor: Modify template detection
The previous solution considered curly braces to be reserved
characters, exclusively for marking templates. But according
to recent development, it will be necessary to deal with
those characters also in non-template context.
This commit enhances template detection code so it will ignore
any occurencies of curly braces unless they form a valid template.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Common/XmlPreprocessor.py | 37 ++++++++++++++-----------------------
1 files changed, 14 insertions(+), 23 deletions(-)
---
diff --git a/Common/XmlPreprocessor.py b/Common/XmlPreprocessor.py
index b20fec8..be2cab2 100644
--- a/Common/XmlPreprocessor.py
+++ b/Common/XmlPreprocessor.py
@@ -31,9 +31,8 @@ class XmlPreprocessor:
This class serves as template processor within a XML DOM tree object.
"""
- _template_re = "\{([^\{\}]+)\}"
- _alias_re = "^\$([a-zA-Z0-9_]+)(\[.+\])*$"
- _func_re = "^([a-zA-Z0-9_]+)\(([^\(\)]*)\)$"
+ _alias_re = "\{\$([a-zA-Z0-9_]+)(\[.+\])*\}"
+ _func_re = "\{([a-zA-Z0-9_]+)\(([^\(\)]*)\)\}"
def __init__(self):
self._definitions = {}
@@ -97,31 +96,23 @@ class XmlPreprocessor:
def _expand_string(self, string):
while True:
- template_match = re.search(self._template_re, string)
- if template_match:
- template_string = template_match.group(0)
- template = template_match.group(1)
- template_result = self._process_template(template)
+ alias_match = re.search(self._alias_re, string)
+ func_match = re.search(self._func_re, string)
- string = string.replace(template_string, template_result)
+ result = None
+
+ if alias_match:
+ template = alias_match.group(0)
+ result = self._process_alias_template(template)
+ elif func_match:
+ template = func_match.group(0)
+ result = self._process_func_template(template)
else:
break
- return string
-
- def _process_template(self, string):
- string = string.strip()
- result = None
+ string = string.replace(template, result)
- if re.match(self._alias_re, string):
- result = self._process_alias_template(string)
- return result
-
- if re.match(self._func_re, string):
- result = self._process_func_template(string)
- return result
-
- raise XmlTemplateError("Unknown template type '%s'" % string)
+ return string
def _process_alias_template(self, string):
result = None
11 years, 9 months
- 2
- 1
[lnst] Fix iproute vlan help output parsing
by Jiří Pírko
commit ef4c9dcdf655e931f172dfec7d7766c404b2a7c8
Author: Jan Tluka <jtluka(a)redhat.com>
Date: Tue Jul 17 14:54:25 2012 +0200
Fix iproute vlan help output parsing
Latest iproute tool package has changed the help output for link
management. This output is parsed by NetConfigDeviceVlan class to check
support for VLANs.
(iproute-2.6.32)
# ip link help
Usage: ip link add link DEV [ name ] NAME
...
(iproute-3.3.0)
# ip link help
Usage: ip link add [link DEV] [ name ] NAME
...
Without the fix the VLANs are unusable with the
latest iproute package (3.3.0). I have tested the patch with both older
and new package version: iproute-2.6.32 and iproute-3.3.0.
NetConfig/NetConfigDevice.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
---
diff --git a/NetConfig/NetConfigDevice.py b/NetConfig/NetConfigDevice.py
index 8e89c36..6a81495 100644
--- a/NetConfig/NetConfigDevice.py
+++ b/NetConfig/NetConfigDevice.py
@@ -181,7 +181,7 @@ class NetConfigDeviceVlan(NetConfigDeviceGeneric):
output = exec_cmd("ip link help", die_on_err=False,
log_outputs=False)[1]
for line in output.split("\n"):
- if re.match(r'^.*ip link add link.*$', line):
+ if re.match(r'^.*ip link add [\[]{0,1}link.*$', line):
return True
return False
11 years, 9 months
- 1
- 0
[PATCH] Fix iproute vlan help output parsing
by Jan Tluka
Latest iproute tool package has changed the help output for link
management. This output is parsed by NetConfigDeviceVlan class to check
support for VLANs.
(iproute-2.6.32)
# ip link help
Usage: ip link add link DEV [ name ] NAME
...
(iproute-3.3.0)
# ip link help
Usage: ip link add [link DEV] [ name ] NAME
...
Without the fix the VLANs are unusable with the
latest iproute package (3.3.0). I have tested the patch with both older
and new package version: iproute-2.6.32 and iproute-3.3.0.
---
NetConfig/NetConfigDevice.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/NetConfig/NetConfigDevice.py b/NetConfig/NetConfigDevice.py
index 8e89c36..6a81495 100644
--- a/NetConfig/NetConfigDevice.py
+++ b/NetConfig/NetConfigDevice.py
@@ -181,7 +181,7 @@ class NetConfigDeviceVlan(NetConfigDeviceGeneric):
output = exec_cmd("ip link help", die_on_err=False,
log_outputs=False)[1]
for line in output.split("\n"):
- if re.match(r'^.*ip link add link.*$', line):
+ if re.match(r'^.*ip link add [\[]{0,1}link.*$', line):
return True
return False
--
1.7.6.5
11 years, 9 months
- 1
- 0
[lnst] gitignore: Adding some convenient ignore patterns
by Jiří Pírko
commit 6121ce4b71455e6b257374946ff440368fdf9d5c
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Mon Jul 2 09:09:14 2012 +0200
gitignore: Adding some convenient ignore patterns
Adding *.bz2 Log/ *.swp ignore patterns
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
.gitignore | 9 +++++++++
1 files changed, 9 insertions(+), 0 deletions(-)
---
diff --git a/.gitignore b/.gitignore
index 85aee89..187b4b7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,8 +3,17 @@
.*~
*~
.#*
+
+Logs/
+
+# vim swap files
+*.swp
+*.swo
+*.swn
+
# additional patterns:
*.pyc
*.pyo
*.gz
+*.bz2
*.orig
11 years, 9 months
- 1
- 0