[PATCH v2 1/6] lnst.Recipes.ENRT.OvSDPDKBondRecipe: add new recipe

From: Jozef Urbanovsky <jurbanov(a)redhat.com&gt;

Signed-off-by: Jozef Urbanovsky <jurbanov(a)redhat.com&gt;
---
 lnst/Recipes/ENRT/OvSDPDKBondRecipe.py | 375 +++++++++++++++++++++++++
 lnst/Recipes/ENRT/__init__.py          |   1 +
 2 files changed, 376 insertions(+)
 create mode 100644 lnst/Recipes/ENRT/OvSDPDKBondRecipe.py

diff --git a/lnst/Recipes/ENRT/OvSDPDKBondRecipe.py
b/lnst/Recipes/ENRT/OvSDPDKBondRecipe.py
new file mode 100644
index 0000000..46ec4d4
--- /dev/null
+++ b/lnst/Recipes/ENRT/OvSDPDKBondRecipe.py
@@ -0,0 +1,375 @@
+import signal
+from lnst.Common.Parameters import StrParam, IntParam, Param
+from lnst.Controller import HostReq, DeviceReq, RecipeParam
+from lnst.Controller.Namespace import Namespace
+from lnst.Recipes.ENRT.BaseEnrtRecipe import BaseEnrtRecipe
+from dataclasses import dataclass
+from lnst.Common.IpAddress import ipaddress
+from lnst.Tests.TestPMD import TestPMD
+
+from lnst.RecipeCommon.Perf.Recipe import RecipeConf as PerfRecipeConf
+from lnst.RecipeCommon.Perf.Measurements import Flow as PerfFlow
+from lnst.RecipeCommon.Perf.Measurements import TRexFlowMeasurement
+from lnst.RecipeCommon.Perf.Measurements import StatCPUMeasurement
+
+
+@dataclass
+class DummyEthDevice:
+    """
+    Auxiliary class to store Ethernet related information,
+    that are no longer available for kernel, therefore LNST Device
+    class can no longer be used.
+    """
+    bus_info: str
+    hwaddr: str
+    _id: str
+    ips: list
+
+
+@dataclass
+class DummyRecipeConfig:
+    """
+    Auxiliary class to store network related information
+    required for the recipe configuration.
+    """
+    eth0: DummyEthDevice
+    eth1: DummyEthDevice
+    netns: Namespace
+    testpmd: None
+    test_br: str = "test_bridge"
+    bond_br: str = "bond_bridge"
+
+
+class OvSDPDKBondRecipe(BaseEnrtRecipe):
+    """
+    This recipe implements Enrt testing for OvS DPDK Bonding recipe
+    network scenario that looks as follows
+
+    .. code-block:: none
+
+                                         +--------+
+                     +-------------------+        +----------------------+
+                     |                   | switch |                      |
+                     |          +--------+        +------------+         |
+                     |          |        +--------+            |         |
+                     |          |                              |         |
+     +---------------|----------|----------------+             |         |
+     |        +------|----------|-------+        |       +-----|---------|-----+
+     |        | +----|-dpdkbond-|-----+ |        |       | +---+---+ +---+---+ |
+     |        | | +--+----+ +---+---+ | |        |       | | dpdk0 | | dpdk1 | |
+     |        | | | dpdk0 | | dpdk1 | | |        |       | +-------+ +-------+ |
+     |        | | +---+---+ +---+---+ | |        |       |        host2        |
+     |        | +-----|---------|-----+ |        |       +---------------------+
+     |        |       |         |       |        |
+     |        |     +-+---------+-+     |        |
+     |        |     | test_patch  |     |        |
+     |        |     +------|------+     |        |
+     |        +------------|------------+        |
+     |                     |                     |
+     | +-------------------|-------------------+ |
+     | |            +------+------+            | |
+     | |         +--| bond_patch  ---+         | |
+     | |         |  +-------------+  |         | |
+     | |         |                   |         | |
+     | | +-------+--------+ +--------+-------+ | |
+     | | | dpdkvhostuser0 | | dpdkvhostuser1 | | |
+     | | +-------+--------+ +--------+-------+ | |
+     | +---------|-------------------|---------+ |
+     |           |   +---------+     |           |
+     |           +---+ testPMD +-----+           |
+     |               +---------+                 |
+     |                  host1                    |
+     +-------------------------------------------+
+
+    The recipe provides additional recipe parameters to configure the bonding
+    device.
+
+        :param bonding_mode:
+            (mandatory test parameter) the bonding mode to be configured on
+            the dpdkbond device.
+        :param lacp_mode:
+            (mandatory test parameter) the lacp mode to be configured
+            on the dpdkbond device.
+        :param dpdk_lcore_mask:
+            (mandatory test parameter) cores to be used for the DPDK in OVS
+        :param pmd_cpu_mask:
+            (mandatory test parameter) cores to be used for the PMD in OVS
+        :param trex_dpdk_cores:
+            (mandatory test parameter) cores to be used by the Trex generator
+        :param trex_dor:
+            (mandatory test parameter) path to the Trex directory
+
+    No subconfiguration are applied through Mixin classes as they are not
+    applicable for this scenario. Subconfiguration for Open vSwitch is included
+    via :any:`ovs_dpdk_bridge_configuration` method.
+
+    The actual test machinery is implemented in the :any:`BaseEnrtRecipe` class.
+    """
+    host1 = HostReq()
+    host1.eth0 = DeviceReq(label="to_switch",
driver=RecipeParam("driver"))
+    host1.eth1 = DeviceReq(label="to_switch",
driver=RecipeParam("driver"))
+
+    host2 = HostReq()
+    host2.eth0 = DeviceReq(label="to_switch",
driver=RecipeParam("driver"))
+    host2.eth1 = DeviceReq(label="to_switch",
driver=RecipeParam("driver"))
+
+    bonding_mode = StrParam(mandatory=True)
+    lacp_mode = StrParam(mandatory=True)
+
+    hugepages = IntParam(default=12)
+    dpdk_socket_mem = IntParam(default=1024)
+    dpdk_lcore_mask = StrParam(mandatory=True)
+    pmd_cpu_mask = StrParam(mandatory=True)
+
+    testpmd_cores = StrParam(mandatory=True)
+    testpmd_dpdk_cores = StrParam(mandatory=True)
+
+    trex_dpdk_cores = StrParam(mandatory=True)
+    trex_dir = StrParam(mandatory=True)
+
+    perf_duration = IntParam(default=60)
+    perf_iterations = IntParam(default=5)
+    perf_msg_size = IntParam(default=64)
+
+    cpu_perf_tool = Param(default=StatCPUMeasurement)
+    net_perf_tool = Param(default=TRexFlowMeasurement)
+
+    def test_wide_configuration(self):
+        """
+        Test wide configuration for this recipe involves adding an IPv4 and
+        IPv6 address to the matched eth0 and eth1 nics on both hosts.
+
+        | host1.eth0 = 192.168.101.1/24 and fc00::1/64
+        | host1.eth1 = 192.168.102.1/24 and fd00::1/64
+
+        | host2.eth0 = 192.168.101.2/24 and fc00::2/64
+        | host2.eth1 = 192.168.102.2/24 and fd00::2/64
+
+        Both hosts are configured to allocate 1G hugepages and their testing
+        nics drivers are overriden to use vfio-pci driver.
+
+        Host1 is configured to accomodate OvS DPDK specific bonding setup and
+        launches an instance of the TestPMD application to reflect traffic that
+        arrives to the OvS bridge of the host.
+
+        """
+        host1, host2 = self.matched.host1, self.matched.host2
+
+        net_addr_0 = "192.168.101"
+        net_addr6_0 = "fc00:0:0:0"
+
+        net_addr_1 = "192.168.102"
+        net_addr6_1 = "fd00:0:0:0"
+
+        for i, host in enumerate([host1, host2]):
+            host.dummy_cfg = DummyRecipeConfig(
+                eth0=DummyEthDevice(host.eth0.bus_info, host.eth0.hwaddr,
_id="eth" + str(i),
+                                    ips=[ipaddress(net_addr_0 + "." + str(i +
1) + "/24"),
+                                         ipaddress(net_addr6_0 + "::" + str(i +
1) + "/64")]
+                                    ),
+                eth1=DummyEthDevice(host.eth1.bus_info, host.eth1.hwaddr,
_id="eth" + str(i + 1),
+                                    ips=[ipaddress(net_addr_1 + "." + str(i +
1) + "/24"),
+                                         ipaddress(net_addr6_1 + "::" + str(i +
1) + "/64")]
+                                    ),
+
+                netns=host,
+                testpmd=None
+            )
+
+            host.run(f"echo -n {self.params.hugepages}
/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages")
+            host.run("modprobe vfio-pci")
+
+            host.run(f"driverctl set-override {host.dummy_cfg.eth0.bus_info}
vfio-pci")
+            host.run(f"driverctl set-override {host.dummy_cfg.eth1.bus_info}
vfio-pci")
+
+        self.ovs_dpdk_bridge_configuration(host1)
+
+        host1.dummy_cfg.testpmd = host1.run(
+            TestPMD(
+                coremask=self.params.testpmd_cores,
+                pmd_coremask=self.params.testpmd_dpdk_cores,
+                mode="macswap",
+                nics=["vhost0", "vhost1"]),
+            bg=True)
+
+        configuration = super().test_wide_configuration()
+        return configuration
+
+    def ovs_dpdk_bridge_configuration(self, host):
+        """
+        Host has an OvS setup with 2 bridges interconnected with patch ports.
+        First bridge, called "bond_bridge", configures dpdkbond that uses 2
dpdk interfaces.
+        Second bridge, called "test_bridge", configures 2 dpdkvhostuser ports.
+        Finally, both bridges have their flows deleted and set to "NORMAL"
action.
+        """
+        host.run("systemctl enable openvswitch")
+        host.run("systemctl start openvswitch")
+        host.run(f"ovs-vsctl --no-wait set Open_vSwitch .
other_config:dpdk-socket-mem="
+                 f"{self.params.dpdk_socket_mem}")
+        host.run(f"ovs-vsctl --no-wait set Open_vSwitch .
other_config:dpdk-lcore-mask="
+                 f"{self.params.dpdk_lcore_mask}")
+        host.run(f"ovs-vsctl --no-wait set Open_vSwitch .
other_config:pmd-cpu-mask="
+                 f"{self.params.pmd_cpu_mask}")
+        host.run("ovs-vsctl --no-wait set Open_vSwitch .
other_config:dpdk-init=true")
+
+        host.run(f"ovs-vsctl add-br {host.dummy_cfg.bond_br} "
+                 f"-- set bridge {host.dummy_cfg.bond_br}
datapath_type=netdev")
+
+        host.run(f"ovs-vsctl add-bond {host.dummy_cfg.bond_br} dpdkbond dpdk0 dpdk1
"
+                 f"bond_mode={self.params.bonding_mode} lacp={self.params.lacp_mode}
"
+                 f"-- set Interface dpdk0 type=dpdk
options:dpdk-devargs={host.dummy_cfg.eth0.bus_info} "
+                 f"-- set Interface dpdk1 type=dpdk
options:dpdk-devargs={host.dummy_cfg.eth1.bus_info}")
+
+        host.run(f"ovs-vsctl add-br {host.dummy_cfg.test_br} "
+                 f"-- set bridge {host.dummy_cfg.test_br}
datapath_type=netdev")
+
+        host.run(f"ovs-vsctl add-port {host.dummy_cfg.test_br} vhost0 "
+                 f"-- set Interface vhost0 type=dpdkvhostuser")
+        host.run(f"ovs-vsctl add-port {host.dummy_cfg.test_br} vhost1 "
+                 f"-- set Interface vhost1 type=dpdkvhostuser")
+
+        host.run(f"ovs-vsctl -- add-port {host.dummy_cfg.test_br} bond_patch "
+                 "-- set interface bond_patch type=patch options:peer=test_patch
"
+                 f"-- add-port {host.dummy_cfg.bond_br} test_patch "
+                 "-- set interface test_patch type=patch
options:peer=bond_patch")
+
+        host.run(f"ovs-ofctl mod-port {host.dummy_cfg.bond_br} dpdk0 up")
+        host.run(f"ovs-ofctl mod-port {host.dummy_cfg.bond_br} dpdk1 up")
+
+        for bridge in (host.dummy_cfg.test_br, host.dummy_cfg.bond_br):
+            host.run(f"ip l set {bridge} up")
+            host.run(f"ovs-ofctl del-flows {bridge}")
+            host.run(f"ovs-ofctl add-flow {bridge} actions=NORMAL")
+
+    def generate_perf_configurations(self, config):
+        """
+        OvSDPDKBondRecipe perf test configuration generator.
+
+        OvSDPDKBondRecipe requires a specific TrexFlowMeasurement
+        configuration, where it needs additional parameters,
+        therefore it overrides the inherited method
+        from the :any:`BaseEnrtRecipe`.
+
+        The generator creates and loops over all flow combinations
+        between the hosts to measure the performance.
+        In addition to that during each flow combination measurement,
+        we add CPU utilization measurement to run in the background.
+
+        Finally for each generated perf test configuration we register
+        measurement evaluators based on the :any:`cpu_perf_evaluators` and
+        :any:`net_perf_evaluators` properties.
+
+        :return: Perf test configuration
+        :rtype: :any:`PerfRecipeConf`
+        """
+        host1, host2 = self.matched.host1, self.matched.host2
+        testpmd_nics = [host1.dummy_cfg.eth0, host1.dummy_cfg.eth1]
+        trex_nics = [host2.dummy_cfg.eth0, host2.dummy_cfg.eth1]
+        flows = []
+        for trex_nic, testpmd_nic in zip(trex_nics, testpmd_nics):
+            trex_bind = dict(mac_addr=trex_nic.hwaddr,
+                             pci_addr=trex_nic.bus_info,
+                             ip_addr=trex_nic.ips[0],
+                             family=trex_nic.ips[0].family)
+            testpmd_bind = dict(mac_addr=testpmd_nic.hwaddr,
+                                pci_addr=testpmd_nic.bus_info,
+                                ip_addr=testpmd_nic.ips[0],
+                                family=testpmd_nic.ips[0].family)
+            flows.append(PerfFlow(
+                type="UDPMultiflow",
+                generator=host2,
+                generator_nic=trex_nic,
+                generator_bind=trex_bind,
+                receiver=host1,
+                receiver_nic=testpmd_nic,
+                receiver_bind=testpmd_bind,
+                msg_size=self.params.perf_msg_size,
+                duration=self.params.perf_duration,
+                parallel_streams=1,
+                cpupin=None))
+
+        perf_recipe_conf = dict(
+            recipe_config=config,
+            flows=flows,
+        )
+
+        cpu_measurement = self.params.cpu_perf_tool([host1, host2],
+                                                    perf_recipe_conf)
+
+        flows_measurement = TRexFlowMeasurement(
+            flows,
+            self.params.trex_dir,
+            self.params.trex_dpdk_cores.split(","),
+            perf_recipe_conf,
+        )
+        perf_conf = PerfRecipeConf(
+            measurements=[
+                cpu_measurement,
+                flows_measurement,
+            ],
+            iterations=self.params.perf_iterations,
+        )
+        perf_conf.register_evaluators(cpu_measurement, self.cpu_perf_evaluators)
+        perf_conf.register_evaluators(flows_measurement, self.net_perf_evaluators)
+        return perf_conf
+
+    def do_perf_tests(self, recipe_config):
+        """
+        OvSDPDKBondRecipe overrides inherited :any:`do_perf_tests` method,
+        due to having its own :any:`generate_perf_configurations`, which returns
+        only a single :any:`PerfRecipeConf` instead of a list.
+        PerfRecipe methods are used to execute, report and evaluate the results.
+        """
+        perf_config = self.generate_perf_configurations(recipe_config)
+        result = self.perf_test(perf_config)
+        self.perf_report_and_evaluate(result)
+
+    def generate_test_wide_description(self, config):
+        """
+        Test wide description is extended with the bus info of the configured interfaces
+        used for the DPDK purposes.
+        Description is further expanded upon wit the OvS configuration on the host,
+        containing names of the newly created bridges and intefaces, bonding mode and
lacp mode
+        of the dpdkbond interface, as well as patch ports used for the connection of the
bridges.
+        Lastly, configuration of the TestPMD instance is appended.
+        """
+        desc = super().generate_test_wide_description(config)
+        host1, host2 = self.matched.host1, self.matched.host2
+        for i, host in enumerate([host1, host2]):
+            desc += [f"Configured host{i} interfaces with bus info:
{host.dummy_cfg.eth0.bus_info} "
+                     f"and {host.dummy_cfg.eth1.bus_info} to be controlled by
vfio-pci"]
+
+        desc += [
+            f"Created OvS bridges: {host1.dummy_cfg.test_br} and
{host1.dummy_cfg.bond_br} on host1 "
+            f"set with datapath_type to netdev",
+            f"Created dpdkbond on {host1.dummy_cfg.bond_br} with bonding_mode set to
"
+            f"{self.params.bonding_mode} using lacp_mode set to "
+            f"{self.params.lacp_mode}",
+            f"Interconnected OvS bridges with patch ports: test_patch and
bond_patch",
+            f"Deleted all flows on both bridges and created a new one with
\"actions=NORMAL\"",
+            f"TestPMD in a forward-mode macswap started on the host1
{host1.dummy_cfg.test_br} vhost0 port"]
+
+        return desc
+
+    def test_wide_deconfiguration(self, config):
+        """
+        Test wide deconfiguration consists of killing a TestPMD instance created in the
setup,
+        removal of the bridges and process of returning the control of interfaces to the
kernel drivers.
+        However, driverctl utility can get stuck due to the bug in the openvswitch, thus
a simple
+        restart of the service is used to free the handles and resume the driverctl
utility's job.
+        """
+        host1, host2 = self.matched.host1, self.matched.host2
+
+        testpmd = host1.dummy_cfg.testpmd
+        if testpmd:
+            testpmd.kill(signal.SIGINT)
+            testpmd.wait()
+
+        host1.run(f"ovs-vsctl del-br {host1.dummy_cfg.test_br}")
+        host1.run(f"ovs-vsctl del-br {host1.dummy_cfg.bond_br}")
+
+        host1.run(f"driverctl unset-override {host1.dummy_cfg.eth0.bus_info} &
sleep 1; systemctl restart openvswitch")
+        host1.run(f"driverctl unset-override {host1.dummy_cfg.eth1.bus_info} &
sleep 1; systemctl restart openvswitch")
+
+        host2.run(f"driverctl unset-override {host2.dummy_cfg.eth0.bus_info}")
+        host2.run(f"driverctl unset-override {host2.dummy_cfg.eth1.bus_info}")
diff --git a/lnst/Recipes/ENRT/__init__.py b/lnst/Recipes/ENRT/__init__.py
index 89d6133..e3c7d1e 100644
--- a/lnst/Recipes/ENRT/__init__.py
+++ b/lnst/Recipes/ENRT/__init__.py
@@ -58,6 +58,7 @@ from lnst.Recipes.ENRT.IpsecEspAeadRecipe  import IpsecEspAeadRecipe
 from lnst.Recipes.ENRT.IpsecEspAhCompRecipe import IpsecEspAhCompRecipe
 from lnst.Recipes.ENRT.NoVirtOvsVxlanRecipe import NoVirtOvsVxlanRecipe
 from lnst.Recipes.ENRT.OvS_DPDK_PvP import OvSDPDKPvPRecipe
+from lnst.Recipes.ENRT.OvSDPDKBondRecipe import OvSDPDKBondRecipe
 from lnst.Recipes.ENRT.PingFloodRecipe import PingFloodRecipe
 from lnst.Recipes.ENRT.SimpleMacsecRecipe import SimpleMacsecRecipe
 from lnst.Recipes.ENRT.ShortLivedConnectionsRecipe import ShortLivedConnectionsRecipe
-- 
2.29.2