[PATCH 1/6] Recipes.ENRT.BasePvPRecipe: fix streams parameter typo

From: Ondrej Lichtner

Signed-off-by: Ondrej Lichtner 
---
 lnst/Recipes/ENRT/BasePvPRecipe.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lnst/Recipes/ENRT/BasePvPRecipe.py b/lnst/Recipes/ENRT/BasePvPRecipe.py
index 89e4293..690c60a 100644
--- a/lnst/Recipes/ENRT/BasePvPRecipe.py
+++ b/lnst/Recipes/ENRT/BasePvPRecipe.py
@@ -75,7 +75,7 @@ class BasePvPRecipe(PingTestAndEvaluate, PerfRecipe):
     perf_duration = IntParam(default=60)
     perf_iterations = IntParam(default=5)
     perf_msg_size = IntParam(default=64)
-    perf_streams = IntParam(default=1)
+    perf_parallel_streams = IntParam(default=1)

nr_hugepages = IntParam(default=13000)
     # TODO: Allow 1G hugepages as well

[PATCH 2/6] Recipes.ENRT.BasePvPRecipe: add evaluators properties

From: Ondrej Lichtner

Includes default values.

Signed-off-by: Ondrej Lichtner 
---
 lnst/Recipes/ENRT/BasePvPRecipe.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/lnst/Recipes/ENRT/BasePvPRecipe.py b/lnst/Recipes/ENRT/BasePvPRecipe.py
index 690c60a..c4987cf 100644
--- a/lnst/Recipes/ENRT/BasePvPRecipe.py
+++ b/lnst/Recipes/ENRT/BasePvPRecipe.py
@@ -10,6 +10,7 @@
 from lnst.RecipeCommon.Perf.Recipe import Recipe as PerfRecipe
 from lnst.RecipeCommon.LibvirtControl import LibvirtControl
 from lnst.RecipeCommon.Perf.Measurements import StatCPUMeasurement
+from lnst.RecipeCommon.Perf.Evaluators import NonzeroFlowEvaluator

VirtioType = Enum('VirtType', 'VHOST_USER, VHOST_NET')

@@ -281,3 +282,27 @@ def test_wide_deconfiguration(self, config):

def test_wide_configuration(self, config):
         pass
+
+    @property
+    def cpu_perf_evaluators(self):
+        """CPU measurement evaluators
+
+        To be overriden by a derived class. Returns the list of evaluators to
+        use for CPU utilization measurement evaluation.
+
+        :return: a list of cpu evaluator objects
+        :rtype: List[BaseEvaluator]
+        """
+        return []
+
+    @property
+    def net_perf_evaluators(self):
+        """Network flow measurement evaluators
+
+        To be overriden bby a derived class. Returns the list of evaluators to
+        use for Network flow measurement evaluation.
+
+        :return: a list of flow evaluator objects
+        :rtype: List[BaseEvaluator]
+        """
+        return [NonzeroFlowEvaluator()]

[PATCH 3/6] TRexFlowMeasurement: extend init to add recipe_conf

From: Ondrej Lichtner

This is used by evaluator classes to access the recipe during evaluation
for parameters or other data useful for evaluating the generated
results.

Signed-off-by: Ondrej Lichtner 
---
 .../Perf/Measurements/TRexFlowMeasurement.py           | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py b/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py
index 5f5c705..800b7b7 100644
--- a/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py
+++ b/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py
@@ -16,7 +16,15 @@
 from lnst.Tests.TRex import TRexServer, TRexClient

class TRexFlowMeasurement(BaseFlowMeasurement):
-    def __init__(self, flows, trex_dir, server_cpu_cores):
+    def __init__(self, flows, trex_dir, server_cpu_cores, recipe_conf=None):
+        super(TRexFlowMeasurement, self).__init__(
+            measurement_conf=dict(
+                flows=flows,
+                trex_dir=trex_dir,
+                server_cpu_cores=server_cpu_cores,
+            ),
+            recipe_conf=recipe_conf,
+        )
         self._flows = flows
         self._trex_dir = trex_dir
         self._server_cpu_cores = server_cpu_cores

[PATCH 4/6] TRexFlowMeasurement: add version property

From: Ondrej Lichtner

Based on the IperfFlowMeasurement implementation, there's a measurement
version indicating the version of the FlowMeasurement class, and there
are multiple host versions available (in case there's multiple hosts
running trex) reporting the used version of the trex software.

Signed-off-by: Ondrej Lichtner 
---
 .../Perf/Measurements/TRexFlowMeasurement.py  | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py b/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py
index 800b7b7..c12e922 100644
--- a/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py
+++ b/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py
@@ -1,5 +1,6 @@
 import time
 import signal
+import re
 from lnst.Controller.Recipe import RecipeError
 from lnst.Controller.RecipeResults import ResultLevel

@@ -16,6 +17,8 @@
 from lnst.Tests.TRex import TRexServer, TRexClient

class TRexFlowMeasurement(BaseFlowMeasurement):
+    _MEASUREMENT_VERSION = 1
+
     def __init__(self, flows, trex_dir, server_cpu_cores, recipe_conf=None):
         super(TRexFlowMeasurement, self).__init__(
             measurement_conf=dict(
@@ -32,6 +35,26 @@ def __init__(self, flows, trex_dir, server_cpu_cores, recipe_conf=None):
         self._running_measurements = []
         self._finished_measurements = []

+        self._hosts_versions = {}
+
+    @property
+    def version(self):
+        if not self._hosts_versions:
+            for flow in self._flows:
+                if flow.generator not in self._hosts_versions:
+                    self._hosts_versions[flow.generator] = self._get_host_trex_version(flow.generator)
+
+        return {"measurement_version": self._MEASUREMENT_VERSION,
+                "hosts_trex_versions": self._hosts_versions}
+
+    def _get_host_trex_version(self, host):
+        version_job = host.run("cd {trex_dir} ; ./t-rex-64 --help", job_level = ResultLevel.DEBUG)
+        if version_job.passed:
+            match = re.match(r"Starting  TRex (v.+?) please wait  ...", version_job.stdout)
+            if match:
+                return match.group(1)
+        return None
+
     def start(self):
         if len(self._running_measurements) > 0:
             raise MeasurementError("Measurement already running!")

[PATCH 5/6] Recipes.ENRT.OvSDPDKPvPRecipe: register evaluators, and some fixes

From: Ondrej Lichtner

Evaluation registration is based on the BaseEnrtRecipe.

Fixed Flow creation broken by d47db62030debb1d5504c16ca0e40caf17647a03
when "generator_nic" and "receiver_nic" were added to the Flow class.

The source and destination bind object now includes the "family"
attribute as it's something usually accessed during evaluation.

Signed-off-by: Ondrej Lichtner 
---
 lnst/Recipes/ENRT/OvS_DPDK_PvP.py | 32 ++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/lnst/Recipes/ENRT/OvS_DPDK_PvP.py b/lnst/Recipes/ENRT/OvS_DPDK_PvP.py
index 899057d..4e2aa00 100644
--- a/lnst/Recipes/ENRT/OvS_DPDK_PvP.py
+++ b/lnst/Recipes/ENRT/OvS_DPDK_PvP.py
@@ -130,34 +130,48 @@ def generate_perf_config(self, config):
         for src_nic, dst_nic in zip(config.generator.nics, config.dut.nics):
             src_bind = dict(mac_addr=src_nic.hwaddr,
                             pci_addr=src_nic.bus_info,
-                            ip_addr=src_nic.ips[0])
+                            ip_addr=src_nic.ips[0],
+                            family=src_nic.ips[0].family)
             dst_bind = dict(mac_addr=dst_nic.hwaddr,
                             pci_addr=dst_nic.bus_info,
-                            ip_addr=dst_nic.ips[0])
+                            ip_addr=dst_nic.ips[0],
+                            family=dst_nic.ips[0].family)
             flows.append(PerfFlow(
                 type="pvp_loop_rate",
                 generator=config.generator.host,
+                generator_nic=src_nic,
                 generator_bind=src_bind,
                 receiver=config.dut.host,
+                receiver_nic=dst_nic,
                 receiver_bind=dst_bind,
                 msg_size=self.params.perf_msg_size,
                 duration=self.params.perf_duration,
                 parallel_streams=self.params.perf_streams,
                 cpupin=None))

-        return PerfRecipeConf(
-            measurements=[
-                self.params.cpu_perf_tool(
-                    [config.generator.host, config.dut.host, config.guest.host]
-                ),
-                TRexFlowMeasurement(
+        perf_recipe_conf=dict(
+            recipe_config=config,
+            flows=flows,
+        )
+
+        cpu_measurement = self.params.cpu_perf_tool([config.generator.host, config.dut.host, config.guest.host], perf_recipe_conf)
+
+        flows_measurement = TRexFlowMeasurement(
                     flows,
                     self.params.trex_dir,
                     self.params.host1_dpdk_cores.split(","),
-                ),
+                    perf_recipe_conf,
+                )
+        perf_conf = PerfRecipeConf(
+            measurements=[
+                cpu_measurement,
+                flows_measurement,
             ],
             iterations=self.params.perf_iterations,
         )
+        perf_conf.register_evaluators(cpu_measurement, self.cpu_perf_evaluators)
+        perf_conf.register_evaluators(flows_measurement, self.net_perf_evaluators)
+        return perf_conf

def test_wide_deconfiguration(self, config):
         try:

[PATCH 6/6] BaselineFlowAverageEvaluator: implement metric selection

From: Ondrej Lichtner

It may sometimes be useful to ignore some metrics during evaluation. At
the moment this is intended to be used with TRexFlowMeasurement where
generator and receiver cpu stats are identical (because both generator
and receiver are often on the same host). And the generator throughtput
results are often less stable and less important.

Signed-off-by: Ondrej Lichtner 
---
 .../BaselineFlowAverageEvaluator.py           | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lnst/RecipeCommon/Perf/Evaluators/BaselineFlowAverageEvaluator.py b/lnst/RecipeCommon/Perf/Evaluators/BaselineFlowAverageEvaluator.py
index 9d65955..f7a2fb1 100644
--- a/lnst/RecipeCommon/Perf/Evaluators/BaselineFlowAverageEvaluator.py
+++ b/lnst/RecipeCommon/Perf/Evaluators/BaselineFlowAverageEvaluator.py
@@ -8,9 +8,19 @@

class BaselineFlowAverageEvaluator(BaselineEvaluator):
-    def __init__(self, pass_difference):
+    def __init__(self, pass_difference, metrics_to_evaluate=None):
         self._pass_difference = pass_difference

+        if metrics_to_evaluate is not None:
+            self._metrics_to_evaluate = metrics_to_evaluate
+        else:
+            self._metrics_to_evaluate = [
+                "generator_results",
+                "generator_cpu_stats",
+                "receiver_results",
+                "receiver_cpu_stats",
+            ]
+
     def describe_group_results(self, recipe, results):
         result = results[0]
         return [
@@ -28,12 +38,7 @@ def compare_result_with_baseline(self, recipe, result, baseline):
             comparison_result = False
             result_text.append("No baseline found for this flow")
         else:
-            for i in [
-                "generator_results",
-                "generator_cpu_stats",
-                "receiver_results",
-                "receiver_cpu_stats",
-            ]:
+            for i in self._metrics_to_evaluate:
                 comparison, text = self._average_diff_comparison(
                     name="{} average".format(i),
                     target=getattr(result, i),

[Jozef Urbanovsky]

Patchset looks good to me, ack.

On Fri, Aug 21, 2020 at 3:17 PM  wrote:
> From: Ondrej Lichtner
> Signed-off-by: Ondrej Lichtner
>
> lnst/Recipes/ENRT/BasePvPRecipe.py | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/lnst/Recipes/ENRT/BasePvPRecipe.py
> b/lnst/Recipes/ENRT/BasePvPRecipe.py
> index 89e4293..690c60a 100644
> --- a/lnst/Recipes/ENRT/BasePvPRecipe.py
> +++ b/lnst/Recipes/ENRT/BasePvPRecipe.py
> @@ -75,7 +75,7 @@ class BasePvPRecipe(PingTestAndEvaluate, PerfRecipe):
>      perf_duration = IntParam(default=60)
>      perf_iterations = IntParam(default=5)
>      perf_msg_size = IntParam(default=64)
> perf_streams = IntParam(default=1)
>
> perf_parallel_streams = IntParam(default=1)
>
> nr_hugepages = IntParam(default=13000)
> # TODO: Allow 1G hugepages as well
>
> --
> 2.28.0
> _______________________________________________
> LNST-developers mailing list -- [e-mail]
> To unsubscribe send an email to
> [e-mail]
> Fedora Code of Conduct:
> https://docs.fedoraproject.org/en-US/project/code-of-conduct/
> List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
> List Archives:
> https://lists.fedorahosted.org/archives/list/[e-mail]...

[Jan Tluka]

Fri, Aug 21, 2020 at 03:08:20PM CEST, [e-mail] wrote:
> From: Ondrej Lichtner
> Signed-off-by: Ondrej Lichtner
>
> lnst/Recipes/ENRT/BasePvPRecipe.py | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/lnst/Recipes/ENRT/BasePvPRecipe.py b/lnst/Recipes/ENRT/BasePvPRecipe.py
> index 89e4293..690c60a 100644
> --- a/lnst/Recipes/ENRT/BasePvPRecipe.py
> +++ b/lnst/Recipes/ENRT/BasePvPRecipe.py
> @@ -75,7 +75,7 @@ class BasePvPRecipe(PingTestAndEvaluate, PerfRecipe):
>     perf_duration = IntParam(default=60)
>     perf_iterations = IntParam(default=5)
>     perf_msg_size = IntParam(default=64)
> perf_streams = IntParam(default=1)
>
> perf_parallel_streams = IntParam(default=1)
>
> nr_hugepages = IntParam(default=13000)
> # TODO: Allow 1G hugepages as well
>
> -- 
> 2.28.0
> _______________________________________________
> LNST-developers mailing list -- [e-mail]
> To unsubscribe send an email to [e-mail]
> Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
> List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
> List Archives: https://lists.fedorahosted.org/archives/list/[e-mail]...

Ack to series.

Acked-by: Jan Tluka

[Ondrej Lichtner]

On Fri, Aug 21, 2020 at 03:08:20PM +0200, [e-mail] wrote:
> From: Ondrej Lichtner
> Signed-off-by: Ondrej Lichtner
>
> lnst/Recipes/ENRT/BasePvPRecipe.py | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/lnst/Recipes/ENRT/BasePvPRecipe.py b/lnst/Recipes/ENRT/BasePvPRecipe.py
> index 89e4293..690c60a 100644
> --- a/lnst/Recipes/ENRT/BasePvPRecipe.py
> +++ b/lnst/Recipes/ENRT/BasePvPRecipe.py
> @@ -75,7 +75,7 @@ class BasePvPRecipe(PingTestAndEvaluate, PerfRecipe):
>      perf_duration = IntParam(default=60)
>      perf_iterations = IntParam(default=5)
>      perf_msg_size = IntParam(default=64)
> perf_streams = IntParam(default=1)
>
> perf_parallel_streams = IntParam(default=1)
>
> nr_hugepages = IntParam(default=13000)
> # TODO: Allow 1G hugepages as well
>
> -- 
> 2.28.0

pushed.

-Ondrej