9:24 a.m.
From: Perry Gagne <pgagne(a)redhat.com>
We decided to simplify the API to just have import and export methods.
We also decided to add compression to the exported file.
Updated docs with new API and examples.
Recipe.py:
- Added recipe reference to RecipeRun, to simplify method call
- Made Recipe pickle-able.
- Make export/import of runs based on RecipeRun directly, rather then
wrapping them in RecipeRunData
- Metadata (datetime and environ) was added to RecipeRun, all other
attributes are accessible via the `run.recipe` attribute.
- Export/import functions where moved to Recipe.py.
Signed-off-by: Perry Gagne <pgagne(a)redhat.com>
---
docs/source/recipe_run_export.rst | 16 +++++
docs/source/tester_api.rst | 1 +
lnst/Controller/Controller.py | 2 +-
lnst/Controller/Recipe.py | 97 ++++++++++++++++++++++++-
lnst/Controller/RecipeRunExport.py | 112 -----------------------------
5 files changed, 114 insertions(+), 114 deletions(-)
create mode 100644 docs/source/recipe_run_export.rst
delete mode 100644 lnst/Controller/RecipeRunExport.py
diff --git a/docs/source/recipe_run_export.rst b/docs/source/recipe_run_export.rst
new file mode 100644
index 0000000..3500655
--- /dev/null
+++ b/docs/source/recipe_run_export.rst
@@ -0,0 +1,16 @@
+Export/Import of Recipe Runs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Recipe runs can be exported to a file for for later analysis.
+For example, to gather data like CPU and iperf statistic when onboarding a new recipe.
+
+The data that is exported is the instance of :py:class:`lnst.Controller.Recipe.RecipeRun`
that was run.
+
+The :py:class:`RecipeRun` object is "pickled" and compressed with LZMA/XZ
compression using :py:mod:`lzma`.
+
+By default the file will be contain a file extension `.lrc` which stands for "LNST
Run, Compressed".
+
+Use :py:meth:`export_recipe_run` to export and :py:meth:`import_recipe_run` to import.
+
+.. autofunction:: lnst.Controller.Recipe.export_recipe_run
+
+.. autofunction:: lnst.Controller.Recipe.import_recipe_run
diff --git a/docs/source/tester_api.rst b/docs/source/tester_api.rst
index e7cf743..65a2e1b 100644
--- a/docs/source/tester_api.rst
+++ b/docs/source/tester_api.rst
@@ -8,3 +8,4 @@ Test developer API
recipe_api
controller_api
parameters
+ recipe_run_export
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py
index 1641ed3..d63ec90 100644
--- a/lnst/Controller/Controller.py
+++ b/lnst/Controller/Controller.py
@@ -156,7 +156,7 @@ class Controller(object):
logging.info(line)
try:
self._map_match(match, req, recipe)
- recipe._init_run(RecipeRun(match,
log_dir=self._log_ctl.get_recipe_log_path()))
+ recipe._init_run(RecipeRun(recipe, match,
log_dir=self._log_ctl.get_recipe_log_path()))
recipe.test()
except Exception as exc:
logging.error("Recipe execution terminated by unexpected
exception")
diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py
index a9f7514..9617495 100644
--- a/lnst/Controller/Recipe.py
+++ b/lnst/Controller/Recipe.py
@@ -7,7 +7,11 @@ olichtne(a)redhat.com (Ondrej Lichtner)
"""
import copy
+import datetime
import logging
+import lzma
+import os
+import pickle
from lnst.Common.Parameters import Parameters, Param
from lnst.Common.Colours import decorate_with_preset
from lnst.Controller.Requirements import _Requirements, HostReq
@@ -126,6 +130,7 @@ class BaseRecipe(object):
def _set_ctl(self, ctl):
self._ctl = ctl
+
@property
def matched(self):
if self.ctl is None:
@@ -151,12 +156,21 @@ class BaseRecipe(object):
self.current_run.add_result(Result(success, description, data,
level, data_level))
+ def __getstate__(self):
+ state = self.__dict__.copy()
+ state['_ctl'] = None
+ return state
+
+
class RecipeRun(object):
- def __init__(self, match, desc=None, log_dir=None):
+ def __init__(self, recipe: BaseRecipe, match, desc=None, log_dir=None):
self._match = match
self._desc = desc
self._results = []
self._log_dir = log_dir
+ self._recipe = recipe
+ self._datetime = datetime.datetime.now()
+ self._environ = os.environ.copy()
def add_result(self, result):
if not isinstance(result, BaseResult):
@@ -196,3 +210,84 @@ class RecipeRun(object):
@property
def overall_result(self):
return all([i.success for i in self.results])
+
+ @property
+ def recipe(self) -> BaseRecipe:
+ return self._recipe
+
+ @property
+ def datetime(self):
+ return self._datetime
+
+ @property
+ def environ(self):
+ return self._environ
+
+
+def export_recipe_run(run: RecipeRun, export_dir: str = None, name: str = None) ->
str:
+ """
+ Export a recipe run to a file. :py:class:`RecipeRun` is pickled and compressed.
+
+ :param run: `RecipeRun` object to export.
+ :type run: :py:class:`RecipeRun`
+ :param export_dir: Directory to export file to. Defaults to :py:attr:`run.log_dir`
+ :type export_dir: str
+ :param name: Name of output (exclusive of directory). Defaults to
`<recipename>-run-<timestamp>.lrc`.
+ :type name: str
+ :return: Path of output file.
+ :rtype: str
+
+ Example::
+
+ ctl = Controller(...)
+ recipe = BondRecipe(...)
+ ctl.run(recipe)
+ ...
+ >>> from lnst.Controller.Recipe import export_recipe_run
+ >>> path = export_recipe_run(recipe.run[0])
+ 2020-10-02 15:20:58 (localhost) - INFO: Exported BondRecipe run data to
/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc
+ >>> print(path)
+
/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc
+
+ """
+ if not name:
+ name =
f"{run.recipe.__class__.__name__}-run-{run.datetime:%Y-%m-%d_%H:%M:%S}.lrc"
+ if not export_dir:
+ export_dir = run.log_dir
+
+ path = os.path.join(export_dir, name)
+ with lzma.open(path, 'wb') as f:
+ pickle.dump(run, f)
+ logging.info(f"Exported {run.recipe.__class__.__name__} run to {path}")
+ return path
+
+
+def import_recipe_run(path: str) -> RecipeRun:
+ """
+ Import a recipe run that was exported using :py:meth:`export_recipe_run`
+
+ :param path: Path to file to import
+ :type path: str
+ :return: object which contains the imported recipe run
+ :rtype: :py:class:`RecipeRun`
+
+ Example::
+
+ >>> from lnst.Controller.Recipe import import_recipe_run
+ >>> run =
import_recipe_run("/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc")
+ >>> type(run)
+ <class 'lnst.Controller.Recipe.RecipeRun'>
+ >>> run.recipe.__class__
+ <class 'lnst.Recipes.ENRT.BondRecipe.BondRecipe'>
+ >>> run.results[38]
+ <lnst.Controller.RecipeResults.Result object at 0x7f20727e1e20>
+ >>> run.results[38].data
+ {'cpu': [[[<lnst.RecipeCommon.Perf.Results.PerfInterval object at
0x7f20727e1e50>,...]]], ... }
+ >>> print(run.results[38].description)
+ CPU Utilization on host host1:
+ cpu 'cpu': 45.40 +-0.00 time units per second
+ cpu 'cpu0': 45.40 +-0.00 time units per second
+ """
+ with lzma.open(path, 'rb') as f:
+ run = pickle.load(f)
+ return run
diff --git a/lnst/Controller/RecipeRunExport.py b/lnst/Controller/RecipeRunExport.py
deleted file mode 100644
index cf40ae4..0000000
--- a/lnst/Controller/RecipeRunExport.py
+++ /dev/null
@@ -1,112 +0,0 @@
-import datetime
-import pickle
-import os
-import logging
-from typing import List, Tuple
-from lnst.Controller.Recipe import BaseRecipe, RecipeRun
-
-
-class RecipeRunData:
- """
- Class used to encapsulate a RecipeRun, this is the object
- that will be pickled and output to a file.
-
- :param recipe_cls:
- class of the Recipe. We do not currently pickle the instance
- of the recipe itself for ease of exporting.
- :type recipe_cls: :py:class: `lnst.Controller.Recipe.BaseRecipe`
-
- :param param: Copy of Recipe parameters.
- :type param: dict
-
- :param req: Copy of Recipe requirements
- :type req: dict
-
- :param environ: A copy of `os.environ` created when the object is instantiated.
- :type environ: dict
-
- :param run: :py:class:`lnst.Controller.Recipe.RecipeRun` instance of the run
- :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
-
- :param datetime: A time stamp that is the result of running `datetime.datetime.now()`
during instantiation
- :type datetime: :py:class:`datetime.datetime`
- """
-
- def __init__(self, recipe: BaseRecipe, run: RecipeRun):
- self.recipe_cls = recipe.__class__
- self.params = recipe.params._to_dict()
- self.req = recipe.req._to_dict()
- self.environ = os.environ.copy()
- self.run = run
- self.datetime = datetime.datetime.now()
-
-
-class RecipeRunExporter:
- """
- Class used to export recipe runs.
-
- """
-
- def __init__(self, recipe: BaseRecipe):
- """
-
- :param recipe: Recipe
- :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
- """
- self.recipe = recipe
- self.recipe_name = self.recipe.__class__.__name__
-
- def export_run(self, run: RecipeRun, dir=None, name=None) -> str:
- """
-
- :param run: The RecipeRun to export
- :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
- :param dir: The path to directory to export to. Does not include file name,
defaults to `run.log_dir`
- :type dir: str
- :param name: Name of file to export. Default
`<recipename>-run-<timestamp>.dat'
- :type name: str
- :return: Path (dir+filename) of exported run.
- :rtype:str
- """
- data = RecipeRunData(self.recipe, run)
- if not name:
- name =
f"{self.recipe_name}-run-{data.datetime:%Y-%m-%d_%H:%M:%S}.dat"
- if not dir:
- dir = run.log_dir
-
- path = os.path.join(dir, name)
-
- with open(path, 'wb') as f:
- pickle.dump(data, f)
-
- logging.info(f"Exported {self.recipe_name} data to {path}")
- return path
-
-
-def export_recipe_runs(recipe: BaseRecipe) -> List[Tuple[str, RecipeRun]]:
- """
- Helper method that exports all runs in a recipe.
- :param recipe: Recipe to export
- :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
- :return: list of files that contain exported recipe runs
- :rtype: list
- """
- exporter = RecipeRunExporter(recipe)
- files = []
- for run in recipe.runs:
- path = exporter.export_run(run)
- files.append((run, path))
- return files
-
-
-def import_recipe_run(path: str) -> RecipeRunData:
- """
- Import recipe runs that have been exported using
:py:class:`lnst.Controller.RecipeRunExport.RecipeRunExporter`
- :param path: path to file to import
- :type path: str
- :return: `RecipeRun` object containing the run and other metadata.
- :rtype: :py:class:`lnst.Controller.RecipeRunExport.RecipeRunData`
- """
- with open(path, 'rb') as f:
- data = pickle.load(f)
- return data
--
2.26.2