9:17 a.m.
From: Perry Gagne <pgagne(a)redhat.com>
We decided to export RecipeRun objects directly instead of wrapping them in
RecipeRunData.
Metadata (datetime and environ) was added to RecipeRun, all other attributes are
accessible via the `run.recipe` attribute.
Export/import functions where moved to Recipe.py.
Signed-off-by: Perry Gagne <pgagne(a)redhat.com>
---
docs/source/recipe_run_export.rst | 15 +++-
lnst/Controller/Recipe.py | 82 +++++++++++++++++++
lnst/Controller/RecipeRunExport.py | 122 -----------------------------
3 files changed, 95 insertions(+), 124 deletions(-)
delete mode 100644 lnst/Controller/RecipeRunExport.py
diff --git a/docs/source/recipe_run_export.rst b/docs/source/recipe_run_export.rst
index 6d366c2..3500655 100644
--- a/docs/source/recipe_run_export.rst
+++ b/docs/source/recipe_run_export.rst
@@ -1,5 +1,16 @@
Export/Import of Recipe Runs
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Recipe runs can be exported to a file for for later analysis.
+For example, to gather data like CPU and iperf statistic when onboarding a new recipe.
-.. automodule:: lnst.Controller.RecipeRunExport
- :members:
\ No newline at end of file
+The data that is exported is the instance of :py:class:`lnst.Controller.Recipe.RecipeRun`
that was run.
+
+The :py:class:`RecipeRun` object is "pickled" and compressed with LZMA/XZ
compression using :py:mod:`lzma`.
+
+By default the file will be contain a file extension `.lrc` which stands for "LNST
Run, Compressed".
+
+Use :py:meth:`export_recipe_run` to export and :py:meth:`import_recipe_run` to import.
+
+.. autofunction:: lnst.Controller.Recipe.export_recipe_run
+
+.. autofunction:: lnst.Controller.Recipe.import_recipe_run
diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py
index 7fb7006..eb9b344 100644
--- a/lnst/Controller/Recipe.py
+++ b/lnst/Controller/Recipe.py
@@ -7,7 +7,11 @@ olichtne(a)redhat.com (Ondrej Lichtner)
"""
import copy
+import datetime
import logging
+import lzma
+import os
+import pickle
from lnst.Common.Parameters import Parameters, Param
from lnst.Common.Colours import decorate_with_preset
from lnst.Controller.Requirements import _Requirements, HostReq
@@ -168,6 +172,8 @@ class RecipeRun(object):
self._results = []
self._log_dir = log_dir
self._recipe = recipe
+ self._datetime = datetime.datetime.now()
+ self._environ = os.environ.copy()
def add_result(self, result):
if not isinstance(result, BaseResult):
@@ -212,3 +218,79 @@ class RecipeRun(object):
def recipe(self) -> BaseRecipe:
return self._recipe
+ @property
+ def datetime(self):
+ return self._datetime
+
+ @property
+ def environ(self):
+ return self._environ
+
+
+def export_recipe_run(run: RecipeRun, export_dir: str = None, name: str = None) ->
str:
+ """
+ Export a recipe run to a file. :py:class:`RecipeRun` is pickled and compressed.
+
+ :param run: `RecipeRun` object to export.
+ :type run: :py:class:`RecipeRun`
+ :param export_dir: Directory to export file to. Defaults to :py:attr:`run.log_dir`
+ :type export_dir: str
+ :param name: Name of output (exclusive of directory). Defaults to
`<recipename>-run-<timestamp>.lrc`.
+ :type name: str
+ :return: Path of output file.
+ :rtype: str
+
+ Example::
+
+ ctl = Controller(...)
+ recipe = BondRecipe(...)
+ ctl.run(recipe)
+ ...
+ >>> from lnst.Controller.Recipe import export_recipe_run
+ >>> path = export_recipe_run(recipe.run[0])
+ 2020-10-02 15:20:58 (localhost) - INFO: Exported BondRecipe run data to
/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc
+ >>> print(path)
+
/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc
+
+ """
+ if not name:
+ name = f"{run.recipe.name}-run-{run.datetime:%Y-%m-%d_%H:%M:%S}.lrc"
+ if not export_dir:
+ export_dir = run.log_dir
+
+ path = os.path.join(export_dir, name)
+ with lzma.open(path, 'wb') as f:
+ pickle.dump(run, f)
+ logging.info(f"Exported {run.recipe.name} run to {path}")
+ return path
+
+
+def import_recipe_run(path: str) -> RecipeRun:
+ """
+ Import a recipe run that was exported using :py:meth:`export_recipe_run`
+
+ :param path: Path to file to import
+ :type path: str
+ :return: object which contains the imported recipe run
+ :rtype: :py:class:`RecipeRun`
+
+ Example::
+
+ >>> from lnst.Controller.Recipe import import_recipe_run
+ >>> run =
import_recipe_run("/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc")
+ >>> type(run)
+ <class 'lnst.Controller.Recipe.RecipeRun'>
+ >>> run.recipe.__class__
+ <class 'lnst.Recipes.ENRT.BondRecipe.BondRecipe'>
+ >>> run.results[38]
+ <lnst.Controller.RecipeResults.Result object at 0x7f20727e1e20>
+ >>> run.results[38].data
+ {'cpu': [[[<lnst.RecipeCommon.Perf.Results.PerfInterval object at
0x7f20727e1e50>,...]]], ... }
+ >>> print(run.results[38].description)
+ CPU Utilization on host host1:
+ cpu 'cpu': 45.40 +-0.00 time units per second
+ cpu 'cpu0': 45.40 +-0.00 time units per second
+ """
+ with lzma.open(path, 'rb') as f:
+ run = pickle.load(f)
+ return run
diff --git a/lnst/Controller/RecipeRunExport.py b/lnst/Controller/RecipeRunExport.py
deleted file mode 100644
index fc27284..0000000
--- a/lnst/Controller/RecipeRunExport.py
+++ /dev/null
@@ -1,122 +0,0 @@
-"""This module provides an API to export data related to a recipe run.
-This export can be used to gather data like CPU and iperf statistic for later analysis,
like when onboarding a new Recipe.
-
-The data that is exported is the instance of :py:class:`lnst.Controller.Recipe.RecipeRun`
that was run.
-
-Data is encapsuled in an :py:class:`RecipeRunData` object which allows us to include
other metadata.
-The `RecipeRunData` object is "pickled" and compressed with LZMA/XZ compression
using :py:mod:`lzma`.
-
-By default the file will be contain a file extension `.lrc` which stands for "LNST
Run, Compressed".
-
-Use :py:meth:`export_recipe_run` to export and :py:meth:`import_recipe_run` to import.
-"""
-
-__author__ = """
-pgagne(a)redhat.com (Perry Gagne)
-"""
-
-import datetime
-import pickle
-import os
-import logging
-import lzma
-from lnst.Controller.Recipe import RecipeRun
-
-
-class RecipeRunData:
- """
- Class used to encapsulate a :py:class:`lnst.Controller.Recipe.RecipeRun`, this is the
object
- that will be pickled, compressed and output to a file.
-
- :param recipe_cls: class of the Recipe. We do not currently pickle the instance of
the recipe itself.
- :type recipe_cls: :py:class:`lnst.Controller.Recipe.BaseRecipe`
-
- :param param: Copy of Recipe parameters.
- :type param: dict
-
- :param req: Copy of Recipe requirements
- :type req: dict
-
- :param environ: A copy of :py:data:`os.environ` created when this object is
instantiated.
- :type environ: dict
-
- :param run: instance of the run
- :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
-
- :param datetime: A time stamp that is the result of running
:py:meth:`datetime.datetime.now` during instantiation
- :type datetime: :py:class:`datetime.datetime`
- """
- def __init__(self, run: RecipeRun):
- self.params = run.recipe.params._to_dict()
- self.req = run.recipe.req._to_dict()
- self.environ = os.environ.copy()
- self.run = run
- self.datetime = datetime.datetime.now()
-
-
-def export_recipe_run(run: RecipeRun, export_dir: str = None, name: str = None) ->
str:
- """
- Export a recipe run to a file. Run is encapsulated, pickled and compressed.
-
- :param run: `RecipeRun` object to export.
- :type run: :py:class: `lnst.Controller.Recipe.RecipeRun`
- :param export_dir: Directory to export file to. Defaults to :py:attr:`run.log_dir`
- :type export_dir: str
- :param name: Name of output (exclusive of directory). Defaults to
`<recipename>-run-<timestamp>.lrc`.
- :type name: str
- :return: Path of output file.
- :rtype: str
-
- Example::
-
- ctl = Controller(...)
- recipe = BondRecipe(...)
- ctl.run(recipe)
- ...
- >>> path = export_recipe_run(recipe.run[0])
- 2020-10-02 15:20:58 (localhost) - INFO: Exported BondRecipe run data to
/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc
- >>> print(path)
-
/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc
-
- """
- run_data = RecipeRunData(run)
- if not name:
- name =
f"{run.recipe.name}-run-{run_data.datetime:%Y-%m-%d_%H:%M:%S}.lrc"
- if not export_dir:
- export_dir = run.log_dir
-
- path = os.path.join(export_dir, name)
- with lzma.open(path, 'wb') as f:
- pickle.dump(run_data, f)
- logging.info(f"Exported {run.recipe.name} run data to {path}")
- return path
-
-
-def import_recipe_run(path: str) -> RecipeRunData:
- """
- Import a recipe run that was exported using :py:meth:`export_recipe_run`
-
- :param path: Path to file to import
- :type path: str
- :return: object which contains the imported recipe run
- :rtype: :py:class:`RecipeRunData`
-
- Example::
-
- >>> run_data =
import_recipe_run("/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc")
- >>> type(run_data)
- <class 'lnst.Controller.RecipeRunExport.RecipeRunData'>
- >>> run_data.recipe.__class__
- <class 'lnst.Recipes.ENRT.BondRecipe.BondRecipe'>
- >>> run_data.run.results[38]
- <lnst.Controller.RecipeResults.Result object at 0x7f20727e1e20>
- >>> run_data.run.results[38].data
- {'cpu': [[[<lnst.RecipeCommon.Perf.Results.PerfInterval object at
0x7f20727e1e50>,...]]], ... }
- >>> print(run_data.run.results[38].description)
- CPU Utilization on host host1:
- cpu 'cpu': 45.40 +-0.00 time units per second
- cpu 'cpu0': 45.40 +-0.00 time units per second
- """
- with lzma.open(path, 'rb') as f:
- run_data = pickle.load(f)
- return run_data
--
2.26.2