9:17 a.m.
From: Perry Gagne <pgagne(a)redhat.com>
We decided to simplify the API a bit to just have and import and export method. We also
decided to add compression to the exported file.
Updated docs with new API and examples.
Recipe.py: Added recipe reference to RecipeRun so we only have to specify one variable
when exporting. Recipe is still excluded from export.
Also added shortcut for recipe name to fix LNST-project/lnst#194
tester_api.rst: Added link to docs for RecipeRunExport
Signed-off-by: Perry Gagne <pgagne(a)redhat.com>
---
docs/source/recipe_run_export.rst | 5 +
docs/source/tester_api.rst | 1 +
lnst/Controller/Controller.py | 2 +-
lnst/Controller/Recipe.py | 16 ++-
lnst/Controller/RecipeRunExport.py | 161 +++++++++++++++--------------
5 files changed, 107 insertions(+), 78 deletions(-)
create mode 100644 docs/source/recipe_run_export.rst
diff --git a/docs/source/recipe_run_export.rst b/docs/source/recipe_run_export.rst
new file mode 100644
index 0000000..6d366c2
--- /dev/null
+++ b/docs/source/recipe_run_export.rst
@@ -0,0 +1,5 @@
+Export/Import of Recipe Runs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: lnst.Controller.RecipeRunExport
+ :members:
\ No newline at end of file
diff --git a/docs/source/tester_api.rst b/docs/source/tester_api.rst
index e7cf743..65a2e1b 100644
--- a/docs/source/tester_api.rst
+++ b/docs/source/tester_api.rst
@@ -8,3 +8,4 @@ Test developer API
recipe_api
controller_api
parameters
+ recipe_run_export
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py
index 1641ed3..d63ec90 100644
--- a/lnst/Controller/Controller.py
+++ b/lnst/Controller/Controller.py
@@ -156,7 +156,7 @@ class Controller(object):
logging.info(line)
try:
self._map_match(match, req, recipe)
- recipe._init_run(RecipeRun(match,
log_dir=self._log_ctl.get_recipe_log_path()))
+ recipe._init_run(RecipeRun(recipe, match,
log_dir=self._log_ctl.get_recipe_log_path()))
recipe.test()
except Exception as exc:
logging.error("Recipe execution terminated by unexpected
exception")
diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py
index a9f7514..3bb6793 100644
--- a/lnst/Controller/Recipe.py
+++ b/lnst/Controller/Recipe.py
@@ -126,6 +126,10 @@ class BaseRecipe(object):
def _set_ctl(self, ctl):
self._ctl = ctl
+ @property
+ def name(self):
+ return self.__class__.__name__
+
@property
def matched(self):
if self.ctl is None:
@@ -152,11 +156,12 @@ class BaseRecipe(object):
level, data_level))
class RecipeRun(object):
- def __init__(self, match, desc=None, log_dir=None):
+ def __init__(self, recipe: BaseRecipe, match, desc=None, log_dir=None):
self._match = match
self._desc = desc
self._results = []
self._log_dir = log_dir
+ self._recipe = recipe
def add_result(self, result):
if not isinstance(result, BaseResult):
@@ -196,3 +201,12 @@ class RecipeRun(object):
@property
def overall_result(self):
return all([i.success for i in self.results])
+
+ @property
+ def recipe(self) -> BaseRecipe:
+ return self._recipe
+
+ def __getstate__(self):
+ state = self.__dict__.copy()
+ state['_recipe'] = None
+ return state
diff --git a/lnst/Controller/RecipeRunExport.py b/lnst/Controller/RecipeRunExport.py
index cf40ae4..9486fe8 100644
--- a/lnst/Controller/RecipeRunExport.py
+++ b/lnst/Controller/RecipeRunExport.py
@@ -1,20 +1,33 @@
+"""This module provides an API to export data related to a recipe run.
This export can be used to gather data like CPU and iperf statistic for later analysis,
like when onboarding a new Recipe.
+
+The data that is exported is the instance of :py:class:`lnst.Controller.Recipe.RecipeRun`
that was run.
+
+Data is encapsuled in an :py:class:`RecipeRunData` object which allows us to include
other metadata. The `RecipeRunData` object is "pickled" and compressed with
LZMA/XZ compression using :py:mod:`lzma`.
+
+By default the file will be contain a file extension `.lrc` which stands for "LNST
Run, Compressed".
+
+Use :py:meth:`export_recipe_run` to export and :py:meth:`import_recipe_run` to import.
+"""
+
+__author__ = """
+pgagne(a)redhat.com (Perry Gagne)
+"""
+
import datetime
import pickle
import os
import logging
-from typing import List, Tuple
-from lnst.Controller.Recipe import BaseRecipe, RecipeRun
+import lzma
+from lnst.Controller.Recipe import RecipeRun
class RecipeRunData:
"""
- Class used to encapsulate a RecipeRun, this is the object
- that will be pickled and output to a file.
+ Class used to encapsulate a :py:class:`lnst.Controller.Recipe.RecipeRun`, this is the
object
+ that will be pickled, compressed and output to a file.
- :param recipe_cls:
- class of the Recipe. We do not currently pickle the instance
- of the recipe itself for ease of exporting.
- :type recipe_cls: :py:class: `lnst.Controller.Recipe.BaseRecipe`
+ :param recipe_cls: class of the Recipe. We do not currently pickle the instance of
the recipe itself.
+ :type recipe_cls: :py:class:`lnst.Controller.Recipe.BaseRecipe`
:param param: Copy of Recipe parameters.
:type param: dict
@@ -22,91 +35,87 @@ class RecipeRunData:
:param req: Copy of Recipe requirements
:type req: dict
- :param environ: A copy of `os.environ` created when the object is instantiated.
+ :param environ: A copy of :py:data:`os.environ` created when this object is
instantiated.
:type environ: dict
- :param run: :py:class:`lnst.Controller.Recipe.RecipeRun` instance of the run
+ :param run: instance of the run
:type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
- :param datetime: A time stamp that is the result of running `datetime.datetime.now()`
during instantiation
+ :param datetime: A time stamp that is the result of running
:py:meth:`datetime.datetime.now` during instantiation
:type datetime: :py:class:`datetime.datetime`
"""
-
- def __init__(self, recipe: BaseRecipe, run: RecipeRun):
- self.recipe_cls = recipe.__class__
- self.params = recipe.params._to_dict()
- self.req = recipe.req._to_dict()
+ def __init__(self, run: RecipeRun):
+ self.recipe_cls = run.recipe.__class__
+ self.params = run.recipe.params._to_dict()
+ self.req = run.recipe.req._to_dict()
self.environ = os.environ.copy()
self.run = run
self.datetime = datetime.datetime.now()
-class RecipeRunExporter:
+def export_recipe_run(run: RecipeRun, export_dir: str = None, name: str = None) ->
str:
"""
- Class used to export recipe runs.
+ Export a recipe run to a file. Run is encapsulated, pickled and compressed.
+
+ :param run: `RecipeRun` object to export.
+ :type run: :py:class: `lnst.Controller.Recipe.RecipeRun`
+ :param export_dir: Directory to export file to. Defaults to :py:attr:`run.log_dir`
+ :type export_dir: str
+ :param name: Name of output (exclusive of directory). Defaults to
`<recipename>-run-<timestamp>.lrc`.
+ :type name: str
+ :return: Path of output file.
+ :rtype: str
+
+ Example::
+
+ ctl = Controller(...)
+ recipe = BondRecipe(...)
+ ctl.run(recipe)
+ ...
+ >>> path = export_recipe_run(recipe.run[0])
+ 2020-10-02 15:20:58 (localhost) - INFO: Exported BondRecipe run data to
/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc
+ >>> print(path)
+
/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc
"""
+ run_data = RecipeRunData(run)
+ if not name:
+ name =
f"{run.recipe.name}-run-{run_data.datetime:%Y-%m-%d_%H:%M:%S}.lrc"
+ if not export_dir:
+ export_dir = run.log_dir
- def __init__(self, recipe: BaseRecipe):
- """
-
- :param recipe: Recipe
- :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
- """
- self.recipe = recipe
- self.recipe_name = self.recipe.__class__.__name__
-
- def export_run(self, run: RecipeRun, dir=None, name=None) -> str:
- """
-
- :param run: The RecipeRun to export
- :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
- :param dir: The path to directory to export to. Does not include file name,
defaults to `run.log_dir`
- :type dir: str
- :param name: Name of file to export. Default
`<recipename>-run-<timestamp>.dat'
- :type name: str
- :return: Path (dir+filename) of exported run.
- :rtype:str
- """
- data = RecipeRunData(self.recipe, run)
- if not name:
- name =
f"{self.recipe_name}-run-{data.datetime:%Y-%m-%d_%H:%M:%S}.dat"
- if not dir:
- dir = run.log_dir
-
- path = os.path.join(dir, name)
-
- with open(path, 'wb') as f:
- pickle.dump(data, f)
-
- logging.info(f"Exported {self.recipe_name} data to {path}")
- return path
-
-
-def export_recipe_runs(recipe: BaseRecipe) -> List[Tuple[str, RecipeRun]]:
- """
- Helper method that exports all runs in a recipe.
- :param recipe: Recipe to export
- :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
- :return: list of files that contain exported recipe runs
- :rtype: list
- """
- exporter = RecipeRunExporter(recipe)
- files = []
- for run in recipe.runs:
- path = exporter.export_run(run)
- files.append((run, path))
- return files
+ path = os.path.join(export_dir, name)
+ with lzma.open(path, 'wb') as f:
+ pickle.dump(run_data, f)
+ logging.info(f"Exported {run.recipe.name} run data to {path}")
+ return path
def import_recipe_run(path: str) -> RecipeRunData:
"""
- Import recipe runs that have been exported using
:py:class:`lnst.Controller.RecipeRunExport.RecipeRunExporter`
- :param path: path to file to import
- :type path: str
- :return: `RecipeRun` object containing the run and other metadata.
- :rtype: :py:class:`lnst.Controller.RecipeRunExport.RecipeRunData`
+ Import a recipe run that was exported using :py:meth:`export_recipe_run`
+
+ :param path: Path to file to import
+ :type path: str
+ :return: object which contains the imported recipe run
+ :rtype: :py:class:`RecipeRunData`
+
+ Example::
+
+ >>> run_data =
import_recipe_run("/tmp/lnst-logs/2020-10-02_15:20:18/BondRecipe_match_0/BondRecipe-run-2020-10-02_15:20:58.lrc")
+ >>> type(run_data)
+ <class 'lnst.Controller.RecipeRunExport.RecipeRunData'>
+ >>> run_data.recipe_cls
+ <class 'lnst.Recipes.ENRT.BondRecipe.BondRecipe'>
+ >>> run_data.run.results[38]
+ <lnst.Controller.RecipeResults.Result object at 0x7f20727e1e20>
+ >>> run_data.run.results[38].data
+ {'cpu': [[[<lnst.RecipeCommon.Perf.Results.PerfInterval object at
0x7f20727e1e50>,...]]], ... }
+ >>> print(run_data.run.results[38].description)
+ CPU Utilization on host host1:
+ cpu 'cpu': 45.40 +-0.00 time units per second
+ cpu 'cpu0': 45.40 +-0.00 time units per second
"""
- with open(path, 'rb') as f:
- data = pickle.load(f)
- return data
+ with lzma.open(path, 'rb') as f:
+ run_data = pickle.load(f)
+ return run_data
--
2.26.2