New subject: [PATCH 2/2] RecipeRunExport.py fixed export path and timestamp

27 Aug 2020

From: Perry Gagne pgagne@redhat.com
Controller.py: Add log_dir to RecipeRun call.
Job.py: Added __get_state__ to control what gets pickled during Recipe Run export.
Remove netns since it contains a lot of stuff that cant be easily exported.
Recipe.py: Added log_dir paramter to RecipeRun in order to more centrally track it
so it can be used for exporting.
RecipeResults.py: Added __getstate__ to DeviceConfigResult to remove _device during pickling.
RecipeRunExport.py: Add RecipeRunExporter and RecipeRunData and related stuff to be used for exporting.
Signed-off-by: Perry Gagne pgagne@redhat.com
---
 lnst/Controller/Controller.py      |   2 +-
 lnst/Controller/Job.py             |   7 ++
 lnst/Controller/Recipe.py          |   7 +-
 lnst/Controller/RecipeResults.py   |   5 ++
 lnst/Controller/RecipeRunExport.py | 110 +++++++++++++++++++++++++++++
 5 files changed, 129 insertions(+), 2 deletions(-)
 create mode 100644 lnst/Controller/RecipeRunExport.py

diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py
index 160e8b7..1641ed3 100644
--- a/lnst/Controller/Controller.py
+++ b/lnst/Controller/Controller.py
@@ -156,7 +156,7 @@ class Controller(object):
                 logging.info(line)
             try:
                 self._map_match(match, req, recipe)
-                recipe._init_run(RecipeRun(match))
+                recipe._init_run(RecipeRun(match, log_dir=self._log_ctl.get_recipe_log_path()))
                 recipe.test()
             except Exception as exc:
                 logging.error("Recipe execution terminated by unexpected exception")
diff --git a/lnst/Controller/Job.py b/lnst/Controller/Job.py
index 1ec85e3..d0757f6 100644
--- a/lnst/Controller/Job.py
+++ b/lnst/Controller/Job.py
@@ -230,3 +230,10 @@ class Job(object):
         attrs.append(repr(self._what))
return ", ".join(attrs)
+
+    def __getstate__(self):
+        #Remove things that can't be pickled
+        #TODO figure out better place holder values
+        state = self.__dict__.copy()
+        state['_netns'] = None
+        return state
diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py
index 26e0737..a9f7514 100644
--- a/lnst/Controller/Recipe.py
+++ b/lnst/Controller/Recipe.py
@@ -152,10 +152,11 @@ class BaseRecipe(object):
                                            level, data_level))
class RecipeRun(object):
-    def __init__(self, match, desc=None):
+    def __init__(self, match, desc=None, log_dir=None):
         self._match = match
         self._desc = desc
         self._results = []
+        self._log_dir = log_dir
def add_result(self, result):
         if not isinstance(result, BaseResult):
@@ -176,6 +177,10 @@ class RecipeRun(object):
             logging.info("Result: {}, What:".format(result_str))
             logging.info("{}".format(result.description))
+    @property
+    def log_dir(self):
+        return self._log_dir
+
     @property
     def match(self):
         return self._match
diff --git a/lnst/Controller/RecipeResults.py b/lnst/Controller/RecipeResults.py
index 38a4170..941ea8d 100644
--- a/lnst/Controller/RecipeResults.py
+++ b/lnst/Controller/RecipeResults.py
@@ -118,6 +118,11 @@ class DeviceConfigResult(BaseResult):
     def device(self):
         return self._device
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        # Remove things that can't be pickled
+        state['_device'] = None
+        return state
class DeviceCreateResult(DeviceConfigResult):
     @property
diff --git a/lnst/Controller/RecipeRunExport.py b/lnst/Controller/RecipeRunExport.py
new file mode 100644
index 0000000..b0ecccb
--- /dev/null
+++ b/lnst/Controller/RecipeRunExport.py
@@ -0,0 +1,110 @@
+import datetime
+import pickle
+import os
+import logging
+from typing import List, Tuple
+from lnst.Controller.Recipe import BaseRecipe, RecipeRun
+
+
+class RecipeRunData:
+    """
+    Class used to encapsulate a RecipeRun, this is the object
+    that will be pickled and output to a file.
+
+    :param recipe_cls:
+        class of the Recipe. We do not currently pickle the instance
+        of the recipe itself for ease of exporting.
+    :type recipe_cls: :py:class: `lnst.Controller.Recipe.BaseRecipe`
+
+    :param param: Copy of Recipe parameters.
+    :type param: dict
+
+    :param req: Copy of Recipe requirements
+    :type req: dict
+
+    :param environ: A copy of `os.environ` created when the object is instantiated.
+    :type environ: dict
+
+    :param run: :py:class:`lnst.Controller.Recipe.RecipeRun` instance of the run
+    :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
+
+    :param datetime: A time stamp that is the result of running `datetime.datetime.now()` during instantiation
+    :type datetime: :py:class:`datetime.datetime`
+    """
+
+    def __init__(self, recipe: BaseRecipe, run: RecipeRun):
+        self.recipe_cls = recipe.__class__
+        self.params = recipe.params._to_dict()
+        self.req = recipe.req._to_dict()
+        self.environ = os.environ.copy()
+        self.run = run
+        self.datetime = datetime.datetime.now()
+
+
+class RecipeRunExporter:
+    """
+    Class used to export recipe runs.
+
+    """
+
+    def __init__(self, recipe: BaseRecipe):
+        """
+
+        :param recipe: Recipe
+        :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
+        """
+        self.recipe = recipe
+        self.recipe_name = self.recipe.__class__.__name__
+
+    def export_run(self, run: RecipeRun, dir=None, name=None) -> str:
+        """
+
+        :param run: The RecipeRun to export
+        :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
+        :param dir: The path to directory to export to. Does not include file name, defaults to `run.log_dir`
+        :type dir: str
+        :param name: Name of file to export. Default `<recipename>-run-<timestamp>.dat'
+        :type name: str
+        :return: Path (dir+filename) of exported run.
+        :rtype:str
+        """
+        data = RecipeRunData(self.recipe, run)
+        if not name:
+            name = f"{self.recipe_name}-run-{run.datetime:%Y-%m-%d_%H:%M:%S}.dat"
+        if not dir:
+            dir = os.path.join(run.log_dir, name)
+
+        with open(dir, 'wb') as f:
+            pickle.dump(data, f)
+
+        logging.info(f"Exported {self.recipe_name} data to {dir}")
+        return dir
+
+
+def export_recipe_runs(recipe: BaseRecipe) -> List[Tuple[str, RecipeRun]]:
+    """
+    Helper method that exports all runs in a recipe.
+    :param recipe: Recipe to export
+    :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
+    :return: list of files that contain exported recipe runs
+    :rtype: list
+    """
+    exporter = RecipeRunExporter(recipe)
+    files = []
+    for run in recipe.runs:
+        path = exporter.export_run(run)
+        files.append((run, path))
+    return files
+
+
+def import_recipe_run(path: str) -> RecipeRunData:
+    """
+    Import recipe runs that have been exported using :py:class:`lnst.Controller.RecipeRunExport.RecipeRunExporter`
+    :param path:  path to file  to import
+    :type path: str
+    :return: `RecipeRun` object containing the run and other metadata.
+    :rtype: :py:class:`lnst.Controller.RecipeRunExport.RecipeRunData`
+    """
+    with open(path, 'rb') as f:
+        data = pickle.load(f)
+    return data
-- 
2.26.2

    

[PATCH 1/2] Add support for exporting recipe run data

Signed-off-by: Perry Gagne pgagne@redhat.com

Signed-off-by: Perry Gagne pgagne@redhat.com