[PATCH] Add support for exporting recipe run data

From: Perry Gagne <pgagne(a)redhat.com&gt;

Controller.py: Add log_dir to RecipeRun call.

Job.py: Added __get_state__ to control what gets pickled during Recipe Run export.
Remove netns since it contains a lot of stuff that cant be easily exported.

Recipe.py: Added log_dir paramter to RecipeRun in order to more centrally track it
so it can be used for exporting.

RecipeResults.py: Added __getstate__ to DeviceConfigResult to remove _device during
pickling.

RecipeRunExport.py: Add RecipeRunExporter and RecipeRunData and related stuff to be used
for exporting.

Signed-off-by: Perry Gagne <pgagne(a)redhat.com&gt;
---
 lnst/Controller/Controller.py      |   2 +-
 lnst/Controller/Job.py             |   7 ++
 lnst/Controller/Recipe.py          |   7 +-
 lnst/Controller/RecipeResults.py   |   5 ++
 lnst/Controller/RecipeRunExport.py | 110 +++++++++++++++++++++++++++++
 5 files changed, 129 insertions(+), 2 deletions(-)
 create mode 100644 lnst/Controller/RecipeRunExport.py

diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py
index 160e8b7..1641ed3 100644
--- a/lnst/Controller/Controller.py
+++ b/lnst/Controller/Controller.py
@@ -156,7 +156,7 @@ class Controller(object):
                 logging.info(line)
             try:
                 self._map_match(match, req, recipe)
-                recipe._init_run(RecipeRun(match))
+                recipe._init_run(RecipeRun(match,
log_dir=self._log_ctl.get_recipe_log_path()))
                 recipe.test()
             except Exception as exc:
                 logging.error("Recipe execution terminated by unexpected
exception")
diff --git a/lnst/Controller/Job.py b/lnst/Controller/Job.py
index 1ec85e3..d0757f6 100644
--- a/lnst/Controller/Job.py
+++ b/lnst/Controller/Job.py
@@ -230,3 +230,10 @@ class Job(object):
         attrs.append(repr(self._what))
 
         return ", ".join(attrs)
+
+    def __getstate__(self):
+        #Remove things that can't be pickled
+        #TODO figure out better place holder values
+        state = self.__dict__.copy()
+        state['_netns'] = None
+        return state
diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py
index 26e0737..a9f7514 100644
--- a/lnst/Controller/Recipe.py
+++ b/lnst/Controller/Recipe.py
@@ -152,10 +152,11 @@ class BaseRecipe(object):
                                            level, data_level))
 
 class RecipeRun(object):
-    def __init__(self, match, desc=None):
+    def __init__(self, match, desc=None, log_dir=None):
         self._match = match
         self._desc = desc
         self._results = []
+        self._log_dir = log_dir
 
     def add_result(self, result):
         if not isinstance(result, BaseResult):
@@ -176,6 +177,10 @@ class RecipeRun(object):
             logging.info("Result: {}, What:".format(result_str))
             logging.info("{}".format(result.description))
 
+    @property
+    def log_dir(self):
+        return self._log_dir
+
     @property
     def match(self):
         return self._match
diff --git a/lnst/Controller/RecipeResults.py b/lnst/Controller/RecipeResults.py
index 38a4170..941ea8d 100644
--- a/lnst/Controller/RecipeResults.py
+++ b/lnst/Controller/RecipeResults.py
@@ -118,6 +118,11 @@ class DeviceConfigResult(BaseResult):
     def device(self):
         return self._device
 
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        # Remove things that can't be pickled
+        state['_device'] = None
+        return state
 
 class DeviceCreateResult(DeviceConfigResult):
     @property
diff --git a/lnst/Controller/RecipeRunExport.py b/lnst/Controller/RecipeRunExport.py
new file mode 100644
index 0000000..b0ecccb
--- /dev/null
+++ b/lnst/Controller/RecipeRunExport.py
@@ -0,0 +1,110 @@
+import datetime
+import pickle
+import os
+import logging
+from typing import List, Tuple
+from lnst.Controller.Recipe import BaseRecipe, RecipeRun
+
+
+class RecipeRunData:
+    """
+    Class used to encapsulate a RecipeRun, this is the object
+    that will be pickled and output to a file.
+
+    :param recipe_cls:
+        class of the Recipe. We do not currently pickle the instance
+        of the recipe itself for ease of exporting.
+    :type recipe_cls: :py:class: `lnst.Controller.Recipe.BaseRecipe`
+
+    :param param: Copy of Recipe parameters.
+    :type param: dict
+
+    :param req: Copy of Recipe requirements
+    :type req: dict
+
+    :param environ: A copy of `os.environ` created when the object is instantiated.
+    :type environ: dict
+
+    :param run: :py:class:`lnst.Controller.Recipe.RecipeRun` instance of the run
+    :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
+
+    :param datetime: A time stamp that is the result of running `datetime.datetime.now()`
during instantiation
+    :type datetime: :py:class:`datetime.datetime`
+    """
+
+    def __init__(self, recipe: BaseRecipe, run: RecipeRun):
+        self.recipe_cls = recipe.__class__
+        self.params = recipe.params._to_dict()
+        self.req = recipe.req._to_dict()
+        self.environ = os.environ.copy()
+        self.run = run
+        self.datetime = datetime.datetime.now()
+
+
+class RecipeRunExporter:
+    """
+    Class used to export recipe runs.
+
+    """
+
+    def __init__(self, recipe: BaseRecipe):
+        """
+
+        :param recipe: Recipe
+        :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
+        """
+        self.recipe = recipe
+        self.recipe_name = self.recipe.__class__.__name__
+
+    def export_run(self, run: RecipeRun, dir=None, name=None) -> str:
+        """
+
+        :param run: The RecipeRun to export
+        :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
+        :param dir: The path to directory to export to. Does not include file name,
defaults to `run.log_dir`
+        :type dir: str
+        :param name: Name of file to export. Default
`<recipename>-run-<timestamp>.dat'
+        :type name: str
+        :return: Path (dir+filename) of exported run.
+        :rtype:str
+        """
+        data = RecipeRunData(self.recipe, run)
+        if not name:
+            name =
f"{self.recipe_name}-run-{run.datetime:%Y-%m-%d_%H:%M:%S}.dat"
+        if not dir:
+            dir = os.path.join(run.log_dir, name)
+
+        with open(dir, 'wb') as f:
+            pickle.dump(data, f)
+
+        logging.info(f"Exported {self.recipe_name} data to {dir}")
+        return dir
+
+
+def export_recipe_runs(recipe: BaseRecipe) -> List[Tuple[str, RecipeRun]]:
+    """
+    Helper method that exports all runs in a recipe.
+    :param recipe: Recipe to export
+    :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
+    :return: list of files that contain exported recipe runs
+    :rtype: list
+    """
+    exporter = RecipeRunExporter(recipe)
+    files = []
+    for run in recipe.runs:
+        path = exporter.export_run(run)
+        files.append((run, path))
+    return files
+
+
+def import_recipe_run(path: str) -> RecipeRunData:
+    """
+    Import recipe runs that have been exported using
:py:class:`lnst.Controller.RecipeRunExport.RecipeRunExporter`
+    :param path:  path to file  to import
+    :type path: str
+    :return: `RecipeRun` object containing the run and other metadata.
+    :rtype: :py:class:`lnst.Controller.RecipeRunExport.RecipeRunData`
+    """
+    with open(path, 'rb') as f:
+        data = pickle.load(f)
+    return data
-- 
2.26.2

[Perry Gagne]

Please disregard this for now, i found a type that i will fix and resubmit
(RecipeRunData.__init__ uses run.datetime instead of data.datetime)

On Wed, Aug 26, 2020 at 3:19 PM <pgagne(a)redhat.com&gt; wrote:

>  From: Perry Gagne <pgagne(a)redhat.com&gt;
>
>  Controller.py: Add log_dir to RecipeRun call.
>
>  Job.py: Added __get_state__ to control what gets pickled during Recipe Run
>  export.
>  Remove netns since it contains a lot of stuff that cant be easily exported.
>
>  Recipe.py: Added log_dir paramter to RecipeRun in order to more centrally
>  track it
>  so it can be used for exporting.
>
>  RecipeResults.py: Added __getstate__ to DeviceConfigResult to remove
>  _device during pickling.
>
>  RecipeRunExport.py: Add RecipeRunExporter and RecipeRunData and related
>  stuff to be used for exporting.
>
>  Signed-off-by: Perry Gagne <pgagne(a)redhat.com&gt;
>  ---
>   lnst/Controller/Controller.py      |   2 +-
>   lnst/Controller/Job.py             |   7 ++
>   lnst/Controller/Recipe.py          |   7 +-
>   lnst/Controller/RecipeResults.py   |   5 ++
>   lnst/Controller/RecipeRunExport.py | 110 +++++++++++++++++++++++++++++
>   5 files changed, 129 insertions(+), 2 deletions(-)
>   create mode 100644 lnst/Controller/RecipeRunExport.py
>
>  diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py
>  index 160e8b7..1641ed3 100644
>  --- a/lnst/Controller/Controller.py
>  +++ b/lnst/Controller/Controller.py
>  @@ -156,7 +156,7 @@ class Controller(object):
>                   logging.info(line)
>               try:
>                   self._map_match(match, req, recipe)
>  -                recipe._init_run(RecipeRun(match))
>  +                recipe._init_run(RecipeRun(match,
>  log_dir=self._log_ctl.get_recipe_log_path()))
>                   recipe.test()
>               except Exception as exc:
>                   logging.error("Recipe execution terminated by unexpected
>  exception")
>  diff --git a/lnst/Controller/Job.py b/lnst/Controller/Job.py
>  index 1ec85e3..d0757f6 100644
>  --- a/lnst/Controller/Job.py
>  +++ b/lnst/Controller/Job.py
>  @@ -230,3 +230,10 @@ class Job(object):
>           attrs.append(repr(self._what))
>
>           return ", ".join(attrs)
>  +
>  +    def __getstate__(self):
>  +        #Remove things that can't be pickled
>  +        #TODO figure out better place holder values
>  +        state = self.__dict__.copy()
>  +        state['_netns'] = None
>  +        return state
>  diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py
>  index 26e0737..a9f7514 100644
>  --- a/lnst/Controller/Recipe.py
>  +++ b/lnst/Controller/Recipe.py
>  @@ -152,10 +152,11 @@ class BaseRecipe(object):
>                                              level, data_level))
>
>   class RecipeRun(object):
>  -    def __init__(self, match, desc=None):
>  +    def __init__(self, match, desc=None, log_dir=None):
>           self._match = match
>           self._desc = desc
>           self._results = []
>  +        self._log_dir = log_dir
>
>       def add_result(self, result):
>           if not isinstance(result, BaseResult):
>  @@ -176,6 +177,10 @@ class RecipeRun(object):
>               logging.info("Result: {}, What:".format(result_str))
>               logging.info("{}".format(result.description))
>
>  +    @property
>  +    def log_dir(self):
>  +        return self._log_dir
>  +
>       @property
>       def match(self):
>           return self._match
>  diff --git a/lnst/Controller/RecipeResults.py
>  b/lnst/Controller/RecipeResults.py
>  index 38a4170..941ea8d 100644
>  --- a/lnst/Controller/RecipeResults.py
>  +++ b/lnst/Controller/RecipeResults.py
>  @@ -118,6 +118,11 @@ class DeviceConfigResult(BaseResult):
>       def device(self):
>           return self._device
>
>  +    def __getstate__(self):
>  +        state = self.__dict__.copy()
>  +        # Remove things that can't be pickled
>  +        state['_device'] = None
>  +        return state
>
>   class DeviceCreateResult(DeviceConfigResult):
>       @property
>  diff --git a/lnst/Controller/RecipeRunExport.py
>  b/lnst/Controller/RecipeRunExport.py
>  new file mode 100644
>  index 0000000..b0ecccb
>  --- /dev/null
>  +++ b/lnst/Controller/RecipeRunExport.py
>  @@ -0,0 +1,110 @@
>  +import datetime
>  +import pickle
>  +import os
>  +import logging
>  +from typing import List, Tuple
>  +from lnst.Controller.Recipe import BaseRecipe, RecipeRun
>  +
>  +
>  +class RecipeRunData:
>  +    """
>  +    Class used to encapsulate a RecipeRun, this is the object
>  +    that will be pickled and output to a file.
>  +
>  +    :param recipe_cls:
>  +        class of the Recipe. We do not currently pickle the instance
>  +        of the recipe itself for ease of exporting.
>  +    :type recipe_cls: :py:class: `lnst.Controller.Recipe.BaseRecipe`
>  +
>  +    :param param: Copy of Recipe parameters.
>  +    :type param: dict
>  +
>  +    :param req: Copy of Recipe requirements
>  +    :type req: dict
>  +
>  +    :param environ: A copy of `os.environ` created when the object is
>  instantiated.
>  +    :type environ: dict
>  +
>  +    :param run: :py:class:`lnst.Controller.Recipe.RecipeRun` instance of
>  the run
>  +    :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
>  +
>  +    :param datetime: A time stamp that is the result of running
>  `datetime.datetime.now()` during instantiation
>  +    :type datetime: :py:class:`datetime.datetime`
>  +    """
>  +
>  +    def __init__(self, recipe: BaseRecipe, run: RecipeRun):
>  +        self.recipe_cls = recipe.__class__
>  +        self.params = recipe.params._to_dict()
>  +        self.req = recipe.req._to_dict()
>  +        self.environ = os.environ.copy()
>  +        self.run = run
>  +        self.datetime = datetime.datetime.now()
>  +
>  +
>  +class RecipeRunExporter:
>  +    """
>  +    Class used to export recipe runs.
>  +
>  +    """
>  +
>  +    def __init__(self, recipe: BaseRecipe):
>  +        """
>  +
>  +        :param recipe: Recipe
>  +        :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
>  +        """
>  +        self.recipe = recipe
>  +        self.recipe_name = self.recipe.__class__.__name__
>  +
>  +    def export_run(self, run: RecipeRun, dir=None, name=None) -> str:
>  +        """
>  +
>  +        :param run: The RecipeRun to export
>  +        :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
>  +        :param dir: The path to directory to export to. Does not include
>  file name, defaults to `run.log_dir`
>  +        :type dir: str
>  +        :param name: Name of file to export. Default
>  `<recipename>-run-<timestamp>.dat'
>  +        :type name: str
>  +        :return: Path (dir+filename) of exported run.
>  +        :rtype:str
>  +        """
>  +        data = RecipeRunData(self.recipe, run)
>  +        if not name:
>  +            name =
>  f"{self.recipe_name}-run-{run.datetime:%Y-%m-%d_%H:%M:%S}.dat"
>  +        if not dir:
>  +            dir = os.path.join(run.log_dir, name)
>  +
>  +        with open(dir, 'wb') as f:
>  +            pickle.dump(data, f)
>  +
>  +        logging.info(f"Exported {self.recipe_name} data to {dir}")
>  +        return dir
>  +
>  +
>  +def export_recipe_runs(recipe: BaseRecipe) -> List[Tuple[str, RecipeRun]]:
>  +    """
>  +    Helper method that exports all runs in a recipe.
>  +    :param recipe: Recipe to export
>  +    :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
>  +    :return: list of files that contain exported recipe runs
>  +    :rtype: list
>  +    """
>  +    exporter = RecipeRunExporter(recipe)
>  +    files = []
>  +    for run in recipe.runs:
>  +        path = exporter.export_run(run)
>  +        files.append((run, path))
>  +    return files
>  +
>  +
>  +def import_recipe_run(path: str) -> RecipeRunData:
>  +    """
>  +    Import recipe runs that have been exported using
>  :py:class:`lnst.Controller.RecipeRunExport.RecipeRunExporter`
>  +    :param path:  path to file  to import
>  +    :type path: str
>  +    :return: `RecipeRun` object containing the run and other metadata.
>  +    :rtype: :py:class:`lnst.Controller.RecipeRunExport.RecipeRunData`
>  +    """
>  +    with open(path, 'rb') as f:
>  +        data = pickle.load(f)
>  +    return data
>  --
>  2.26.2
>
>