From: Perry Gagne pgagne@redhat.com
Controller.py: Add log_dir to RecipeRun call.
Job.py: Added __get_state__ to control what gets pickled during Recipe Run export. Remove netns since it contains a lot of stuff that cant be easily exported.
Recipe.py: Added log_dir paramter to RecipeRun in order to more centrally track it so it can be used for exporting.
RecipeResults.py: Added __getstate__ to DeviceConfigResult to remove _device during pickling.
RecipeRunExport.py: Add RecipeRunExporter and RecipeRunData and related stuff to be used for exporting.
Signed-off-by: Perry Gagne pgagne@redhat.com --- lnst/Controller/Controller.py | 2 +- lnst/Controller/Job.py | 7 ++ lnst/Controller/Recipe.py | 7 +- lnst/Controller/RecipeResults.py | 5 ++ lnst/Controller/RecipeRunExport.py | 110 +++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 lnst/Controller/RecipeRunExport.py
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py index 160e8b7..1641ed3 100644 --- a/lnst/Controller/Controller.py +++ b/lnst/Controller/Controller.py @@ -156,7 +156,7 @@ class Controller(object): logging.info(line) try: self._map_match(match, req, recipe) - recipe._init_run(RecipeRun(match)) + recipe._init_run(RecipeRun(match, log_dir=self._log_ctl.get_recipe_log_path())) recipe.test() except Exception as exc: logging.error("Recipe execution terminated by unexpected exception") diff --git a/lnst/Controller/Job.py b/lnst/Controller/Job.py index 1ec85e3..d0757f6 100644 --- a/lnst/Controller/Job.py +++ b/lnst/Controller/Job.py @@ -230,3 +230,10 @@ class Job(object): attrs.append(repr(self._what))
return ", ".join(attrs) + + def __getstate__(self): + #Remove things that can't be pickled + #TODO figure out better place holder values + state = self.__dict__.copy() + state['_netns'] = None + return state diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py index 26e0737..a9f7514 100644 --- a/lnst/Controller/Recipe.py +++ b/lnst/Controller/Recipe.py @@ -152,10 +152,11 @@ class BaseRecipe(object): level, data_level))
class RecipeRun(object): - def __init__(self, match, desc=None): + def __init__(self, match, desc=None, log_dir=None): self._match = match self._desc = desc self._results = [] + self._log_dir = log_dir
def add_result(self, result): if not isinstance(result, BaseResult): @@ -176,6 +177,10 @@ class RecipeRun(object): logging.info("Result: {}, What:".format(result_str)) logging.info("{}".format(result.description))
+ @property + def log_dir(self): + return self._log_dir + @property def match(self): return self._match diff --git a/lnst/Controller/RecipeResults.py b/lnst/Controller/RecipeResults.py index 38a4170..941ea8d 100644 --- a/lnst/Controller/RecipeResults.py +++ b/lnst/Controller/RecipeResults.py @@ -118,6 +118,11 @@ class DeviceConfigResult(BaseResult): def device(self): return self._device
+ def __getstate__(self): + state = self.__dict__.copy() + # Remove things that can't be pickled + state['_device'] = None + return state
class DeviceCreateResult(DeviceConfigResult): @property diff --git a/lnst/Controller/RecipeRunExport.py b/lnst/Controller/RecipeRunExport.py new file mode 100644 index 0000000..b0ecccb --- /dev/null +++ b/lnst/Controller/RecipeRunExport.py @@ -0,0 +1,110 @@ +import datetime +import pickle +import os +import logging +from typing import List, Tuple +from lnst.Controller.Recipe import BaseRecipe, RecipeRun + + +class RecipeRunData: + """ + Class used to encapsulate a RecipeRun, this is the object + that will be pickled and output to a file. + + :param recipe_cls: + class of the Recipe. We do not currently pickle the instance + of the recipe itself for ease of exporting. + :type recipe_cls: :py:class: `lnst.Controller.Recipe.BaseRecipe` + + :param param: Copy of Recipe parameters. + :type param: dict + + :param req: Copy of Recipe requirements + :type req: dict + + :param environ: A copy of `os.environ` created when the object is instantiated. + :type environ: dict + + :param run: :py:class:`lnst.Controller.Recipe.RecipeRun` instance of the run + :type run: :py:class:`lnst.Controller.Recipe.RecipeRun` + + :param datetime: A time stamp that is the result of running `datetime.datetime.now()` during instantiation + :type datetime: :py:class:`datetime.datetime` + """ + + def __init__(self, recipe: BaseRecipe, run: RecipeRun): + self.recipe_cls = recipe.__class__ + self.params = recipe.params._to_dict() + self.req = recipe.req._to_dict() + self.environ = os.environ.copy() + self.run = run + self.datetime = datetime.datetime.now() + + +class RecipeRunExporter: + """ + Class used to export recipe runs. + + """ + + def __init__(self, recipe: BaseRecipe): + """ + + :param recipe: Recipe + :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe` + """ + self.recipe = recipe + self.recipe_name = self.recipe.__class__.__name__ + + def export_run(self, run: RecipeRun, dir=None, name=None) -> str: + """ + + :param run: The RecipeRun to export + :type run: :py:class:`lnst.Controller.Recipe.RecipeRun` + :param dir: The path to directory to export to. Does not include file name, defaults to `run.log_dir` + :type dir: str + :param name: Name of file to export. Default `<recipename>-run-<timestamp>.dat' + :type name: str + :return: Path (dir+filename) of exported run. + :rtype:str + """ + data = RecipeRunData(self.recipe, run) + if not name: + name = f"{self.recipe_name}-run-{run.datetime:%Y-%m-%d_%H:%M:%S}.dat" + if not dir: + dir = os.path.join(run.log_dir, name) + + with open(dir, 'wb') as f: + pickle.dump(data, f) + + logging.info(f"Exported {self.recipe_name} data to {dir}") + return dir + + +def export_recipe_runs(recipe: BaseRecipe) -> List[Tuple[str, RecipeRun]]: + """ + Helper method that exports all runs in a recipe. + :param recipe: Recipe to export + :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe` + :return: list of files that contain exported recipe runs + :rtype: list + """ + exporter = RecipeRunExporter(recipe) + files = [] + for run in recipe.runs: + path = exporter.export_run(run) + files.append((run, path)) + return files + + +def import_recipe_run(path: str) -> RecipeRunData: + """ + Import recipe runs that have been exported using :py:class:`lnst.Controller.RecipeRunExport.RecipeRunExporter` + :param path: path to file to import + :type path: str + :return: `RecipeRun` object containing the run and other metadata. + :rtype: :py:class:`lnst.Controller.RecipeRunExport.RecipeRunData` + """ + with open(path, 'rb') as f: + data = pickle.load(f) + return data
Please disregard this for now, i found a type that i will fix and resubmit (RecipeRunData.__init__ uses run.datetime instead of data.datetime)
On Wed, Aug 26, 2020 at 3:19 PM pgagne@redhat.com wrote:
From: Perry Gagne pgagne@redhat.com
Controller.py: Add log_dir to RecipeRun call.
Job.py: Added __get_state__ to control what gets pickled during Recipe Run export. Remove netns since it contains a lot of stuff that cant be easily exported.
Recipe.py: Added log_dir paramter to RecipeRun in order to more centrally track it so it can be used for exporting.
RecipeResults.py: Added __getstate__ to DeviceConfigResult to remove _device during pickling.
RecipeRunExport.py: Add RecipeRunExporter and RecipeRunData and related stuff to be used for exporting.
Signed-off-by: Perry Gagne pgagne@redhat.com
lnst/Controller/Controller.py | 2 +- lnst/Controller/Job.py | 7 ++ lnst/Controller/Recipe.py | 7 +- lnst/Controller/RecipeResults.py | 5 ++ lnst/Controller/RecipeRunExport.py | 110 +++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 lnst/Controller/RecipeRunExport.py
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py index 160e8b7..1641ed3 100644 --- a/lnst/Controller/Controller.py +++ b/lnst/Controller/Controller.py @@ -156,7 +156,7 @@ class Controller(object): logging.info(line) try: self._map_match(match, req, recipe)
recipe._init_run(RecipeRun(match))
recipe._init_run(RecipeRun(match,
log_dir=self._log_ctl.get_recipe_log_path())) recipe.test() except Exception as exc: logging.error("Recipe execution terminated by unexpected exception") diff --git a/lnst/Controller/Job.py b/lnst/Controller/Job.py index 1ec85e3..d0757f6 100644 --- a/lnst/Controller/Job.py +++ b/lnst/Controller/Job.py @@ -230,3 +230,10 @@ class Job(object): attrs.append(repr(self._what))
return ", ".join(attrs)
- def __getstate__(self):
#Remove things that can't be pickled
#TODO figure out better place holder values
state = self.__dict__.copy()
state['_netns'] = None
return state
diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py index 26e0737..a9f7514 100644 --- a/lnst/Controller/Recipe.py +++ b/lnst/Controller/Recipe.py @@ -152,10 +152,11 @@ class BaseRecipe(object): level, data_level))
class RecipeRun(object):
- def __init__(self, match, desc=None):
def __init__(self, match, desc=None, log_dir=None): self._match = match self._desc = desc self._results = []
self._log_dir = log_dir
def add_result(self, result): if not isinstance(result, BaseResult):
@@ -176,6 +177,10 @@ class RecipeRun(object): logging.info("Result: {}, What:".format(result_str)) logging.info("{}".format(result.description))
- @property
- def log_dir(self):
return self._log_dir
- @property def match(self): return self._match
diff --git a/lnst/Controller/RecipeResults.py b/lnst/Controller/RecipeResults.py index 38a4170..941ea8d 100644 --- a/lnst/Controller/RecipeResults.py +++ b/lnst/Controller/RecipeResults.py @@ -118,6 +118,11 @@ class DeviceConfigResult(BaseResult): def device(self): return self._device
- def __getstate__(self):
state = self.__dict__.copy()
# Remove things that can't be pickled
state['_device'] = None
return state
class DeviceCreateResult(DeviceConfigResult): @property diff --git a/lnst/Controller/RecipeRunExport.py b/lnst/Controller/RecipeRunExport.py new file mode 100644 index 0000000..b0ecccb --- /dev/null +++ b/lnst/Controller/RecipeRunExport.py @@ -0,0 +1,110 @@ +import datetime +import pickle +import os +import logging +from typing import List, Tuple +from lnst.Controller.Recipe import BaseRecipe, RecipeRun
+class RecipeRunData:
- """
- Class used to encapsulate a RecipeRun, this is the object
- that will be pickled and output to a file.
- :param recipe_cls:
class of the Recipe. We do not currently pickle the instance
of the recipe itself for ease of exporting.
- :type recipe_cls: :py:class: `lnst.Controller.Recipe.BaseRecipe`
- :param param: Copy of Recipe parameters.
- :type param: dict
- :param req: Copy of Recipe requirements
- :type req: dict
- :param environ: A copy of `os.environ` created when the object is
instantiated.
- :type environ: dict
- :param run: :py:class:`lnst.Controller.Recipe.RecipeRun` instance of
the run
- :type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
- :param datetime: A time stamp that is the result of running
`datetime.datetime.now()` during instantiation
- :type datetime: :py:class:`datetime.datetime`
- """
- def __init__(self, recipe: BaseRecipe, run: RecipeRun):
self.recipe_cls = recipe.__class__
self.params = recipe.params._to_dict()
self.req = recipe.req._to_dict()
self.environ = os.environ.copy()
self.run = run
self.datetime = datetime.datetime.now()
+class RecipeRunExporter:
- """
- Class used to export recipe runs.
- """
- def __init__(self, recipe: BaseRecipe):
"""
:param recipe: Recipe
:type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
"""
self.recipe = recipe
self.recipe_name = self.recipe.__class__.__name__
- def export_run(self, run: RecipeRun, dir=None, name=None) -> str:
"""
:param run: The RecipeRun to export
:type run: :py:class:`lnst.Controller.Recipe.RecipeRun`
:param dir: The path to directory to export to. Does not include
file name, defaults to `run.log_dir`
:type dir: str
:param name: Name of file to export. Default
`<recipename>-run-<timestamp>.dat'
:type name: str
:return: Path (dir+filename) of exported run.
:rtype:str
"""
data = RecipeRunData(self.recipe, run)
if not name:
name =
f"{self.recipe_name}-run-{run.datetime:%Y-%m-%d_%H:%M:%S}.dat"
if not dir:
dir = os.path.join(run.log_dir, name)
with open(dir, 'wb') as f:
pickle.dump(data, f)
logging.info(f"Exported {self.recipe_name} data to {dir}")
return dir
+def export_recipe_runs(recipe: BaseRecipe) -> List[Tuple[str, RecipeRun]]:
- """
- Helper method that exports all runs in a recipe.
- :param recipe: Recipe to export
- :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe`
- :return: list of files that contain exported recipe runs
- :rtype: list
- """
- exporter = RecipeRunExporter(recipe)
- files = []
- for run in recipe.runs:
path = exporter.export_run(run)
files.append((run, path))
- return files
+def import_recipe_run(path: str) -> RecipeRunData:
- """
- Import recipe runs that have been exported using
:py:class:`lnst.Controller.RecipeRunExport.RecipeRunExporter`
- :param path: path to file to import
- :type path: str
- :return: `RecipeRun` object containing the run and other metadata.
- :rtype: :py:class:`lnst.Controller.RecipeRunExport.RecipeRunData`
- """
- with open(path, 'rb') as f:
data = pickle.load(f)
- return data
-- 2.26.2
lnst-developers@lists.fedorahosted.org