From: Perry Gagne pgagne@redhat.com
Controller.py: Add log_dir to RecipeRun call.
Job.py: Added __get_state__ to control what gets pickled during Recipe Run export. Remove netns since it contains a lot of stuff that cant be easily exported.
Recipe.py: Added log_dir paramter to RecipeRun in order to more centrally track it so it can be used for exporting.
RecipeResults.py: Added __getstate__ to DeviceConfigResult to remove _device during pickling.
RecipeRunExport.py: Add RecipeRunExporter and RecipeRunData and related stuff to be used for exporting.
Signed-off-by: Perry Gagne pgagne@redhat.com --- lnst/Controller/Controller.py | 2 +- lnst/Controller/Job.py | 7 ++ lnst/Controller/Recipe.py | 7 +- lnst/Controller/RecipeResults.py | 5 ++ lnst/Controller/RecipeRunExport.py | 110 +++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 lnst/Controller/RecipeRunExport.py
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py index 160e8b7..1641ed3 100644 --- a/lnst/Controller/Controller.py +++ b/lnst/Controller/Controller.py @@ -156,7 +156,7 @@ class Controller(object): logging.info(line) try: self._map_match(match, req, recipe) - recipe._init_run(RecipeRun(match)) + recipe._init_run(RecipeRun(match, log_dir=self._log_ctl.get_recipe_log_path())) recipe.test() except Exception as exc: logging.error("Recipe execution terminated by unexpected exception") diff --git a/lnst/Controller/Job.py b/lnst/Controller/Job.py index 1ec85e3..d0757f6 100644 --- a/lnst/Controller/Job.py +++ b/lnst/Controller/Job.py @@ -230,3 +230,10 @@ class Job(object): attrs.append(repr(self._what))
return ", ".join(attrs) + + def __getstate__(self): + #Remove things that can't be pickled + #TODO figure out better place holder values + state = self.__dict__.copy() + state['_netns'] = None + return state diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py index 26e0737..a9f7514 100644 --- a/lnst/Controller/Recipe.py +++ b/lnst/Controller/Recipe.py @@ -152,10 +152,11 @@ class BaseRecipe(object): level, data_level))
class RecipeRun(object): - def __init__(self, match, desc=None): + def __init__(self, match, desc=None, log_dir=None): self._match = match self._desc = desc self._results = [] + self._log_dir = log_dir
def add_result(self, result): if not isinstance(result, BaseResult): @@ -176,6 +177,10 @@ class RecipeRun(object): logging.info("Result: {}, What:".format(result_str)) logging.info("{}".format(result.description))
+ @property + def log_dir(self): + return self._log_dir + @property def match(self): return self._match diff --git a/lnst/Controller/RecipeResults.py b/lnst/Controller/RecipeResults.py index 38a4170..941ea8d 100644 --- a/lnst/Controller/RecipeResults.py +++ b/lnst/Controller/RecipeResults.py @@ -118,6 +118,11 @@ class DeviceConfigResult(BaseResult): def device(self): return self._device
+ def __getstate__(self): + state = self.__dict__.copy() + # Remove things that can't be pickled + state['_device'] = None + return state
class DeviceCreateResult(DeviceConfigResult): @property diff --git a/lnst/Controller/RecipeRunExport.py b/lnst/Controller/RecipeRunExport.py new file mode 100644 index 0000000..b0ecccb --- /dev/null +++ b/lnst/Controller/RecipeRunExport.py @@ -0,0 +1,110 @@ +import datetime +import pickle +import os +import logging +from typing import List, Tuple +from lnst.Controller.Recipe import BaseRecipe, RecipeRun + + +class RecipeRunData: + """ + Class used to encapsulate a RecipeRun, this is the object + that will be pickled and output to a file. + + :param recipe_cls: + class of the Recipe. We do not currently pickle the instance + of the recipe itself for ease of exporting. + :type recipe_cls: :py:class: `lnst.Controller.Recipe.BaseRecipe` + + :param param: Copy of Recipe parameters. + :type param: dict + + :param req: Copy of Recipe requirements + :type req: dict + + :param environ: A copy of `os.environ` created when the object is instantiated. + :type environ: dict + + :param run: :py:class:`lnst.Controller.Recipe.RecipeRun` instance of the run + :type run: :py:class:`lnst.Controller.Recipe.RecipeRun` + + :param datetime: A time stamp that is the result of running `datetime.datetime.now()` during instantiation + :type datetime: :py:class:`datetime.datetime` + """ + + def __init__(self, recipe: BaseRecipe, run: RecipeRun): + self.recipe_cls = recipe.__class__ + self.params = recipe.params._to_dict() + self.req = recipe.req._to_dict() + self.environ = os.environ.copy() + self.run = run + self.datetime = datetime.datetime.now() + + +class RecipeRunExporter: + """ + Class used to export recipe runs. + + """ + + def __init__(self, recipe: BaseRecipe): + """ + + :param recipe: Recipe + :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe` + """ + self.recipe = recipe + self.recipe_name = self.recipe.__class__.__name__ + + def export_run(self, run: RecipeRun, dir=None, name=None) -> str: + """ + + :param run: The RecipeRun to export + :type run: :py:class:`lnst.Controller.Recipe.RecipeRun` + :param dir: The path to directory to export to. Does not include file name, defaults to `run.log_dir` + :type dir: str + :param name: Name of file to export. Default `<recipename>-run-<timestamp>.dat' + :type name: str + :return: Path (dir+filename) of exported run. + :rtype:str + """ + data = RecipeRunData(self.recipe, run) + if not name: + name = f"{self.recipe_name}-run-{data.datetime:%Y-%m-%d_%H:%M:%S}.dat" + if not dir: + dir = os.path.join(run.log_dir, name) + + with open(dir, 'wb') as f: + pickle.dump(data, f) + + logging.info(f"Exported {self.recipe_name} data to {dir}") + return dir + + +def export_recipe_runs(recipe: BaseRecipe) -> List[Tuple[str, RecipeRun]]: + """ + Helper method that exports all runs in a recipe. + :param recipe: Recipe to export + :type recipe: :py:class: `lnst.Controller.Recipe.BaseRecipe` + :return: list of files that contain exported recipe runs + :rtype: list + """ + exporter = RecipeRunExporter(recipe) + files = [] + for run in recipe.runs: + path = exporter.export_run(run) + files.append((run, path)) + return files + + +def import_recipe_run(path: str) -> RecipeRunData: + """ + Import recipe runs that have been exported using :py:class:`lnst.Controller.RecipeRunExport.RecipeRunExporter` + :param path: path to file to import + :type path: str + :return: `RecipeRun` object containing the run and other metadata. + :rtype: :py:class:`lnst.Controller.RecipeRunExport.RecipeRunData` + """ + with open(path, 'rb') as f: + data = pickle.load(f) + return data