From 799df2b8eb411f8d5c0d239469ea304cf81424ff Mon Sep 17 00:00:00 2001 From: Cody Wild Date: Fri, 26 Jul 2019 16:25:14 -0700 Subject: [PATCH 1/2] Pretty hacky version of nested ingredients --- sacred/experiment.py | 4 +++- sacred/initialize.py | 3 ++- sacred/run.py | 14 +++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/sacred/experiment.py b/sacred/experiment.py index aa1014ef..286b497b 100755 --- a/sacred/experiment.py +++ b/sacred/experiment.py @@ -36,7 +36,7 @@ class Experiment(Ingredient): """ def __init__(self, name=None, ingredients=(), interactive=False, - base_dir=None): + base_dir=None, nest_ingredients=False): """ Create a new experiment with the given name and optional ingredients. @@ -87,6 +87,7 @@ def __init__(self, name=None, ingredients=(), interactive=False, self.observers = [] self.current_run = None self.captured_out_filter = None + self.nest_ingredients = nest_ingredients """Filter function to be applied to captured output of a run""" self.option_hooks = [] @@ -444,6 +445,7 @@ def _create_run(self, command_name=None, config_updates=None, force=options.get(ForceOption.get_flag(), False), log_level=options.get(LoglevelOption.get_flag(), None)) + if info is not None: run.info.update(info) diff --git a/sacred/initialize.py b/sacred/initialize.py index 020b752c..57586934 100755 --- a/sacred/initialize.py +++ b/sacred/initialize.py @@ -419,7 +419,8 @@ def create_run(experiment, command_name, config_updates=None, run = Run(config, config_modifications, main_function, copy(experiment.observers), root_logger, run_logger, experiment_info, host_info, pre_runs, post_runs, - experiment.captured_out_filter) + experiment.captured_out_filter, sorted_ingredients, + experiment.nest_ingredients) if hasattr(main_function, 'unobserved'): run.unobserved = main_function.unobserved diff --git a/sacred/run.py b/sacred/run.py index 10cc6d25..52974c0f 100755 --- a/sacred/run.py +++ b/sacred/run.py @@ -12,6 +12,7 @@ from sacred.utils import (SacredInterrupt, join_paths, IntervalTimer) from sacred.stdout_capturing import get_stdcapturer +from sacred.observers import FileStorageObserver class Run: @@ -19,7 +20,8 @@ class Run: def __init__(self, config, config_modifications, main_function, observers, root_logger, run_logger, experiment_info, host_info, - pre_run_hooks, post_run_hooks, captured_out_filter=None): + pre_run_hooks, post_run_hooks, captured_out_filter=None, + sorted_ingredients=None, nest_ingredients=False): self._id = None """The ID of this run as assigned by the first observer""" @@ -102,6 +104,10 @@ def __init__(self, config, config_modifications, main_function, observers, self.capture_mode = None """Determines the way the stdout/stderr are captured""" + self.sorted_ingredients = sorted_ingredients + + self.nest_ingredients = nest_ingredients + self._heartbeat = None self._failed_observers = [] self._output_file = None @@ -326,6 +332,12 @@ def _emit_started(self): ) if self._id is None: self._id = _id + if observer.__class__.__name__ == 'FileStorageObserver' and self.nest_ingredients: + for ingredient in self.sorted_ingredients: + ingredient_observer_dir = os.path.join(observer.dir, ingredient.path) + new_observer = FileStorageObserver.create(ingredient_observer_dir) + ingredient.observers.append(new_observer) + # do not catch any exceptions on startup: # the experiment SHOULD fail if any of the observers fails if self._id is None: From 3b54c48f167275d80587a7b2eb6a2ff37b988a77 Mon Sep 17 00:00:00 2001 From: Cody Wild Date: Thu, 22 Aug 2019 12:08:52 -0700 Subject: [PATCH 2/2] actually fix merge issues, black is weird --- sacred/experiment.py | 2 +- sacred/initialize.py | 2 +- sacred/run.py | 11 ++++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/sacred/experiment.py b/sacred/experiment.py index b2c9ef21..897faaf6 100755 --- a/sacred/experiment.py +++ b/sacred/experiment.py @@ -56,7 +56,7 @@ def __init__( interactive: bool = False, base_dir: Optional[PathType] = None, additional_host_info: List[HostInfoGetter] = None, - nest_ingredients: bool = False + nest_ingredients: bool = False, ): """ Create a new experiment with the given name and optional ingredients. diff --git a/sacred/initialize.py b/sacred/initialize.py index 5aae04bd..55422a9e 100755 --- a/sacred/initialize.py +++ b/sacred/initialize.py @@ -471,7 +471,7 @@ def create_run( post_runs, experiment.captured_out_filter, sorted_ingredients, - experiment.nest_ingredients + experiment.nest_ingredients, ) if hasattr(main_function, "unobserved"): diff --git a/sacred/run.py b/sacred/run.py index 627ff31f..cc6aef87 100755 --- a/sacred/run.py +++ b/sacred/run.py @@ -31,7 +31,7 @@ def __init__( post_run_hooks, captured_out_filter=None, sorted_ingredients=None, - nest_ingredients=False + nest_ingredients=False, ): self._id = None @@ -341,9 +341,14 @@ def _emit_started(self): ) if self._id is None: self._id = _id - if observer.__class__.__name__ == 'FileStorageObserver' and self.nest_ingredients: + if ( + observer.__class__.__name__ == "FileStorageObserver" + and self.nest_ingredients + ): for ingredient in self.sorted_ingredients: - ingredient_observer_dir = os.path.join(observer.dir, ingredient.path) + ingredient_observer_dir = os.path.join( + observer.dir, ingredient.path + ) new_observer = FileStorageObserver.create(ingredient_observer_dir) ingredient.observers.append(new_observer) # do not catch any exceptions on startup: