Skip to content

Commit

Permalink
Merge pull request #333 from caracal-pipeline/dump-config
Browse files Browse the repository at this point in the history 
Add option to dump the recipe config file to the stimela run command
  • Loading branch information
@landmanbester
landmanbester authored Sep 4, 2024
2 parents 1c6a973 + cfacb49 commit 077b2e4
Showing 1 changed file with 32 additions and 26 deletions.
58 changes: 32 additions & 26 deletions stimela/commands/run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def resolve_recipe_file(filename: str):
return fname
else:
raise FileNotFoundError(f"{filename} resolves to {fname}, which doesn't exist")
# else check for implicit extension
# else check for implicit extension
else:
for ext in _yaml_extensions:
path = f"{fname}{ext}"
Expand Down Expand Up @@ -138,7 +138,7 @@ def load_recipe_files(filenames: List[str]):
log_exception(f"error in definition of recipe '{name}'", exc)
sys.exit(2)
recipe_names.append(name)

try:
stimela.CONFIG.merge_with(update_conf)
except Exception as exc:
Expand All @@ -149,7 +149,7 @@ def load_recipe_files(filenames: List[str]):

@cli.command("run",
help="""
Execute a single cab, or a recipe from a YML file.
Execute a single cab, or a recipe from a YML file.
If the YML files contains multiple recipes, specify the recipe name as an extra argument.
Use PARAM=VALUE to specify parameters for the recipe or cab. You can also use X.Y.Z=FOO to
change any and all config and/or recipe settings.
Expand All @@ -163,21 +163,21 @@ def load_recipe_files(filenames: List[str]):
help="""forcefully skip specific recipe step(s). Use commas, or give multiple times to
cherry-pick steps. Use [BEGIN]:[END] to specify a range of steps.""")
@click.option("-t", "--tags", "tags", metavar="TAG(s)", multiple=True,
help="""only runs steps wth the given tags (and also steps tagged as "always").
help="""only runs steps wth the given tags (and also steps tagged as "always").
Use commas, or give multiple times for multiple tags.""")
@click.option("--skip-tags", "skip_tags", metavar="TAG(s)", multiple=True,
help="""explicitly skips steps wth the given tags.
help="""explicitly skips steps wth the given tags.
Use commas, or give multiple times for multiple tags.""")
@click.option("-e", "--enable-step", "enable_steps", metavar="STEP(s)", multiple=True,
help="""Force-enable steps even if the recipe marks them as skipped. Use commas, or give multiple times
help="""Force-enable steps even if the recipe marks them as skipped. Use commas, or give multiple times
for multiple steps.""")
@click.option("-c", "--config", "config_equals", metavar="X.Y.Z=VALUE", nargs=1, multiple=True,
help="""tweak configuration options.""")
@click.option("-a", "--assign", metavar="PARAM VALUE", nargs=2, multiple=True,
help="""assigns values to parameters: equivalent to PARAM=VALUE, but plays nicer with the shell's
help="""assigns values to parameters: equivalent to PARAM=VALUE, but plays nicer with the shell's
tab completion feature.""")
@click.option("-C", "--config-assign", metavar="X.Y.Z VALUE", nargs=2, multiple=True,
help="""tweak configuration options: same function -c/--config, but plays nicer with the shell's
help="""tweak configuration options: same function -c/--config, but plays nicer with the shell's
tab completion feature.""")
@click.option("-l", "--last-recipe", is_flag=True,
help="""if multiple recipes are defined, selects the last one for execution.""")
Expand All @@ -193,8 +193,10 @@ def load_recipe_files(filenames: List[str]):
help="""Selects the kubernetes backend (shortcut for -C opts.backend.select=kube)""")
@click.option("--slurm", "enable_slurm", is_flag=True,
help="""Enables the slurm backend wrapper (shortcut for -C backend.slurm.enable=True)""")
@click.argument("parameters", nargs=-1, metavar="filename.yml ... [recipe or cab name] [PARAM=VALUE] ...", required=True)
def run(parameters: List[str] = [], dry_run: bool = False, last_recipe: bool = False, profile: Optional[int] = None,
@click.option("-dc", "--dump-config", is_flag=True,
help="""Dump the equivalent stimela config to a file""")
@click.argument("parameters", nargs=-1, metavar="filename.yml ... [recipe or cab name] [PARAM=VALUE] ...", required=True)
def run(parameters: List[str] = [], dump_config: bool = False, dry_run: bool = False, last_recipe: bool = False, profile: Optional[int] = None,
assign: List[Tuple[str, str]] = [],
config_equals: List[str] = [],
config_assign: List[Tuple[str, str]] = [],
Expand Down Expand Up @@ -227,7 +229,7 @@ def convert_value(value):
log_exception(f"error parsing value for --assign {key} {value}", exc)
errcode = 2

# parse arguments as recipe name, parameter assignments, or dotlist for OmegaConf
# parse arguments as recipe name, parameter assignments, or dotlist for OmegaConf
for pp in parameters:
if "=" in pp:
key, value = pp.split("=", 1)
Expand Down Expand Up @@ -309,7 +311,7 @@ def log_available_runnables():
else:
recipe_name = available_recipes[-1]
log.info(f"-l/--last-recipe specified, selecting '{recipe_name}'")
# nothing specified, either we have exactly 1 recipe defined (pick that), or 0 recipes and 1 cab
# nothing specified, either we have exactly 1 recipe defined (pick that), or 0 recipes and 1 cab
elif len(available_recipes) == 1:
recipe_name = available_recipes[0]
log.info(f"found single recipe '{recipe_name}', selecting it implicitly")
Expand Down Expand Up @@ -337,14 +339,14 @@ def log_available_runnables():

# are we running a standalone cab?
if cab_name is not None:
# create step config by merging in settings (var=value pairs from the command line)
# create step config by merging in settings (var=value pairs from the command line)
outer_step = Step(cab=cab_name, params=params)
outer_step.name = cab_name
# provide basic substitutions for running the step below
subst = SubstitutionNS()
info = SubstitutionNS(fqname=cab_name, label=cab_name, label_parts=[], suffix='', taskname=cab_name)
subst._add_('info', info, nosubst=True)
subst._add_('config', stimela.CONFIG, nosubst=True)
subst._add_('config', stimela.CONFIG, nosubst=True)
subst._add_('current', SubstitutionNS(**params))
# create step logger manually, since we won't be doing the normal recipe-level log management
step_logger = stimela.logger().getChild(cab_name)
Expand All @@ -361,7 +363,7 @@ def log_available_runnables():
for name in outer_step.missing_params:
missing[name] = outer_step.inputs_outputs[name].info
# don't report unresolved implicits, since that's just a consequence of a missing input
for name in outer_step.unresolved_params:
for name in outer_step.unresolved_params:
if not outer_step.inputs_outputs[name].implicit:
missing[name] = outer_step.inputs_outputs[name].info
#
Expand Down Expand Up @@ -393,8 +395,8 @@ def log_available_runnables():
log_exception(RecipeValidationError(f"error validating recipe '{recipe_name}'", exc))
for line in traceback.format_exc().split("\n"):
log.debug(line)
sys.exit(1)
sys.exit(1)

for key, value in params.items():
try:
recipe.assign_value(key, value, override=True)
Expand All @@ -414,14 +416,14 @@ def log_available_runnables():
log_exception(RecipeValidationError(f"pre-validation of recipe '{recipe_name}' failed", exc))
for line in traceback.format_exc().split("\n"):
log.debug(line)
sys.exit(1)
sys.exit(1)

# select recipe substeps based on command line, and exit if nothing to run
if not build_skips:
if not build_skips:
selection_options = []
for opts in (tags, skip_tags, step_ranges, skip_ranges, enable_steps):
selection_options.append(set(itertools.chain(*(opt.split(",") for opt in opts))))

try:
if not recipe.restrict_steps(*selection_options):
sys.exit(0)
Expand All @@ -445,6 +447,11 @@ def log_available_runnables():
for line in outer_step.summary(params=params):
log.debug(line)

if dump_config:
filename = os.path.join(logdir, "stimela.recipe.config.yaml")
log.info(f"recipe config will be saved under {filename}")
OmegaConf.save(stimela.CONFIG, f=filename)

if dry_run:
log.info("dry run was requested, exiting")
sys.exit(0)
Expand All @@ -461,7 +468,7 @@ def elapsed():
stimela.backends.close_backends(log)

if not isinstance(exc, ScabhaBaseException) or not exc.logged:
log_exception(StimelaRuntimeError(f"build failed after {elapsed()}", exc,
log_exception(StimelaRuntimeError(f"build failed after {elapsed()}", exc,
tb=not isinstance(exc, ScabhaBaseException)))
else:
log.error("build failed, exiting with error code 1")
Expand All @@ -478,11 +485,11 @@ def elapsed():
except Exception as exc:
stimela.backends.close_backends(log)

task_stats.save_profiling_stats(outer_step.log,
task_stats.save_profiling_stats(outer_step.log,
print_depth=profile if profile is not None else stimela.CONFIG.opts.profile.print_depth,
unroll_loops=stimela.CONFIG.opts.profile.unroll_loops)
if not isinstance(exc, ScabhaBaseException) or not exc.logged:
log_exception(StimelaRuntimeError(f"run failed after {elapsed()}", exc,
log_exception(StimelaRuntimeError(f"run failed after {elapsed()}", exc,
tb=not isinstance(exc, ScabhaBaseException)))
else:
log.error("run failed, exiting with error code 1")
Expand All @@ -503,11 +510,10 @@ def elapsed():
stimela.backends.close_backends(log)

if not build:
task_stats.save_profiling_stats(outer_step.log,
task_stats.save_profiling_stats(outer_step.log,
print_depth=profile if profile is not None else stimela.CONFIG.opts.profile.print_depth,
unroll_loops=stimela.CONFIG.opts.profile.unroll_loops)

last_log_dir = stimelogging.get_logfile_dir(outer_step.log) or '.'
outer_step.log.info(f"last log directory was {stimelogging.apply_style(last_log_dir, 'bold green')}")
return 0

0 comments on commit 077b2e4

Please sign in to comment.