From 49a073e3a7f851eac84b59dece64865acf24182d Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 13 Jan 2025 10:44:58 -0800 Subject: [PATCH] Fix configs to allow the use of configs as keys of other configs (#2206) * Fix configs to allow the use of configs as keys of other configs This allows things like this: ``` def git_info(args): info = { "commit": ["git", "rev-parse", "HEAD"], "branch": ["git", "rev-parse", "--abbrev-ref", "HEAD"], "message": ["git", "log", "-1", "--pretty=%B"], } cfg = {} for key, cmd in info.items(): cfg[key] = check_output(cmd, text=True).strip() if cfg["branch"].startswith("feature/"): cfg["branch_type"] = "feature" elif cfg["branch"].startswith("main"): cfg["branch_type"] = "main" else: cfg["branch_type"] = "default"] print(f"cfg is {cfg}") return cfg class MyExampleCICDFlow(FlowSpec): git_config = Config("git_config", default_value={}, parser=git_info) flow_config = Config("flow_config", default="my_config.json") @timeout(seconds=flow_config[git_config.branch_type].timeout) @step def start(self): # More code ``` * Fix error message that referred to non-existing object --- metaflow/user_configs/config_parameters.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/metaflow/user_configs/config_parameters.py b/metaflow/user_configs/config_parameters.py index c43a3092237..8adb563a3b6 100644 --- a/metaflow/user_configs/config_parameters.py +++ b/metaflow/user_configs/config_parameters.py @@ -171,7 +171,7 @@ def __iter__(self): yield "%s%d" % (UNPACK_KEY, id(self)) def __getitem__(self, key): - if key == "%s%d" % (UNPACK_KEY, id(self)): + if isinstance(key, str) and key == "%s%d" % (UNPACK_KEY, id(self)): return self if self._access is None: raise KeyError(key) @@ -196,12 +196,23 @@ def __call__(self, ctx=None, deploy_time=False): if flow_cls is None: # We are not executing inside a flow (ie: not the CLI) raise MetaflowException( - "Config object can only be used directly in the FlowSpec defining them. " - "If using outside of the FlowSpec, please use ConfigEval" + "Config object can only be used directly in the FlowSpec defining them " + "(or their flow decorators)." ) if self._access is not None: # Build the final expression by adding all the fields in access as . fields - self._config_expr = ".".join([self._config_expr] + self._access) + access_list = [self._config_expr] + for a in self._access: + if isinstance(a, str): + access_list.append(a) + elif isinstance(a, DelayEvaluator): + # Supports things like config[other_config.selector].var + access_list.append(a()) + else: + raise MetaflowException( + "Field '%s' of type '%s' is not supported" % (str(a), type(a)) + ) + self._config_expr = ".".join(access_list) # Evaluate the expression setting the config values as local variables try: return eval( @@ -369,7 +380,7 @@ def __len__(self): def __getitem__(self, key): self._init_delayed_evaluator() - if key.startswith(UNPACK_KEY): + if isinstance(key, str) and key.startswith(UNPACK_KEY): return self._delayed_evaluator[key] return DelayEvaluator(self.name.lower())[key]