Skip to content

Commit

Permalink
Fix configs to allow the use of configs as keys of other configs (#2206)
Browse files Browse the repository at this point in the history
* Fix configs to allow the use of configs as keys of other configs

This allows things like this:

```
def git_info(args):
    info = {
        "commit": ["git", "rev-parse", "HEAD"],
        "branch": ["git", "rev-parse", "--abbrev-ref", "HEAD"],
        "message": ["git", "log", "-1", "--pretty=%B"],
    }
    cfg = {}
    for key, cmd in info.items():
        cfg[key] = check_output(cmd, text=True).strip()
    if cfg["branch"].startswith("feature/"):
        cfg["branch_type"] = "feature"
    elif cfg["branch"].startswith("main"):
        cfg["branch_type"] = "main"
    else:
        cfg["branch_type"] = "default"]
    print(f"cfg is {cfg}")
    return cfg

class MyExampleCICDFlow(FlowSpec):
    git_config = Config("git_config", default_value={}, parser=git_info)
    flow_config = Config("flow_config", default="my_config.json")

    @timeout(seconds=flow_config[git_config.branch_type].timeout)
    @step
    def start(self):
        # More code
```

* Fix error message that referred to non-existing object
  • Loading branch information
romain-intel authored Jan 13, 2025
1 parent 96f5271 commit 49a073e
Showing 1 changed file with 16 additions and 5 deletions.
21 changes: 16 additions & 5 deletions metaflow/user_configs/config_parameters.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ def __iter__(self):
yield "%s%d" % (UNPACK_KEY, id(self))

def __getitem__(self, key):
if key == "%s%d" % (UNPACK_KEY, id(self)):
if isinstance(key, str) and key == "%s%d" % (UNPACK_KEY, id(self)):
return self
if self._access is None:
raise KeyError(key)
Expand All @@ -196,12 +196,23 @@ def __call__(self, ctx=None, deploy_time=False):
if flow_cls is None:
# We are not executing inside a flow (ie: not the CLI)
raise MetaflowException(
"Config object can only be used directly in the FlowSpec defining them. "
"If using outside of the FlowSpec, please use ConfigEval"
"Config object can only be used directly in the FlowSpec defining them "
"(or their flow decorators)."
)
if self._access is not None:
# Build the final expression by adding all the fields in access as . fields
self._config_expr = ".".join([self._config_expr] + self._access)
access_list = [self._config_expr]
for a in self._access:
if isinstance(a, str):
access_list.append(a)
elif isinstance(a, DelayEvaluator):
# Supports things like config[other_config.selector].var
access_list.append(a())
else:
raise MetaflowException(
"Field '%s' of type '%s' is not supported" % (str(a), type(a))
)
self._config_expr = ".".join(access_list)
# Evaluate the expression setting the config values as local variables
try:
return eval(
Expand Down Expand Up @@ -369,7 +380,7 @@ def __len__(self):

def __getitem__(self, key):
self._init_delayed_evaluator()
if key.startswith(UNPACK_KEY):
if isinstance(key, str) and key.startswith(UNPACK_KEY):
return self._delayed_evaluator[key]
return DelayEvaluator(self.name.lower())[key]

Expand Down

0 comments on commit 49a073e

Please sign in to comment.