From 503562812b28f63a2aab39b5f314236f580f19da Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 18 Aug 2024 08:47:39 -0700 Subject: [PATCH] Fix an issue with _graph_info (#1966) _graph_info could contain non POD which broke things particularly in the presence of extensions --- metaflow/flowspec.py | 3 ++- metaflow/graph.py | 7 ++++--- metaflow/util.py | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index 505376ce349..6f277f4f1a9 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -17,6 +17,7 @@ ) from .graph import FlowGraph from .unbounded_foreach import UnboundedForeachInput +from .util import to_pod from .metaflow_config import INCLUDE_FOREACH_STACK, MAXIMUM_FOREACH_VALUE_CHARS # For Python 3 compatibility @@ -201,7 +202,7 @@ def _set_constants(self, graph, kwargs): "decorators": [ { "name": deco.name, - "attributes": deco.attributes, + "attributes": to_pod(deco.attributes), "statically_defined": deco.statically_defined, } for deco in flow_decorators(self) diff --git a/metaflow/graph.py b/metaflow/graph.py index 4ea41e0114f..153ebfe740a 100644 --- a/metaflow/graph.py +++ b/metaflow/graph.py @@ -3,6 +3,9 @@ import re +from .util import to_pod + + def deindent_docstring(doc): if doc: # Find the indent to remove from the docstring. We consider the following possibilities: @@ -72,7 +75,6 @@ def _expr_str(self, expr): return "%s.%s" % (expr.value.id, expr.attr) def _parse(self, func_ast): - self.num_args = len(func_ast.args.args) tail = func_ast.body[-1] @@ -262,7 +264,6 @@ def node_specs(): ) def output_steps(self): - steps_info = {} graph_structure = [] @@ -286,7 +287,7 @@ def node_to_dict(name, node): "decorators": [ { "name": deco.name, - "attributes": deco.attributes, + "attributes": to_pod(deco.attributes), "statically_defined": deco.statically_defined, } for deco in node.decorators diff --git a/metaflow/util.py b/metaflow/util.py index aa03825e4ad..e95ab9f0f87 100644 --- a/metaflow/util.py +++ b/metaflow/util.py @@ -426,6 +426,25 @@ def is_within_directory(abs_directory, target): tar.extractall(path, members, numeric_owner=numeric_owner) +def to_pod(value): + """ + Convert a python object to plain-old-data (POD) format. + + Parameters + ---------- + value : Any + Value to convert to POD format. The value can be a string, number, list, + dictionary, or a nested structure of these types. + """ + if isinstance(value, (str, int, float)): + return value + if isinstance(value, dict): + return {to_pod(k): to_pod(v) for k, v in value.items()} + if isinstance(value, (list, set, tuple)): + return [to_pod(v) for v in value] + return str(value) + + if sys.version_info[:2] > (3, 5): from metaflow._vendor.packaging.version import parse as version_parse else: