diff --git a/.binder/environment.yml b/.binder/environment.yml index c5f0618b..ef50142f 100644 --- a/.binder/environment.yml +++ b/.binder/environment.yml @@ -10,9 +10,9 @@ dependencies: - h5io =0.2.4 - h5io_browser =0.0.16 - pandas =2.2.2 -- pyiron_base =0.9.10 -- pyiron_contrib =0.1.17 -- pyiron_snippets =0.1.3 +- pyiron_base =0.9.12 +- pyiron_contrib =0.1.18 +- pyiron_snippets =0.1.4 - python-graphviz =0.20.3 - toposort =1.10 - typeguard =4.3.0 diff --git a/.ci_support/environment.yml b/.ci_support/environment.yml index c5f0618b..ef50142f 100644 --- a/.ci_support/environment.yml +++ b/.ci_support/environment.yml @@ -10,9 +10,9 @@ dependencies: - h5io =0.2.4 - h5io_browser =0.0.16 - pandas =2.2.2 -- pyiron_base =0.9.10 -- pyiron_contrib =0.1.17 -- pyiron_snippets =0.1.3 +- pyiron_base =0.9.12 +- pyiron_contrib =0.1.18 +- pyiron_snippets =0.1.4 - python-graphviz =0.20.3 - toposort =1.10 - typeguard =4.3.0 diff --git a/.ci_support/lower_bound.yml b/.ci_support/lower_bound.yml index b6b04d89..456eb3fa 100644 --- a/.ci_support/lower_bound.yml +++ b/.ci_support/lower_bound.yml @@ -8,11 +8,11 @@ dependencies: - executorlib =0.0.1 - graphviz =9.0.0 - h5io =0.2.2 -- h5io_browser =0.0.12 +- h5io_browser =0.0.14 - pandas =2.2.0 -- pyiron_base =0.8.3 -- pyiron_contrib =0.1.16 -- pyiron_snippets =0.1.0 +- pyiron_base =0.9.12 +- pyiron_contrib =0.1.18 +- pyiron_snippets =0.1.4 - python-graphviz =0.20.0 - toposort =1.10 - typeguard =4.2.0 diff --git a/docs/environment.yml b/docs/environment.yml index fdd9c94f..174d29b3 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -16,9 +16,9 @@ dependencies: - h5io =0.2.4 - h5io_browser =0.0.16 - pandas =2.2.2 -- pyiron_base =0.9.10 -- pyiron_contrib =0.1.17 -- pyiron_snippets =0.1.3 +- pyiron_base =0.9.12 +- pyiron_contrib =0.1.18 +- pyiron_snippets =0.1.4 - python-graphviz =0.20.3 - toposort =1.10 - typeguard =4.3.0 diff --git a/notebooks/deepdive.ipynb b/notebooks/deepdive.ipynb index dfb381d8..856d74b9 100644 --- a/notebooks/deepdive.ipynb +++ b/notebooks/deepdive.ipynb @@ -5521,15 +5521,18 @@ "- Also related, there is currently zero filtering of which data, or to which depth the graph gets stored -- it's all or nothing\n", "- If the source code for nodes gets modified between saving and loading, weird stuff is likely to happen, and some of it may happen silently.\n", "\n", - "Lastly, we currently use two backends: `tinybase.storage.H5ioStorage` and `h5io` directly. They have slightly different strengths:\n", - "- `\"h5io\"` (the default) \n", + "Lastly, we currently use three backends: `pickle`, ``tinybase.storage.H5ioStorage` and `h5io` directly. They have slightly different strengths:\n", + "- `\"h5io\"` \n", " - Will let you save and load any nodes that defined by subclassing (this includes all nodes defined using the decorators)\n", " - Will preserve changes to a macro (replace/add/remove/rewire)\n", " - Has trouble with some data\n", "- `\"tinybase\"`\n", " - Requires all nodes to have been instantiated with the creator (`wf.create...`; this means moving node definitions to a `.py` file in your pythonpath and registering it as a node package -- not particularly difficult!)\n", " - _Ignores_ changes to a macro (will crash nicely if the macro IO changed)\n", - " - Falls back to `pickle` for data failures, so can handle a wider variety of data IO objects" + " - Falls back to `pickle` for data failures, so can handle a wider variety of data IO objects\n", + "- `\"pickle\"`\n", + " - Can't handle unpickleable IO items, if that's a problem for your data\n", + " - Stored in byte format; not browsable and needs to be loaded to inspect it at all" ] }, { diff --git a/pyiron_workflow/__init__.py b/pyiron_workflow/__init__.py index 43b6030f..ab992d40 100644 --- a/pyiron_workflow/__init__.py +++ b/pyiron_workflow/__init__.py @@ -46,7 +46,7 @@ from pyiron_workflow.logging import logger from pyiron_workflow.nodes.macro import Macro, as_macro_node, macro_node from pyiron_workflow.nodes.transform import ( - # as_dataclass_node, # Not pickling nicely yet + as_dataclass_node, dataclass_node, inputs_to_dataframe, inputs_to_dict, diff --git a/pyiron_workflow/mixin/storage.py b/pyiron_workflow/mixin/storage.py index 19ac24b5..bfd2f086 100644 --- a/pyiron_workflow/mixin/storage.py +++ b/pyiron_workflow/mixin/storage.py @@ -8,9 +8,11 @@ from abc import ABC, abstractmethod from importlib import import_module import os +import pickle import sys from typing import Optional +import cloudpickle import h5io from pyiron_snippets.files import FileObject, DirectoryObject @@ -84,6 +86,76 @@ def _delete(self): """Remove an existing save-file for this backend""" +class PickleStorage(StorageInterface): + + _PICKLE_STORAGE_FILE_NAME = "pickle.pckl" + _CLOUDPICKLE_STORAGE_FILE_NAME = "cloudpickle.cpckl" + + def __init__(self, owner: HasPickleStorage): + super().__init__(owner=owner) + + @property + def owner(self) -> HasPickleStorage: + return self._owner + + def _save(self): + try: + with open(self._pickle_storage_file_path, "wb") as file: + pickle.dump(self.owner, file) + except Exception: + self._delete() + with open(self._cloudpickle_storage_file_path, "wb") as file: + cloudpickle.dump(self.owner, file) + + def _load(self): + if self._has_pickle_contents: + with open(self._pickle_storage_file_path, "rb") as file: + inst = pickle.load(file) + elif self._has_cloudpickle_contents: + with open(self._cloudpickle_storage_file_path, "rb") as file: + inst = cloudpickle.load(file) + + if inst.__class__ != self.owner.__class__: + raise TypeError( + f"{self.owner.label} cannot load, as it has type " + f"{self.owner.__class__.__name__}, but the saved node has type " + f"{inst.__class__.__name__}" + ) + self.owner.__setstate__(inst.__getstate__()) + + def _delete_file(self, file: str): + FileObject(file, self.owner.storage_directory).delete() + + def _delete(self): + if self._has_pickle_contents: + self._delete_file(self._PICKLE_STORAGE_FILE_NAME) + elif self._has_cloudpickle_contents: + self._delete_file(self._CLOUDPICKLE_STORAGE_FILE_NAME) + + def _storage_path(self, file: str): + return str((self.owner.storage_directory.path / file).resolve()) + + @property + def _pickle_storage_file_path(self) -> str: + return self._storage_path(self._PICKLE_STORAGE_FILE_NAME) + + @property + def _cloudpickle_storage_file_path(self) -> str: + return self._storage_path(self._CLOUDPICKLE_STORAGE_FILE_NAME) + + @property + def _has_contents(self) -> bool: + return self._has_pickle_contents or self._has_cloudpickle_contents + + @property + def _has_pickle_contents(self) -> bool: + return os.path.isfile(self._pickle_storage_file_path) + + @property + def _has_cloudpickle_contents(self) -> bool: + return os.path.isfile(self._cloudpickle_storage_file_path) + + class H5ioStorage(StorageInterface): _H5IO_STORAGE_FILE_NAME = "h5io.h5" @@ -278,7 +350,15 @@ def load(self): Raises: TypeError: when the saved node has a different class name. """ - self.storage.load() + if self.storage.has_contents: + self.storage.load() + else: + # Check for saved content using any other backend + for backend in self.allowed_backends(): + interface = self._storage_interfaces()[backend](self) + if interface.has_contents: + interface.load() + break save.__doc__ += _save_load_warnings @@ -333,6 +413,13 @@ def storage(self) -> StorageInterface: raise ValueError(f"{self.label} does not have a storage backend set") return self._storage_interfaces()[self.storage_backend](self) + @property + def any_storage_has_contents(self): + return any( + self._storage_interfaces()[backend](self).has_contents + for backend in self.allowed_backends() + ) + @property def import_ready(self) -> bool: """ @@ -365,6 +452,18 @@ def report_import_readiness(self, tabs=0, report_so_far=""): ) +class HasPickleStorage(HasStorage, ABC): + @classmethod + def _storage_interfaces(cls): + interfaces = super(HasPickleStorage, cls)._storage_interfaces() + interfaces["pickle"] = PickleStorage + return interfaces + + @classmethod + def default_backend(cls): + return "pickle" + + class HasH5ioStorage(HasStorage, ABC): @classmethod def _storage_interfaces(cls): @@ -387,7 +486,3 @@ def to_storage(self, storage: TinybaseStorage): @abstractmethod def from_storage(self, storage: TinybaseStorage): pass - - @classmethod - def default_backend(cls): - return "tinybase" diff --git a/pyiron_workflow/node.py b/pyiron_workflow/node.py index e6a0f441..b213cf06 100644 --- a/pyiron_workflow/node.py +++ b/pyiron_workflow/node.py @@ -22,7 +22,11 @@ from pyiron_workflow.mixin.run import Runnable, ReadinessError from pyiron_workflow.mixin.semantics import Semantic from pyiron_workflow.mixin.single_output import ExploitsSingleOutput -from pyiron_workflow.mixin.storage import HasH5ioStorage, HasTinybaseStorage +from pyiron_workflow.mixin.storage import ( + HasH5ioStorage, + HasTinybaseStorage, + HasPickleStorage, +) from pyiron_workflow.topology import ( get_nodes_in_data_tree, set_run_connections_according_to_linear_dag, @@ -48,6 +52,7 @@ class Node( HasWorkingDirectory, HasH5ioStorage, HasTinybaseStorage, + HasPickleStorage, ABC, ): """ @@ -138,7 +143,8 @@ class Node( - As long as you haven't put anything unpickleable on them, or defined them in an unpicklable place (e.g. in the `` of another function), you can simple (un)pickle nodes. There is no save/load interface for this right - now, just import pickle and do it. + now, just import pickle and do it. The "pickle" backend to the `Node.save` + method will fall back on `cloudpickle` as needed to overcome this. - Saving is triggered manually, or by setting a flag to save after the nodes runs. - At the end of instantiation, nodes will load automatically if they find saved @@ -160,11 +166,11 @@ class Node( your graph this could be expensive in terms of storage space and/or time. - [ALPHA ISSUE] Similarly, there is no way to save only part of a graph; only the entire graph may be saved at once. - - [ALPHA ISSUE] There are two possible back-ends for saving: one leaning on + - [ALPHA ISSUE] There are three possible back-ends for saving: one leaning on `tinybase.storage.GenericStorage` (in practice, - `H5ioStorage(GenericStorage)`), that is the default, and the other that - uses the `h5io` module directly. The backend used is always the one on the - graph root. + `H5ioStorage(GenericStorage)`), and the other that uses the `h5io` module + directly. The third (default) option is to use `(cloud)pickle`. The backend + used is always the one on the graph root. - [ALPHA ISSUE] The `h5io` backend is deprecated -- it can't handle custom reconstructors (i.e. when `__reduce__` returns a tuple with some non-standard callable as its first entry), and basically all our nodes do @@ -191,8 +197,9 @@ class Node( requirement is as simple as moving all the desired nodes off to a `.py` file, registering it, and building the composite from there. - [ALPHA ISSUE] Restrictions to macros: - - For the `h5io` backend: there are none; if a macro is modified, saved, - and reloaded, the modifications will be reflected in the loaded state. + - For the `h5io` and `pickle` backends: there are none; if a macro is + modified, saved, and reloaded, the modifications will be reflected in + the loaded state. Note there is a little bit of danger here, as the macro class still corresponds to the un-modified macro class. - For the `tinybase` backend: the macro will re-instantiate its original @@ -251,9 +258,9 @@ class Node( received output from this call. (Default is False.) save_after_run (bool): Whether to trigger a save after each run of the node (currently causes the entire graph to save). (Default is False.) - storage_backend (Literal["h5io" | "tinybase"] | None): The flag for the the - backend to use for saving and loading; for nodes in a graph the value on - the root node is always used. + storage_backend (Literal["h5io" | "tinybase", "pickle"] | None): The flag for + the backend to use for saving and loading; for nodes in a graph the value + on the root node is always used. signals (pyiron_workflow.io.Signals): A container for input and output signals, which are channels for controlling execution flow. By default, has a :attr:`signals.inputs.run` channel which has a callback to the :meth:`run` method @@ -324,7 +331,7 @@ def __init__( parent: Optional[Composite] = None, overwrite_save: bool = False, run_after_init: bool = False, - storage_backend: Literal["h5io", "tinybase"] | None = "h5io", + storage_backend: Literal["h5io", "tinybase", "pickle"] | None = "pickle", save_after_run: bool = False, **kwargs, ): @@ -377,7 +384,7 @@ def _after_node_setup( self.delete_storage() do_load = False else: - do_load = sys.version_info >= (3, 11) and self.storage.has_contents + do_load = sys.version_info >= (3, 11) and self.any_storage_has_contents if do_load and run_after_init: raise ValueError( diff --git a/pyiron_workflow/nodes/composite.py b/pyiron_workflow/nodes/composite.py index d82d39fd..5271486d 100644 --- a/pyiron_workflow/nodes/composite.py +++ b/pyiron_workflow/nodes/composite.py @@ -102,7 +102,7 @@ def __init__( parent: Optional[Composite] = None, overwrite_save: bool = False, run_after_init: bool = False, - storage_backend: Optional[Literal["h5io", "tinybase"]] = None, + storage_backend: Optional[Literal["h5io", "tinybase", "pickle"]] = None, save_after_run: bool = False, strict_naming: bool = True, **kwargs, diff --git a/pyiron_workflow/nodes/for_loop.py b/pyiron_workflow/nodes/for_loop.py index 3b4f3c55..fab04f57 100644 --- a/pyiron_workflow/nodes/for_loop.py +++ b/pyiron_workflow/nodes/for_loop.py @@ -201,7 +201,7 @@ def __init__( parent: Optional[Composite] = None, overwrite_save: bool = False, run_after_init: bool = False, - storage_backend: Optional[Literal["h5io", "tinybase"]] = None, + storage_backend: Optional[Literal["h5io", "tinybase", "pickle"]] = None, save_after_run: bool = False, strict_naming: bool = True, body_node_executor: Optional[Executor] = None, diff --git a/pyiron_workflow/nodes/function.py b/pyiron_workflow/nodes/function.py index e1302a78..96d9ac57 100644 --- a/pyiron_workflow/nodes/function.py +++ b/pyiron_workflow/nodes/function.py @@ -421,7 +421,10 @@ def decorator(node_function): factory_made = function_node_factory( node_function, validate_output_labels, use_cache, *output_labels ) - factory_made._class_returns_from_decorated_function = node_function + factory_made._reduce_imports_as = ( + node_function.__module__, + node_function.__qualname__, + ) factory_made.preview_io() return factory_made diff --git a/pyiron_workflow/nodes/macro.py b/pyiron_workflow/nodes/macro.py index 97707192..36886f78 100644 --- a/pyiron_workflow/nodes/macro.py +++ b/pyiron_workflow/nodes/macro.py @@ -302,7 +302,7 @@ def _scrape_output_labels(cls): return scraped_labels def _prepopulate_ui_nodes_from_graph_creator_signature( - self, storage_backend: Literal["h5io", "tinybase"] + self, storage_backend: Literal["h5io", "tinybase", "pickle"] ): ui_nodes = [] for label, (type_hint, default) in self.preview_inputs().items(): @@ -534,7 +534,10 @@ def decorator(graph_creator): factory_made = macro_node_factory( graph_creator, validate_output_labels, use_cache, *output_labels ) - factory_made._class_returns_from_decorated_function = graph_creator + factory_made._reduce_imports_as = ( + graph_creator.__module__, + graph_creator.__qualname__, + ) factory_made.preview_io() return factory_made diff --git a/pyiron_workflow/nodes/static_io.py b/pyiron_workflow/nodes/static_io.py index 11373960..66cbe7e0 100644 --- a/pyiron_workflow/nodes/static_io.py +++ b/pyiron_workflow/nodes/static_io.py @@ -32,7 +32,7 @@ def __init__( parent: Optional[Composite] = None, overwrite_save: bool = False, run_after_init: bool = False, - storage_backend: Optional[Literal["h5io", "tinybase"]] = None, + storage_backend: Optional[Literal["h5io", "tinybase", "pickle"]] = None, save_after_run: bool = False, **kwargs, ): diff --git a/pyiron_workflow/nodes/transform.py b/pyiron_workflow/nodes/transform.py index 7f8482bc..4c3a7ac3 100644 --- a/pyiron_workflow/nodes/transform.py +++ b/pyiron_workflow/nodes/transform.py @@ -387,11 +387,15 @@ def dataclass_node_factory( ) if not is_dataclass(dataclass): dataclass = as_dataclass(dataclass) + module, qualname = dataclass.__module__, dataclass.__qualname__ + dataclass.__qualname__ += ".dataclass" # So output type hints know where to find it return ( - f"{DataclassNode.__name__}{dataclass.__name__}", + dataclass.__name__, (DataclassNode,), { "dataclass": dataclass, + "__module__": module, + "__qualname__": qualname, "_output_type_hint": dataclass, "__doc__": dataclass.__doc__, "use_cache": use_cache, @@ -400,7 +404,7 @@ def dataclass_node_factory( ) -def as_dataclass_node(dataclass: type, use_cache: bool = True): +def as_dataclass_node(dataclass: type): """ Decorates a dataclass as a dataclass node -- i.e. a node whose inputs correspond to dataclass fields and whose output is an instance of the dataclass. @@ -439,7 +443,7 @@ def as_dataclass_node(dataclass: type, use_cache: bool = True): >>> >>> f = Foo() >>> print(f.readiness_report) - DataclassNodeFoo readiness: False + Foo readiness: False STATE: running: False failed: False @@ -450,9 +454,12 @@ def as_dataclass_node(dataclass: type, use_cache: bool = True): complex_ ready: True >>> f(necessary="input as a node kwarg") - Foo(necessary='input as a node kwarg', bar='bar', answer=42, complex_=[1, 2, 3]) + Foo.dataclass(necessary='input as a node kwarg', bar='bar', answer=42, complex_=[1, 2, 3]) """ - cls = dataclass_node_factory(dataclass, use_cache) + dataclass_node_factory.clear(dataclass.__name__) # Force a fresh class + module, qualname = dataclass.__module__, dataclass.__qualname__ + cls = dataclass_node_factory(dataclass) + cls._reduce_imports_as = (module, qualname) cls.preview_io() return cls @@ -509,7 +516,7 @@ def dataclass_node(dataclass: type, use_cache: bool = True, *node_args, **node_k complex_ ready: True >>> f(necessary="input as a node kwarg") - Foo(necessary='input as a node kwarg', bar='bar', answer=42, complex_=[1, 2, 3]) + Foo.dataclass(necessary='input as a node kwarg', bar='bar', answer=42, complex_=[1, 2, 3]) """ cls = dataclass_node_factory(dataclass) cls.preview_io() diff --git a/pyiron_workflow/workflow.py b/pyiron_workflow/workflow.py index 5f80d851..95d831bf 100644 --- a/pyiron_workflow/workflow.py +++ b/pyiron_workflow/workflow.py @@ -204,7 +204,7 @@ def __init__( *nodes: Node, overwrite_save: bool = False, run_after_init: bool = False, - storage_backend: Optional[Literal["h5io", "tinybase"]] = None, + storage_backend: Optional[Literal["h5io", "tinybase", "pickle"]] = None, save_after_run: bool = False, strict_naming: bool = True, inputs_map: Optional[dict | bidict] = None, diff --git a/pyproject.toml b/pyproject.toml index 40ec2a0e..45b9ba9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,9 +40,9 @@ dependencies = [ "h5io==0.2.4", "h5io_browser==0.0.16", "pandas==2.2.2", - "pyiron_base==0.9.10", - "pyiron_contrib==0.1.17", - "pyiron_snippets==0.1.3", + "pyiron_base==0.9.12", + "pyiron_contrib==0.1.18", + "pyiron_snippets==0.1.4", "toposort==1.10", "typeguard==4.3.0", ] diff --git a/tests/unit/mixin/test_preview.py b/tests/unit/mixin/test_preview.py index 0c8ac5f2..f070749a 100644 --- a/tests/unit/mixin/test_preview.py +++ b/tests/unit/mixin/test_preview.py @@ -46,7 +46,7 @@ def scraper_decorator(fnc): factory_made = scraper_factory( fnc, validate_output_labels, io_defining_function_uses_self, *output_labels ) - factory_made._class_returns_from_decorated_function = fnc + factory_made._reduce_imports_as = (fnc.__module__, fnc.__qualname__) factory_made.preview_io() return factory_made return scraper_decorator diff --git a/tests/unit/nodes/test_macro.py b/tests/unit/nodes/test_macro.py index eb2145d2..8b9fee52 100644 --- a/tests/unit/nodes/test_macro.py +++ b/tests/unit/nodes/test_macro.py @@ -488,7 +488,6 @@ def test_storage_for_modified_macros(self): Macro.create.demo.AddPlusOne() ) - modified_result = macro() if backend == "h5io": @@ -496,7 +495,7 @@ def test_storage_for_modified_macros(self): TypeError, msg="h5io can't handle custom reconstructors" ): macro.save() - else: + elif backend in ["tinybase", "pickle"]: macro.save() reloaded = Macro.create.demo.AddThree( label="m", storage_backend=backend @@ -525,14 +524,36 @@ def test_storage_for_modified_macros(self): rerun = reloaded() if backend == "tinybase": + self.assertIsInstance( + reloaded.two, + Macro.create.standard.Add, + msg="tinybase is re-instantiating the original macro " + "class and then carefully loading particular " + "pieces of data; that means each child is" + ) self.assertDictEqual( original_result, rerun, - msg="Rerunning should re-execute the _original_ " + msg="Rerunning re-executes the _original_ " "functionality" ) - else: - raise ValueError(f"Unexpected backend {backend}?") + elif backend == "pickle": + self.assertIsInstance( + reloaded.two, + Macro.create.demo.AddPlusOne, + msg="pickle instantiates the macro node class, but " + "but then uses its serialized state, so we retain " + "the replaced node." + ) + self.assertDictEqual( + modified_result, + rerun, + msg="Rerunning re-executes the _replaced_ functionality" + ) + else: + raise ValueError( + f"Backend {backend} not recognized -- write a test for it" + ) finally: macro.storage.delete() diff --git a/tests/unit/nodes/test_transform.py b/tests/unit/nodes/test_transform.py index b2d9718b..04d44898 100644 --- a/tests/unit/nodes/test_transform.py +++ b/tests/unit/nodes/test_transform.py @@ -6,6 +6,7 @@ from pandas import DataFrame from pyiron_workflow.channels import NOT_DATA +from pyiron_workflow.nodes.function import as_function_node from pyiron_workflow.nodes.transform import ( Transformer, as_dataclass_node, @@ -16,6 +17,15 @@ list_to_outputs, ) +@as_dataclass_node +class MyData: + stuff: bool = False + +@as_function_node +def Downstream(x: MyData.dataclass): + x.stuff = True + return x + class TestTransformer(unittest.TestCase): def test_pickle(self): @@ -249,6 +259,35 @@ class DecoratedDCLike: "instantiation" ) + def test_dataclass_typing_and_storage(self): + md = MyData() + + with self.assertRaises( + TypeError, + msg="Wrongly typed input should not connect" + ): + Downstream(5) + + ds = Downstream(md) + out = ds.pull() + self.assertTrue( + out.stuff, + msg="Sanity check" + ) + + rmd = pickle.loads(pickle.dumps(md)) + self.assertIs( + rmd.outputs.dataclass.type_hint, + MyData.dataclass, + msg="Type hint should be findable on the scope of the node decorating it" + ) + ds2 = Downstream(rmd) + out = ds2.pull() + self.assertTrue( + out.stuff, + msg="Flow should be able to survive (de)serialization" + ) + if __name__ == '__main__': unittest.main() diff --git a/tests/unit/test_node.py b/tests/unit/test_node.py index 42ce5fab..17a92227 100644 --- a/tests/unit/test_node.py +++ b/tests/unit/test_node.py @@ -20,7 +20,7 @@ class ANode(Node): """To de-abstract the class""" def _setup_node(self) -> None: - self._inputs = Inputs(InputData("x", self, type_hint=int)) + self._inputs = Inputs(InputData("x", self, type_hint=int),) self._outputs = OutputsWithInjection( OutputDataWithInjection("y", self, type_hint=int), ) @@ -472,9 +472,68 @@ def test_storage(self): force_run.outputs.y.value, msg="Destroying the save should allow immediate re-running" ) + + hard_input = ANode(label="hard", storage_backend=backend) + hard_input.inputs.x.type_hint = callable + hard_input.inputs.x = lambda x: x * 2 + if backend == "pickle": + hard_input.save() + reloaded = ANode( + label=hard_input.label, + storage_backend=backend + ) + self.assertEqual( + reloaded.inputs.x.value(4), + hard_input.inputs.x.value(4), + msg="Cloud pickle should be strong enough to recover this" + ) + else: + with self.assertRaises( + (TypeError, AttributeError), + msg="Other backends are not powerful enough for some values" + ): + hard_input.save() finally: + hard_input.delete_storage() self.n1.delete_storage() + @unittest.skipIf(sys.version_info < (3, 11), "Storage will only work in 3.11+") + def test_storage_compatibility(self): + try: + self.n1.storage_backend = "tinybase" + self.n1.inputs.x = 42 + self.n1.save() + new_n1 = ANode(label=self.n1.label, storage_backend="pickle") + self.assertEqual( + new_n1.inputs.x.value, + self.n1.inputs.x.value, + msg="Even though the new node has a different storage backend, it " + "should still _load_ the data saved with a different backend. To " + "really avoid loading, delete or move the existing save file, or " + "give your new node a different label." + ) + new_n1() + new_n1.save() # With a different backend now + + tiny_n1 = ANode(label=self.n1.label, storage_backend="tinybase") + self.assertIs( + tiny_n1.outputs.y.value, + NOT_DATA, + msg="By explicitly specifying a particular backend, we expect to " + "recover that backend's save-file, even if it is outdated" + ) + + pick_n1 = ANode(label=self.n1.label, storage_backend="pickle") + self.assertEqual( + pick_n1.outputs.y.value, + new_n1.outputs.y.value, + msg="If we specify the more-recently-saved backend, we expect to load " + "the corresponding save file, where output exists" + ) + finally: + self.n1.delete_storage() + + @unittest.skipIf(sys.version_info < (3, 11), "Storage will only work in 3.11+") def test_save_after_run(self): for backend in Node.allowed_backends(): diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index 649ae46c..3d2a6ebf 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -488,8 +488,8 @@ def test_storage_scopes(self): for backend in Workflow.allowed_backends(): with self.subTest(backend): - try: - for backend in Workflow.allowed_backends(): + for backend in Workflow.allowed_backends(): + try: if backend == "h5io": with self.subTest(backend): with self.assertRaises( @@ -503,24 +503,24 @@ def test_storage_scopes(self): wf.storage_backend = backend wf.save() Workflow(wf.label, storage_backend=backend) - finally: - wf.storage.delete() + finally: + wf.storage.delete() with self.subTest("No unimportable nodes for either back-end"): - try: - wf.import_type_mismatch = wf.create.demo.dynamic() - for backend in Workflow.allowed_backends(): + for backend in Workflow.allowed_backends(): + try: + wf.import_type_mismatch = wf.create.demo.dynamic() with self.subTest(backend): - with self.assertRaises( - TypeNotFoundError, - msg="Imported object is function but node type is node -- " - "should fail early on save" - ): - wf.storage_backend = backend - wf.save() - finally: - wf.remove_child(wf.import_type_mismatch) - wf.storage.delete() + with self.assertRaises( + TypeNotFoundError, + msg="Imported object is function but node type is node " + "-- should fail early on save" + ): + wf.storage_backend = backend + wf.save() + finally: + wf.remove_child(wf.import_type_mismatch) + wf.storage.delete() if "h5io" in Workflow.allowed_backends(): wf.add_child(PlusOne(label="local_but_importable"))