From 03b77dd830b223aeb6b6ee31607ce4b2f6788791 Mon Sep 17 00:00:00 2001 From: Ryan Roussel Date: Mon, 23 Oct 2023 16:01:39 -0500 Subject: [PATCH 01/10] initial commit to bring pydantic to v2 --- environment.yml | 2 +- lume_model/base.py | 131 +++++++++++++++------ lume_model/torch/model.py | 37 +++--- lume_model/utils.py | 2 +- lume_model/variables.py | 80 ++----------- requirements.txt | 2 +- tests/conftest.py | 4 +- tests/keras/__init__.py | 0 tests/test_base.py | 33 +++++- tests/{keras => test_keras}/test_layers.py | 0 tests/{keras => test_keras}/test_model.py | 0 tests/{torch => test_torch}/test_model.py | 6 +- tests/{torch => test_torch}/test_module.py | 0 tests/test_variables.py | 12 +- tests/torch/__init__.py | 0 15 files changed, 170 insertions(+), 139 deletions(-) delete mode 100644 tests/keras/__init__.py rename tests/{keras => test_keras}/test_layers.py (100%) rename tests/{keras => test_keras}/test_model.py (100%) rename tests/{torch => test_torch}/test_model.py (98%) rename tests/{torch => test_torch}/test_module.py (100%) delete mode 100644 tests/torch/__init__.py diff --git a/environment.yml b/environment.yml index 5581ee9..895354a 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,6 @@ channels: - conda-forge dependencies: - python=3.9 - - pydantic==1.10.9 + - pydantic>2.3 - numpy - pyyaml diff --git a/lume_model/base.py b/lume_model/base.py index 5606818..62ea5f7 100644 --- a/lume_model/base.py +++ b/lume_model/base.py @@ -3,15 +3,15 @@ import yaml import logging from abc import ABC, abstractmethod -from typing import Any, Callable, Union +from typing import Any, Callable, Union, TextIO from types import FunctionType, MethodType import numpy as np -from pydantic import BaseModel, validator +from pydantic import BaseModel, ConfigDict, field_validator, SerializeAsAny from lume_model.variables import ( InputVariable, - OutputVariable, + OutputVariable, ScalarInputVariable, ScalarOutputVariable, ) from lume_model.utils import ( try_import_module, @@ -23,7 +23,6 @@ logger = logging.getLogger(__name__) - JSON_ENCODERS = { # function/method type distinguished for class members and not recognized as callables FunctionType: lambda x: f"{x.__module__}.{x.__qualname__}", @@ -96,7 +95,7 @@ def process_keras_model( def recursive_serialize( - v, + v: dict[str, Any], base_key: str = "", file_prefix: Union[str, os.PathLike] = "", save_models: bool = True, @@ -121,11 +120,13 @@ def recursive_serialize( if isinstance(value, dict): v[key] = recursive_serialize(value, key) elif torch is not None and isinstance(value, torch.nn.Module): - v[key] = process_torch_module(value, base_key, key, file_prefix, save_models) + v[key] = process_torch_module(value, base_key, key, file_prefix, + save_models) elif isinstance(value, list) and torch is not None and any( isinstance(ele, torch.nn.Module) for ele in value): v[key] = [ - process_torch_module(value[i], base_key, f"{key}_{i}", file_prefix, save_models) + process_torch_module(value[i], base_key, f"{key}_{i}", file_prefix, + save_models) for i in range(len(value)) ] elif keras is not None and isinstance(value, keras.Model): @@ -164,7 +165,6 @@ def recursive_deserialize(v): def json_dumps( v, *, - default, base_key="", file_prefix: Union[str, os.PathLike] = "", save_models: bool = True, @@ -181,8 +181,8 @@ def json_dumps( Returns: JSON formatted string. """ - v = recursive_serialize(v, base_key, file_prefix, save_models) - v = json.dumps(v, default=default) + v = recursive_serialize(v.model_dump(), base_key, file_prefix, save_models) + v = json.dumps(v) return v @@ -232,7 +232,8 @@ def model_kwargs_from_dict(config: dict) -> dict: """ config = deserialize_variables(config) if all(key in config.keys() for key in ["input_variables", "output_variables"]): - config["input_variables"], config["output_variables"] = variables_from_dict(config) + config["input_variables"], config["output_variables"] = variables_from_dict( + config) _ = config.pop("model_class", None) return config @@ -247,15 +248,44 @@ class LUMEBaseModel(BaseModel, ABC): input_variables: List defining the input variables and their order. output_variables: List defining the output variables and their order. """ - input_variables: list[InputVariable] - output_variables: list[OutputVariable] + input_variables: list[SerializeAsAny[InputVariable]] + output_variables: list[SerializeAsAny[OutputVariable]] + + model_config = ConfigDict(arbitrary_types_allowed=True, validate_assignment=True) - class Config: - extra = "allow" - json_dumps = json_dumps - json_loads = json_loads - validate_assignment = True - arbitrary_types_allowed = True + @field_validator("input_variables", mode="before") + def validate_input_variables(cls, value): + new_value = [] + if isinstance(value, dict): + for name, val in value.items(): + if isinstance(val, dict): + if val["variable_type"] == "scalar": + new_value.append(ScalarInputVariable(name=name, **val)) + elif isinstance(val, InputVariable): + new_value.append(val) + else: + raise TypeError(f"type {type(val)} not supported") + elif isinstance(value, list): + new_value = value + + return new_value + + @field_validator("output_variables", mode="before") + def validate_output_variables(cls, value): + new_value = [] + if isinstance(value, dict): + for name, val in value.items(): + if isinstance(val, dict): + if val["variable_type"] == "scalar": + new_value.append(ScalarOutputVariable(name=name, **val)) + elif isinstance(val, OutputVariable): + new_value.append(val) + else: + raise TypeError(f"type {type(val)} not supported") + elif isinstance(value, list): + new_value = value + + return new_value def __init__( self, @@ -274,7 +304,7 @@ def __init__( else: super().__init__(**kwargs) - @validator("input_variables", "output_variables") + @field_validator("input_variables", "output_variables") def unique_variable_names(cls, value): verify_unique_variable_names(value) return value @@ -291,12 +321,35 @@ def output_names(self) -> list[str]: def evaluate(self, input_dict: dict[str, Any]) -> dict[str, Any]: pass - def yaml( + def to_json(self, **kwargs) -> str: + return json_dumps(self, **kwargs) + + def dict(self, **kwargs) -> dict[str, Any]: + config = super().model_dump(**kwargs) + return {"model_class": self.__class__.__name__} | config + + def json(self, **kwargs) -> str: + result = self.to_json(**kwargs) + config = json.loads(result) + config = {"model_class": self.__class__.__name__} | config + + return json.dumps(config) + + def yaml(self, **kwargs): + """serialize first then dump to yaml string""" + output = json.loads( + self.to_json( + **kwargs, + ) + ) + return yaml.dump(output, default_flow_style=None, sort_keys=False) + + def dump( self, - file: Union[str, os.PathLike] = None, + file: Union[str, os.PathLike], save_models: bool = True, base_key: str = "", - ) -> str: + ): """Returns and optionally saves YAML formatted string defining the model. Args: @@ -307,13 +360,25 @@ def yaml( Returns: YAML formatted string defining the model. """ - file_prefix = "" - if file is not None: - file_prefix = os.path.splitext(file)[0] - config = json.loads(self.json(base_key=base_key, file_prefix=file_prefix, save_models=save_models)) - s = yaml.dump({"model_class": self.__class__.__name__} | config, - default_flow_style=None, sort_keys=False) - if file is not None: - with open(file, "w") as f: - f.write(s) - return s + file_prefix = os.path.splitext(file)[0] + + with open(file, "w") as f: + f.write(self.yaml( + base_key=base_key, + file_prefix=file_prefix, + save_models=save_models) + ) + + @classmethod + def from_file(cls, filename: str): + if not os.path.exists(filename): + raise OSError(f"file {filename} is not found") + + with open(filename, "r") as file: + return cls.from_yaml(file) + + @classmethod + def from_yaml(cls, yaml_obj: [str, TextIO]): + return cls.model_validate(yaml.safe_load(yaml_obj)) + + diff --git a/lume_model/torch/model.py b/lume_model/torch/model.py index b8dcbb7..7d91cfe 100644 --- a/lume_model/torch/model.py +++ b/lume_model/torch/model.py @@ -4,7 +4,8 @@ from copy import deepcopy import torch -from pydantic import validator +import yaml +from pydantic import validator, field_validator from botorch.models.transforms.input import ReversibleInputTransform from lume_model.base import LUMEBaseModel @@ -43,19 +44,21 @@ class TorchModel(LUMEBaseModel): device: Union[torch.device, str] = "cpu" fixed_model: bool = True - def __init__( - self, - config: Union[dict, str] = None, - **kwargs, - ): - """Initializes TorchModel. - - Args: - config: Model configuration as dictionary, YAML or JSON formatted string or file path. This overrides - all other arguments. - **kwargs: See class attributes. + def __init__(self, *args, **kwargs): + """ + Initialize Xopt. """ - super().__init__(config, **kwargs) + if len(args) == 1: + if len(kwargs) > 0: + raise ValueError("cannot specify yaml string and kwargs for Xopt init") + super().__init__(**yaml.safe_load(args[0])) + elif len(args) > 1: + raise ValueError( + "arguments to Xopt must be either a single yaml string " + "or a keyword arguments passed directly to pydantic" + ) + else: + super().__init__(**kwargs) # set precision self.model.to(dtype=self.dtype) @@ -81,14 +84,16 @@ def dtype(self): def _tkwargs(self): return {"device": self.device, "dtype": self.dtype} - @validator("model", pre=True) + @field_validator("model", mode="before") def validate_torch_model(cls, v): if isinstance(v, (str, os.PathLike)): if os.path.exists(v): v = torch.load(v) + else: + raise ValueError(f"path {v} does not exist!!") return v - @validator("input_transformers", "output_transformers", pre=True) + @field_validator("input_transformers", "output_transformers", mode="before") def validate_botorch_transformers(cls, v): if not isinstance(v, list): raise ValueError("Transformers must be passed as list.") @@ -102,7 +107,7 @@ def validate_botorch_transformers(cls, v): v = loaded_transformers return v - @validator("output_format") + @field_validator("output_format") def validate_output_format(cls, v): supported_formats = ["tensor", "variable", "raw"] if v not in supported_formats: diff --git a/lume_model/utils.py b/lume_model/utils.py index 8c63ba5..2ec2a87 100644 --- a/lume_model/utils.py +++ b/lume_model/utils.py @@ -51,7 +51,7 @@ def verify_unique_variable_names(variables: Union[list[InputVariable], list[Outp raise ValueError(f"{var_str} names {non_unique_names} are not unique.") -def serialize_variables(v): +def serialize_variables(v: dict): """Performs custom serialization for in- and output variables. Args: diff --git a/lume_model/variables.py b/lume_model/variables.py index 47e83fd..3f247bf 100644 --- a/lume_model/variables.py +++ b/lume_model/variables.py @@ -10,66 +10,12 @@ import numpy as np import logging from typing import Any, List, Union, Optional, Generic, TypeVar, Literal -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, Field, validator, ConfigDict from pydantic.generics import GenericModel logger = logging.getLogger(__name__) -class PropertyBaseModel(GenericModel): - """ - Generic base class used for the Variables. This extends the pydantic GenericModel - to serialize properties. - - TODO: - Workaround for serializing properties with pydantic until - https://github.com/samuelcolvin/pydantic/issues/935 - is solved. This solution is referenced in the issue. - """ - - @classmethod - def get_properties(cls): - return [ - prop - for prop in dir(cls) - if isinstance(getattr(cls, prop), property) - and prop not in ("__values__", "fields") - ] - - def dict( - self, - *, - include: Union["AbstractSetIntStr", "MappingIntStrAny"] = None, - exclude: Union["AbstractSetIntStr", "MappingIntStrAny"] = None, - by_alias: bool = False, - skip_defaults: bool = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - ) -> "DictStrAny": - attribs = super().dict( - include=include, - exclude=exclude, - by_alias=by_alias, - skip_defaults=skip_defaults, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - props = self.get_properties() - # Include and exclude properties - if include: - props = [prop for prop in props if prop in include] - if exclude: - props = [prop for prop in props if prop not in exclude] - - # Update the attribute dict with the properties - if props: - attribs.update({prop: getattr(self, prop) for prop in props}) - - return attribs - - class NumpyNDArray(np.ndarray): """ Custom type validator for numpy ndarray. @@ -144,7 +90,7 @@ def shape(self) -> tuple: Value = TypeVar("Value") -class Variable(PropertyBaseModel, Generic[Value]): +class Variable(BaseModel, Generic[Value]): """ Minimum requirements for a Variable @@ -157,13 +103,10 @@ class Variable(PropertyBaseModel, Generic[Value]): """ - name: str = Field(...) # name required + name: str value: Optional[Value] = None precision: Optional[int] = None - class Config: - allow_population_by_field_name = True # do not use alias only-init - class InputVariable(Variable, Generic[Value]): """ @@ -183,14 +126,7 @@ class InputVariable(Variable, Generic[Value]): """ default: Value # required default - is_constant: bool = False - - class Config: - allow_mutation = True - - def __init__(self, **kwargs): - super(Variable, self).__init__(**kwargs) - self.Config.allow_mutation = not self.is_constant + is_constant: bool = Field(False) class OutputVariable(Variable, Generic[Value]): @@ -210,8 +146,8 @@ class OutputVariable(Variable, Generic[Value]): """ - default: Optional[Value] - value_range: Optional[list] = Field(alias="range") + default: Optional[Value] = None + value_range: Optional[list] = Field(None, alias="range") class ImageVariable(BaseModel, NDVariableBase): @@ -283,7 +219,6 @@ class ScalarVariable(BaseModel): parent_variable: str = ( None # indicates that this variable is an attribute of another ) - value_range: list = Field(..., alias="range") # range required class ImageInputVariable(InputVariable[Image], ImageVariable): @@ -399,7 +334,7 @@ class ScalarInputVariable(InputVariable[float], ScalarVariable): ``` """ - pass + value_range: list[float] class ScalarOutputVariable(OutputVariable[float], ScalarVariable): @@ -423,7 +358,6 @@ class ScalarOutputVariable(OutputVariable[float], ScalarVariable): ``` """ - pass diff --git a/requirements.txt b/requirements.txt index 2f4a2d8..d7fdd15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -pydantic==1.10.9 +pydantic numpy pyyaml diff --git a/tests/conftest.py b/tests/conftest.py index de1bef8..9711c90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,8 +29,8 @@ def rootdir() -> str: @pytest.fixture(scope="session") def simple_variables() -> dict[str, Union[list[ScalarInputVariable], list[ScalarOutputVariable]]]: - input_variables = [ScalarInputVariable(name="input1", default=1.0, range=[0.0, 5.0]), - ScalarInputVariable(name="input2", default=2.0, range=[1.0, 3.0])] + input_variables = [ScalarInputVariable(name="input1", default=1.0, value_range=[0.0, 5.0]), + ScalarInputVariable(name="input2", default=2.0, value_range=[1.0, 3.0])] output_variables = [ScalarOutputVariable(name="output1"), ScalarOutputVariable(name="output2")] return {"input_variables": input_variables, "output_variables": output_variables} diff --git a/tests/keras/__init__.py b/tests/keras/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_base.py b/tests/test_base.py index b42acba..f91fbfc 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,6 +1,7 @@ import os import pytest +import yaml from lume_model.base import LUMEBaseModel @@ -25,7 +26,9 @@ def predict(self, input_dict): _ = NoEvaluateModel(**simple_variables) # init child class with evaluate function - _ = ExampleModel(**simple_variables) + example_model = ExampleModel(**simple_variables) + assert example_model.input_variables == simple_variables["input_variables"] + assert example_model.output_variables == simple_variables["output_variables"] # input and output variables sharing names is fine input_variables = simple_variables["input_variables"] @@ -35,16 +38,38 @@ def predict(self, input_dict): _ = ExampleModel(**simple_variables) input_variables[0].name = original_name - def test_serialization(self, simple_variables): + def test_dict(self, simple_variables): + example_model = ExampleModel(**simple_variables) + dict_output = example_model.dict() + assert isinstance(dict_output["input_variables"], list) + assert isinstance(dict_output["output_variables"], list) + assert len(dict_output["input_variables"]) == 2 + + def test_json(self, simple_variables): + example_model = ExampleModel(**simple_variables) + dict_output = example_model.json() + + def test_yaml_serialization(self, simple_variables): + example_model = ExampleModel(**simple_variables) + yaml_output = example_model.yaml() + dict_output = yaml.safe_load(yaml_output) + dict_output["input_variables"]["input1"]["type"] = "scalar" + + # test loading from yaml + loaded_model = ExampleModel(**dict_output) + assert loaded_model == example_model + + def test_file_serialization(self, simple_variables): example_model = ExampleModel(**simple_variables) file = "test_model.yml" - _ = example_model.yaml(file) + example_model.dump(file) + os.remove(file) def test_deserialization_from_config(self, simple_variables): example_model = ExampleModel(**simple_variables) file = "test_model.yml" - _ = example_model.yaml(file) + _ = example_model.dump(file) loaded_model = ExampleModel(file) os.remove(file) assert loaded_model.input_variables == example_model.input_variables diff --git a/tests/keras/test_layers.py b/tests/test_keras/test_layers.py similarity index 100% rename from tests/keras/test_layers.py rename to tests/test_keras/test_layers.py diff --git a/tests/keras/test_model.py b/tests/test_keras/test_model.py similarity index 100% rename from tests/keras/test_model.py rename to tests/test_keras/test_model.py diff --git a/tests/torch/test_model.py b/tests/test_torch/test_model.py similarity index 98% rename from tests/torch/test_model.py rename to tests/test_torch/test_model.py index fdb3a3e..a98052d 100644 --- a/tests/torch/test_model.py +++ b/tests/test_torch/test_model.py @@ -76,8 +76,10 @@ def test_model_from_objects( assert california_model.output_transformers == [output_transformer] def test_model_from_yaml(self, rootdir: str, california_model): - file = f"{rootdir}/test_files/california_regression/torch_model.yml" - yaml_model = TorchModel(file) + file = os.path.join( + rootdir, "test_files", "california_regression", "torch_model.yml" + ) + yaml_model = TorchModel.from_file(file) assert_model_equality(yaml_model, california_model) def test_model_as_yaml(self, rootdir: str, california_model): diff --git a/tests/torch/test_module.py b/tests/test_torch/test_module.py similarity index 100% rename from tests/torch/test_module.py rename to tests/test_torch/test_module.py diff --git a/tests/test_variables.py b/tests/test_variables.py index bda0b57..a109f1f 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -299,18 +299,18 @@ def test_output_array_variable(variable_name, default, dim_labels): { "col1": { "row1": ScalarInputVariable( - name="col1_row1", default=0, value_range=[-1, -1] + name="col1_row1", default=0,value_range=[-1, -1] ), "row2": ScalarInputVariable( - name="col1_row2", default=0, value_range=[-1, 1] + name="col1_row2", default=0,value_range=[-1, 1] ), }, "col2": { "row1": ScalarInputVariable( - name="col2_row1", default=0, value_range=[-1, -1] + name="col2_row1", default=0,value_range=[-1, -1] ), "row2": ScalarInputVariable( - name="col2_row2", default=0, value_range=[-1, 1] + name="col2_row2", default=0,value_range=[-1, 1] ), }, }, @@ -326,10 +326,10 @@ def test_output_array_variable(variable_name, default, dim_labels): }, "col2": { "row1": ScalarInputVariable( - name="col2_row1", default=0, value_range=[-1, -1] + name="col2_row1", default=0,value_range=[-1, -1] ), "row2": ScalarInputVariable( - name="col2_row2", default=0, value_range=[-1, 1] + name="col2_row2", default=0,value_range=[-1, 1] ), }, }, diff --git a/tests/torch/__init__.py b/tests/torch/__init__.py deleted file mode 100644 index e69de29..0000000 From e85a3ec3c3703b7c0676127543d57b0f7a1f4785 Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 24 Oct 2023 09:39:46 -0700 Subject: [PATCH 02/10] Update dev environment to pydantic>2.3 --- dev-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-environment.yml b/dev-environment.yml index 9adc93d..e1d2f87 100644 --- a/dev-environment.yml +++ b/dev-environment.yml @@ -4,7 +4,7 @@ channels: - conda-forge dependencies: - python=3.9 - - pydantic==1.10.9 + - pydantic>2.3 - numpy - pyyaml - tensorflow From 2a18651dd150b785b3c6bd17e278346a51ac6c44 Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 24 Oct 2023 09:56:53 -0700 Subject: [PATCH 03/10] Comment out tests for ImageVariables and ArrayVariables --- tests/test_variables.py | 515 ++++++++++++++++++++-------------------- 1 file changed, 258 insertions(+), 257 deletions(-) diff --git a/tests/test_variables.py b/tests/test_variables.py index a109f1f..447a33c 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -1,13 +1,14 @@ import pytest import numpy as np from pydantic import ValidationError + from lume_model.variables import ( ScalarInputVariable, ScalarOutputVariable, - ImageInputVariable, - ImageOutputVariable, - ArrayInputVariable, - ArrayOutputVariable, + # ImageInputVariable, + # ImageOutputVariable, + # ArrayInputVariable, + # ArrayOutputVariable, TableVariable, ) @@ -86,209 +87,209 @@ def test_output_scalar_variable(variable_name, default, value_range): ) -@pytest.mark.parametrize( - "variable_name,default,value_range,axis_labels,x_min,y_min,x_max,y_max", - [ - ("test", np.array([[1, 2,], [3, 4]]), [0, 1], ["x", "y"], 0, 0, 1, 1), - pytest.param( - "test", 1.0, [0, 1], ["x", "y"], 0, 0, 1, 1, marks=pytest.mark.xfail - ), - ("test", np.empty((3, 3)), [0, 1], ["x", "y"], 0, 0, 1, 1), - ], -) -def test_input_image_variable( - variable_name, default, value_range, axis_labels, x_min, y_min, x_max, y_max -): - # test correctly typed - ImageInputVariable( - name=variable_name, - default=default, - value_range=value_range, - axis_labels=axis_labels, - x_min=x_min, - y_min=y_min, - x_max=x_max, - y_max=y_max, - ) - - # test missing name - with pytest.raises(ValidationError): - ImageInputVariable( - default=default, - value_range=value_range, - axis_labels=axis_labels, - x_min=x_min, - y_min=y_min, - x_max=x_max, - y_max=y_max, - ) - - # test missing axis labels - with pytest.raises(ValidationError): - ImageInputVariable( - name=variable_name, - default=default, - value_range=value_range, - x_min=x_min, - y_min=y_min, - x_max=x_max, - y_max=y_max, - ) - - -@pytest.mark.parametrize( - "variable_name,default,axis_labels", - [ - ("test", np.array([[1, 2,], [3, 4]]), ["x", "y"],), - pytest.param("test", 1.0, ["x", "y"], marks=pytest.mark.xfail), - ], -) -def test_output_image_variable(variable_name, default, axis_labels): - shape = default.shape - ImageOutputVariable( - name=variable_name, default=default, shape=shape, axis_labels=axis_labels, - ) - - # test missing name - with pytest.raises(ValidationError): - ImageOutputVariable( - default=default, shape=shape, axis_labels=axis_labels, - ) - - # test missing axis labels - with pytest.raises(ValidationError): - ImageOutputVariable( - name=variable_name, default=default, - ) - - # test missing value - ImageOutputVariable( - name=variable_name, axis_labels=axis_labels, - ) - - -@pytest.mark.parametrize( - "variable_name,default,value_range,axis_labels,x_min,y_min,x_max,y_max", - [ - ("test", np.array([[1, 2,], [3, 4]]), [0, 1], ["x", "y"], 0, 0, 1, 1), - pytest.param( - "test", 1.0, [0, 1], ["x", "y"], 0, 0, 1, 1, marks=pytest.mark.xfail - ), - ], -) -def test_image_variable_shape( - variable_name, default, value_range, axis_labels, x_min, y_min, x_max, y_max -): - shape = default.shape - - # test correctly typed - variable = ImageInputVariable( - name=variable_name, - default=default, - value_range=value_range, - axis_labels=axis_labels, - x_min=x_min, - y_min=y_min, - x_max=x_max, - y_max=y_max, - ) - - assert shape == variable.shape - - -@pytest.mark.parametrize( - "variable_name,default,value_range,axis_labels,x_min,y_min,x_max,y_max", - [("test", np.array([[1, 2,], [3, 4]]), [0, 1], ["x", "y"], 0, 0, 1, 1)], -) -def test_input_image_variable_color_mode( - variable_name, default, value_range, axis_labels, x_min, y_min, x_max, y_max -): - - random_rgb_default = np.random.rand(10, 10, 3) - - # test correctly typed - variable = ImageInputVariable( - name=variable_name, - default=random_rgb_default, - value_range=value_range, - axis_labels=axis_labels, - x_min=x_min, - y_min=y_min, - x_max=x_max, - y_max=y_max, - ) - - with pytest.raises(ValueError): - random_rgb_default = np.random.rand(10, 10, 2) - # test correctly typed - variable = ImageInputVariable( - name=variable_name, - default=random_rgb_default, - value_range=value_range, - axis_labels=axis_labels, - x_min=x_min, - y_min=y_min, - x_max=x_max, - y_max=y_max, - ) - - -@pytest.mark.parametrize( - "variable_name,default,value_range,dim_labels", - [ - ("test", np.array([[1, 2,], [3, 4]]), [0, 5], ["x, y"]), - pytest.param("test", [0, 1], [0, 5], ["x", "y"], marks=pytest.mark.xfail), - ], -) -def test_input_array_variable(variable_name, default, value_range, dim_labels): - # test correctly typed - ArrayInputVariable( - name=variable_name, - default=default, - value_range=value_range, - dim_labels=dim_labels, - ) - - # test missing name - with pytest.raises(ValidationError): - ArrayInputVariable( - default=default, value_range=value_range, dim_labels=dim_labels, - ) - - # test missing axis labels - ArrayInputVariable( - name=variable_name, default=default, value_range=value_range, - ) - - -@pytest.mark.parametrize( - "variable_name,default,dim_labels", - [ - ("test", np.array([[1, 2,], [3, 4]]), ["x", "y"],), - pytest.param("test", 1.0, ["x", "y"], marks=pytest.mark.xfail), - ], -) -def test_output_array_variable(variable_name, default, dim_labels): - shape = default.shape - ArrayOutputVariable( - name=variable_name, default=default, shape=shape, dim_labels=dim_labels, - ) - - # test missing name - with pytest.raises(ValidationError): - ArrayOutputVariable( - default=default, shape=shape, dim_labels=dim_labels, - ) - - # test missing labels - ArrayOutputVariable( - name=variable_name, default=default, - ) - - # test missing value - ArrayOutputVariable( - name=variable_name, dim_labels=dim_labels, - ) +# @pytest.mark.parametrize( +# "variable_name,default,value_range,axis_labels,x_min,y_min,x_max,y_max", +# [ +# ("test", np.array([[1, 2,], [3, 4]]), [0, 1], ["x", "y"], 0, 0, 1, 1), +# pytest.param( +# "test", 1.0, [0, 1], ["x", "y"], 0, 0, 1, 1, marks=pytest.mark.xfail +# ), +# ("test", np.empty((3, 3)), [0, 1], ["x", "y"], 0, 0, 1, 1), +# ], +# ) +# def test_input_image_variable( +# variable_name, default, value_range, axis_labels, x_min, y_min, x_max, y_max +# ): +# # test correctly typed +# ImageInputVariable( +# name=variable_name, +# default=default, +# value_range=value_range, +# axis_labels=axis_labels, +# x_min=x_min, +# y_min=y_min, +# x_max=x_max, +# y_max=y_max, +# ) +# +# # test missing name +# with pytest.raises(ValidationError): +# ImageInputVariable( +# default=default, +# value_range=value_range, +# axis_labels=axis_labels, +# x_min=x_min, +# y_min=y_min, +# x_max=x_max, +# y_max=y_max, +# ) +# +# # test missing axis labels +# with pytest.raises(ValidationError): +# ImageInputVariable( +# name=variable_name, +# default=default, +# value_range=value_range, +# x_min=x_min, +# y_min=y_min, +# x_max=x_max, +# y_max=y_max, +# ) + + +# @pytest.mark.parametrize( +# "variable_name,default,axis_labels", +# [ +# ("test", np.array([[1, 2,], [3, 4]]), ["x", "y"],), +# pytest.param("test", 1.0, ["x", "y"], marks=pytest.mark.xfail), +# ], +# ) +# def test_output_image_variable(variable_name, default, axis_labels): +# shape = default.shape +# ImageOutputVariable( +# name=variable_name, default=default, shape=shape, axis_labels=axis_labels, +# ) +# +# # test missing name +# with pytest.raises(ValidationError): +# ImageOutputVariable( +# default=default, shape=shape, axis_labels=axis_labels, +# ) +# +# # test missing axis labels +# with pytest.raises(ValidationError): +# ImageOutputVariable( +# name=variable_name, default=default, +# ) +# +# # test missing value +# ImageOutputVariable( +# name=variable_name, axis_labels=axis_labels, +# ) + + +# @pytest.mark.parametrize( +# "variable_name,default,value_range,axis_labels,x_min,y_min,x_max,y_max", +# [ +# ("test", np.array([[1, 2,], [3, 4]]), [0, 1], ["x", "y"], 0, 0, 1, 1), +# pytest.param( +# "test", 1.0, [0, 1], ["x", "y"], 0, 0, 1, 1, marks=pytest.mark.xfail +# ), +# ], +# ) +# def test_image_variable_shape( +# variable_name, default, value_range, axis_labels, x_min, y_min, x_max, y_max +# ): +# shape = default.shape +# +# # test correctly typed +# variable = ImageInputVariable( +# name=variable_name, +# default=default, +# value_range=value_range, +# axis_labels=axis_labels, +# x_min=x_min, +# y_min=y_min, +# x_max=x_max, +# y_max=y_max, +# ) +# +# assert shape == variable.shape + + +# @pytest.mark.parametrize( +# "variable_name,default,value_range,axis_labels,x_min,y_min,x_max,y_max", +# [("test", np.array([[1, 2,], [3, 4]]), [0, 1], ["x", "y"], 0, 0, 1, 1)], +# ) +# def test_input_image_variable_color_mode( +# variable_name, default, value_range, axis_labels, x_min, y_min, x_max, y_max +# ): +# +# random_rgb_default = np.random.rand(10, 10, 3) +# +# # test correctly typed +# variable = ImageInputVariable( +# name=variable_name, +# default=random_rgb_default, +# value_range=value_range, +# axis_labels=axis_labels, +# x_min=x_min, +# y_min=y_min, +# x_max=x_max, +# y_max=y_max, +# ) +# +# with pytest.raises(ValueError): +# random_rgb_default = np.random.rand(10, 10, 2) +# # test correctly typed +# variable = ImageInputVariable( +# name=variable_name, +# default=random_rgb_default, +# value_range=value_range, +# axis_labels=axis_labels, +# x_min=x_min, +# y_min=y_min, +# x_max=x_max, +# y_max=y_max, +# ) + + +# @pytest.mark.parametrize( +# "variable_name,default,value_range,dim_labels", +# [ +# ("test", np.array([[1, 2,], [3, 4]]), [0, 5], ["x, y"]), +# pytest.param("test", [0, 1], [0, 5], ["x", "y"], marks=pytest.mark.xfail), +# ], +# ) +# def test_input_array_variable(variable_name, default, value_range, dim_labels): +# # test correctly typed +# ArrayInputVariable( +# name=variable_name, +# default=default, +# value_range=value_range, +# dim_labels=dim_labels, +# ) +# +# # test missing name +# with pytest.raises(ValidationError): +# ArrayInputVariable( +# default=default, value_range=value_range, dim_labels=dim_labels, +# ) +# +# # test missing axis labels +# ArrayInputVariable( +# name=variable_name, default=default, value_range=value_range, +# ) + + +# @pytest.mark.parametrize( +# "variable_name,default,dim_labels", +# [ +# ("test", np.array([[1, 2,], [3, 4]]), ["x", "y"],), +# pytest.param("test", 1.0, ["x", "y"], marks=pytest.mark.xfail), +# ], +# ) +# def test_output_array_variable(variable_name, default, dim_labels): +# shape = default.shape +# ArrayOutputVariable( +# name=variable_name, default=default, shape=shape, dim_labels=dim_labels, +# ) +# +# # test missing name +# with pytest.raises(ValidationError): +# ArrayOutputVariable( +# default=default, shape=shape, dim_labels=dim_labels, +# ) +# +# # test missing labels +# ArrayOutputVariable( +# name=variable_name, default=default, +# ) +# +# # test missing value +# ArrayOutputVariable( +# name=variable_name, dim_labels=dim_labels, +# ) @pytest.mark.parametrize( @@ -335,56 +336,56 @@ def test_output_array_variable(variable_name, default, dim_labels): }, marks=pytest.mark.xfail, ), - pytest.param( - None, - { - "col1": ArrayInputVariable( - name="test", default=np.array([1, 2]), value_range=[0, 10] - ), - "col2": { - "row1": ScalarInputVariable( - name="col2_row1", default=0, value_range=[-1, -1] - ), - "row2": ScalarInputVariable( - name="col2_row2", default=0, value_range=[-1, 1] - ), - }, - }, - marks=pytest.mark.xfail, - ), - ( - ["row1", "row2"], - { - "col1": ArrayInputVariable( - name="test", default=np.array([1, 2]), value_range=[0, 10] - ), - "col2": { - "row1": ScalarInputVariable( - name="col2_row1", default=0, value_range=[-1, -1] - ), - "row2": ScalarInputVariable( - name="col2_row2", default=0, value_range=[-1, 1] - ), - }, - }, - ), - pytest.param( - ["row1", "row2"], - { - "col1": ArrayInputVariable( - name="test", default=np.array([1, 2, 3, 4]), value_range=[0, 10] - ), - "col2": { - "row1": ScalarInputVariable( - name="col2_row1", default=0, value_range=[-1, -1] - ), - "row2": ScalarInputVariable( - name="col2_row2", default=0, value_range=[-1, 1] - ), - }, - }, - marks=pytest.mark.xfail, - ), + # pytest.param( + # None, + # { + # "col1": ArrayInputVariable( + # name="test", default=np.array([1, 2]), value_range=[0, 10] + # ), + # "col2": { + # "row1": ScalarInputVariable( + # name="col2_row1", default=0, value_range=[-1, -1] + # ), + # "row2": ScalarInputVariable( + # name="col2_row2", default=0, value_range=[-1, 1] + # ), + # }, + # }, + # marks=pytest.mark.xfail, + # ), + # ( + # ["row1", "row2"], + # { + # "col1": ArrayInputVariable( + # name="test", default=np.array([1, 2]), value_range=[0, 10] + # ), + # "col2": { + # "row1": ScalarInputVariable( + # name="col2_row1", default=0, value_range=[-1, -1] + # ), + # "row2": ScalarInputVariable( + # name="col2_row2", default=0, value_range=[-1, 1] + # ), + # }, + # }, + # ), + # pytest.param( + # ["row1", "row2"], + # { + # "col1": ArrayInputVariable( + # name="test", default=np.array([1, 2, 3, 4]), value_range=[0, 10] + # ), + # "col2": { + # "row1": ScalarInputVariable( + # name="col2_row1", default=0, value_range=[-1, -1] + # ), + # "row2": ScalarInputVariable( + # name="col2_row2", default=0, value_range=[-1, 1] + # ), + # }, + # }, + # marks=pytest.mark.xfail, + # ), ], ) def test_variable_table(rows, variables): From 83b888e49783a92b75c6112a9abb03cf893077bc Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 24 Oct 2023 11:42:29 -0700 Subject: [PATCH 04/10] Remove unused import --- lume_model/variables.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lume_model/variables.py b/lume_model/variables.py index 3f247bf..2814b81 100644 --- a/lume_model/variables.py +++ b/lume_model/variables.py @@ -11,7 +11,6 @@ import logging from typing import Any, List, Union, Optional, Generic, TypeVar, Literal from pydantic import BaseModel, Field, validator, ConfigDict -from pydantic.generics import GenericModel logger = logging.getLogger(__name__) From 5e39840871df1c0078c635bab2c315b97ec4fc67 Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 24 Oct 2023 11:45:54 -0700 Subject: [PATCH 05/10] Fix yaml() and dump() methods for LUMEBaseModel --- lume_model/base.py | 68 +++++++++++++++++++++++++++++----------------- tests/test_base.py | 1 - 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/lume_model/base.py b/lume_model/base.py index 62ea5f7..0aa2027 100644 --- a/lume_model/base.py +++ b/lume_model/base.py @@ -173,7 +173,6 @@ def json_dumps( Args: v: Object to dump. - default: Default for json.dumps(). base_key: Base key for serialization. file_prefix: Prefix for generated filenames. save_models: Determines whether models are saved to file. @@ -287,20 +286,23 @@ def validate_output_variables(cls, value): return new_value - def __init__( - self, - config: Union[dict, str] = None, - **kwargs, - ): + def __init__(self, *args, **kwargs): """Initializes LUMEBaseModel. Args: - config: Model configuration as dictionary, YAML or JSON formatted string or file path. This overrides - all other arguments. + *args: Accepts a single argument which is the model configuration as dictionary, YAML or JSON + formatted string or file path. **kwargs: See class attributes. """ - if config is not None: - self.__init__(**parse_config(config)) + if len(args) == 1: + if len(kwargs) > 0: + raise ValueError("Cannot specify YAML string and keyword arguments for LUMEBaseModel init.") + super().__init__(**parse_config(args[0])) + elif len(args) > 1: + raise ValueError( + "Arguments to LUMEBaseModel must be either a single YAML string " + "or keyword arguments passed directly to pydantic." + ) else: super().__init__(**kwargs) @@ -335,38 +337,54 @@ def json(self, **kwargs) -> str: return json.dumps(config) - def yaml(self, **kwargs): - """serialize first then dump to yaml string""" + def yaml( + self, + base_key: str = "", + file_prefix: str = "", + save_models: bool = False, + ) -> str: + """Serializes the object and returns a YAML formatted string defining the model. + + Args: + base_key: Base key for serialization. + file_prefix: Prefix for generated filenames. + save_models: Determines whether models are saved to file. + + Returns: + YAML formatted string defining the model. + """ output = json.loads( self.to_json( - **kwargs, + base_key=base_key, + file_prefix=file_prefix, + save_models=save_models, ) ) - return yaml.dump(output, default_flow_style=None, sort_keys=False) + s = yaml.dump({"model_class": self.__class__.__name__} | output, + default_flow_style=None, sort_keys=False) + return s def dump( self, file: Union[str, os.PathLike], - save_models: bool = True, base_key: str = "", + save_models: bool = True, ): """Returns and optionally saves YAML formatted string defining the model. Args: - file: If not None, the YAML formatted string is saved to given file path. - save_models: Determines whether models are saved to file. + file: File path to which the YAML formatted string and corresponding files are saved. base_key: Base key for serialization. - - Returns: - YAML formatted string defining the model. + save_models: Determines whether models are saved to file. """ file_prefix = os.path.splitext(file)[0] - with open(file, "w") as f: - f.write(self.yaml( - base_key=base_key, - file_prefix=file_prefix, - save_models=save_models) + f.write( + self.yaml( + base_key=base_key, + file_prefix=file_prefix, + save_models=save_models, + ) ) @classmethod diff --git a/tests/test_base.py b/tests/test_base.py index f91fbfc..27bbc7b 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,5 +1,4 @@ import os - import pytest import yaml From 74af64463bde350391dfde43358719209c768fcc Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 24 Oct 2023 11:47:47 -0700 Subject: [PATCH 06/10] Fix yaml() and dump() methods for torch models --- lume_model/torch/model.py | 29 +++++------- lume_model/torch/module.py | 81 +++++++++++++++++++++------------ tests/test_torch/test_model.py | 2 +- tests/test_torch/test_module.py | 2 +- 4 files changed, 66 insertions(+), 48 deletions(-) diff --git a/lume_model/torch/model.py b/lume_model/torch/model.py index 7d91cfe..cc65b83 100644 --- a/lume_model/torch/model.py +++ b/lume_model/torch/model.py @@ -4,8 +4,7 @@ from copy import deepcopy import torch -import yaml -from pydantic import validator, field_validator +from pydantic import field_validator from botorch.models.transforms.input import ReversibleInputTransform from lume_model.base import LUMEBaseModel @@ -13,8 +12,8 @@ InputVariable, OutputVariable, ScalarInputVariable, - ScalarOutputVariable, - ImageOutputVariable, + # ScalarOutputVariable, + # ImageOutputVariable, ) logger = logging.getLogger(__name__) @@ -45,20 +44,14 @@ class TorchModel(LUMEBaseModel): fixed_model: bool = True def __init__(self, *args, **kwargs): + """Initializes TorchModel. + + Args: + *args: Accepts a single argument which is the model configuration as dictionary, YAML or JSON + formatted string or file path. + **kwargs: See class attributes. """ - Initialize Xopt. - """ - if len(args) == 1: - if len(kwargs) > 0: - raise ValueError("cannot specify yaml string and kwargs for Xopt init") - super().__init__(**yaml.safe_load(args[0])) - elif len(args) > 1: - raise ValueError( - "arguments to Xopt must be either a single yaml string " - "or a keyword arguments passed directly to pydantic" - ) - else: - super().__init__(**kwargs) + super().__init__(*args, **kwargs) # set precision self.model.to(dtype=self.dtype) @@ -90,7 +83,7 @@ def validate_torch_model(cls, v): if os.path.exists(v): v = torch.load(v) else: - raise ValueError(f"path {v} does not exist!!") + raise ValueError(f"Path {v} does not exist!") return v @field_validator("input_transformers", "output_transformers", mode="before") diff --git a/lume_model/torch/module.py b/lume_model/torch/module.py index b58f0ac..b68e82b 100644 --- a/lume_model/torch/module.py +++ b/lume_model/torch/module.py @@ -6,7 +6,7 @@ import torch -from lume_model.base import json_dumps, parse_config +from lume_model.base import parse_config, recursive_serialize from lume_model.torch.model import TorchModel @@ -18,8 +18,7 @@ class TorchModule(torch.nn.Module): """ def __init__( self, - config: Union[dict, str] = None, - *, + *args, model: TorchModel = None, input_order: list[str] = None, output_order: list[str] = None, @@ -27,8 +26,8 @@ def __init__( """Initializes TorchModule. Args: - config: Model configuration as dictionary, YAML or JSON formatted string or file path. This overrides - all other arguments. + *args: Accepts a single argument which is the model configuration as dictionary, YAML or JSON + formatted string or file path. Keyword Args: model: The TorchModel instance to wrap around. If config is None, this has to be defined. @@ -37,13 +36,19 @@ def __init__( output_order: Output names in the order they are returned by the model. If None, the output order of the TorchModel is used. """ - if all(arg is None for arg in [config, model]): - raise ValueError("Either config or model has to be defined.") + if all(arg is None for arg in [*args, model]): + raise ValueError("Either a YAML string has to be given or model has to be defined.") super().__init__() - if config is not None: - kwargs = parse_config(config) + if len(args) == 1: + if not all(v is None for v in [model, input_order, output_order]): + raise ValueError("Cannot specify YAML string and keyword arguments for TorchModule init.") + kwargs = parse_config(args[0]) kwargs["model"] = TorchModel(kwargs["model"]) self.__init__(**kwargs) + elif len(args) > 1: + raise ValueError( + "Arguments to TorchModule must be either a single YAML string or keyword arguments." + ) else: self._model = model self._input_order = input_order @@ -86,43 +91,63 @@ def forward(self, x: torch.Tensor): def yaml( self, - file: Union[str, os.PathLike] = None, - save_models: bool = True, base_key: str = "", + file_prefix: str = "", + save_models: bool = False, ) -> str: - """Returns and optionally saves YAML formatted string defining the TorchModule instance. + """Serializes the object and returns a YAML formatted string defining the TorchModule instance. Args: - file: If not None, YAML formatted string is saved to given file path. - save_models: Determines whether models are saved to file. base_key: Base key for serialization. + file_prefix: Prefix for generated filenames. + save_models: Determines whether models are saved to file. Returns: YAML formatted string defining the TorchModule instance. """ - file_prefix = "" - if file is not None: - file_prefix = os.path.splitext(file)[0] - # get TorchModel config d = {} for k, v in inspect.signature(TorchModule.__init__).parameters.items(): - if k not in ["self", "config", "model"]: + if k not in ["self", "args", "model"]: d[k] = getattr(self, k) - config = json.loads( - json_dumps(d, default=None, base_key=base_key, file_prefix=file_prefix, save_models=save_models) + output = json.loads( + json.dumps(recursive_serialize(d, base_key, file_prefix, save_models)) ) - model_config = json.loads( - self._model.json(base_key=base_key, file_prefix=file_prefix, save_models=save_models) + model_output = json.loads( + self._model.to_json( + base_key=base_key, + file_prefix=file_prefix, + save_models=save_models, + ) ) - config["model"] = model_config + output["model"] = model_output # create YAML formatted string - s = yaml.dump({"model_class": self.__class__.__name__} | config, + s = yaml.dump({"model_class": self.__class__.__name__} | output, default_flow_style=None, sort_keys=False) - if file is not None: - with open(file, "w") as f: - f.write(s) return s + def dump( + self, + file: Union[str, os.PathLike], + save_models: bool = True, + base_key: str = "", + ): + """Returns and optionally saves YAML formatted string defining the model. + + Args: + file: File path to which the YAML formatted string and corresponding files are saved. + base_key: Base key for serialization. + save_models: Determines whether models are saved to file. + """ + file_prefix = os.path.splitext(file)[0] + with open(file, "w") as f: + f.write( + self.yaml( + save_models=save_models, + base_key=base_key, + file_prefix=file_prefix, + ) + ) + def evaluate_model(self, x: dict[str, torch.Tensor]): """Placeholder method to modify model calls.""" return self._model.evaluate(x) diff --git a/tests/test_torch/test_model.py b/tests/test_torch/test_model.py index a98052d..d121b84 100644 --- a/tests/test_torch/test_model.py +++ b/tests/test_torch/test_model.py @@ -85,7 +85,7 @@ def test_model_from_yaml(self, rootdir: str, california_model): def test_model_as_yaml(self, rootdir: str, california_model): filename = "test_torch_model" file = f"{filename}.yml" - _ = california_model.yaml(file) + california_model.dump(file) yaml_model = TorchModel(file) assert_model_equality(yaml_model, california_model) os.remove(file) diff --git a/tests/test_torch/test_module.py b/tests/test_torch/test_module.py index 91f938a..472fb9b 100644 --- a/tests/test_torch/test_module.py +++ b/tests/test_torch/test_module.py @@ -101,7 +101,7 @@ def test_module_from_yaml(self, rootdir: str, california_module): def test_module_as_yaml(self, rootdir: str, california_module): filename = "test_torch_module" file = f"{filename}.yml" - _ = california_module.yaml(file) + california_module.dump(file) yaml_module = TorchModule(file) assert_module_equality(yaml_module, california_module) os.remove(file) From b6a2e926bd5c9742d518b3be691b4400def5c56c Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 24 Oct 2023 11:50:11 -0700 Subject: [PATCH 07/10] Fix yaml() and dump() methods and validators for KerasModel --- lume_model/keras/model.py | 20 +++++++++++--------- tests/test_keras/test_model.py | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lume_model/keras/model.py b/lume_model/keras/model.py index 1cca10d..6492b25 100644 --- a/lume_model/keras/model.py +++ b/lume_model/keras/model.py @@ -4,15 +4,15 @@ import keras import numpy as np -from pydantic import validator +from pydantic import field_validator from lume_model.base import LUMEBaseModel from lume_model.variables import ( InputVariable, OutputVariable, ScalarInputVariable, - ScalarOutputVariable, - ImageOutputVariable, + # ScalarOutputVariable, + # ImageOutputVariable, ) logger = logging.getLogger(__name__) @@ -33,26 +33,28 @@ class KerasModel(LUMEBaseModel): def __init__( self, - config: Union[dict, str] = None, + *args, **kwargs, ): """Initializes KerasModel. Args: - config: Model configuration as dictionary, YAML or JSON formatted string or file path. This overrides - all other arguments. + *args: Accepts a single argument which is the model configuration as dictionary, YAML or JSON + formatted string or file path. **kwargs: See class attributes. """ - super().__init__(config, **kwargs) + super().__init__(*args, **kwargs) - @validator("model", pre=True) + @field_validator("model", mode="before") def validate_keras_model(cls, v): if isinstance(v, (str, os.PathLike)): if os.path.exists(v): v = keras.models.load_model(v) + else: + raise ValueError(f"Path {v} does not exist!") return v - @validator("output_format") + @field_validator("output_format") def validate_output_format(cls, v): supported_formats = ["array", "variable", "raw"] if v not in supported_formats: diff --git a/tests/test_keras/test_model.py b/tests/test_keras/test_model.py index c63906e..37971ff 100644 --- a/tests/test_keras/test_model.py +++ b/tests/test_keras/test_model.py @@ -73,7 +73,7 @@ def test_model_from_yaml(self, rootdir: str, iris_model): def test_model_as_yaml(self, rootdir: str, iris_model): filename = "test_keras_model" file = f"{filename}.yml" - _ = iris_model.yaml(file) + iris_model.dump(file) yaml_model = KerasModel(file) assert_model_equality(yaml_model, iris_model) os.remove(file) From 1c31dce31b3a9af2f43719a7bd5bcbe1884fd80a Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 24 Oct 2023 11:54:05 -0700 Subject: [PATCH 08/10] Add __init__ modules to fix namespace for pytest --- tests/test_keras/__init__.py | 0 tests/test_torch/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_keras/__init__.py create mode 100644 tests/test_torch/__init__.py diff --git a/tests/test_keras/__init__.py b/tests/test_keras/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_torch/__init__.py b/tests/test_torch/__init__.py new file mode 100644 index 0000000..e69de29 From d181a8ad2b95a06f1d17eee6d4c0fe1bbe09ae2a Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 24 Oct 2023 12:49:24 -0700 Subject: [PATCH 09/10] Restructure model subpackages --- lume_model/keras/README.md | 45 ------------------- lume_model/keras/__init__.py | 4 -- lume_model/{models.py => models/__init__.py} | 5 ++- .../layers.py => models/keras_layers.py} | 0 .../{keras/model.py => models/keras_model.py} | 0 .../{torch/model.py => models/torch_model.py} | 0 .../module.py => models/torch_module.py} | 2 +- lume_model/torch/__init__.py | 5 --- tests/conftest.py | 6 +-- .../test_keras_layers.py} | 8 ++-- .../test_keras_model.py} | 2 +- tests/{ => models}/test_models.py | 3 +- .../test_torch_model.py} | 3 +- .../test_torch_module.py} | 6 ++- tests/test_keras/__init__.py | 0 tests/test_torch/__init__.py | 0 16 files changed, 19 insertions(+), 70 deletions(-) delete mode 100644 lume_model/keras/README.md delete mode 100644 lume_model/keras/__init__.py rename lume_model/{models.py => models/__init__.py} (89%) rename lume_model/{keras/layers.py => models/keras_layers.py} (100%) rename lume_model/{keras/model.py => models/keras_model.py} (100%) rename lume_model/{torch/model.py => models/torch_model.py} (100%) rename lume_model/{torch/module.py => models/torch_module.py} (99%) delete mode 100644 lume_model/torch/__init__.py rename tests/{test_keras/test_layers.py => models/test_keras_layers.py} (83%) rename tests/{test_keras/test_model.py => models/test_keras_model.py} (99%) rename tests/{ => models}/test_models.py (83%) rename tests/{test_torch/test_model.py => models/test_torch_model.py} (99%) rename tests/{test_torch/test_module.py => models/test_torch_module.py} (98%) delete mode 100644 tests/test_keras/__init__.py delete mode 100644 tests/test_torch/__init__.py diff --git a/lume_model/keras/README.md b/lume_model/keras/README.md deleted file mode 100644 index 088fd61..0000000 --- a/lume_model/keras/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Model development with the Keras/tensorflow toolkit - -At present, only the tensorflow v2 backend is supported for this toolkit. - -The `KerasModel` packaged in the toolkit will be compatible with models saved using the `keras.save_model()` method. - -## Development requirements: -- The model must be trained using the custom scaling layers provided in `lume_model.keras.layers` OR using preprocessing layers packaged with Keras OR the custom layers must be defined during build and made accessible during loading by the user. Custom layers are not supported out-of-the box by this toolkit. -- The keras model must use named input layers such that the model will accept a dictionary input OR the `KerasModel` must be subclassed and the `format_input` and `format_output` member functions must be overwritten with proper formatting of model input from a dictionary mapping input variable names to values and proper output parsing into a dictionary, respectively. This will require use of the Keras functional API for model construction. - -An example of a model built using the functional API is given below: - -```python -from tensorflow import keras -from tensorflow.keras.layers import Dense -import tensorflow as tf - -sepal_length_input = keras.Input(shape=(1,), name="SepalLength") -sepal_width_input = keras.Input(shape=(1,), name="SepalWidth") -petal_length_input = keras.Input(shape=(1,), name="PetalLength") -petal_width_input = keras.Input(shape=(1,), name="PetalWidth") -inputs = [sepal_length_input, sepal_width_input, petal_length_input, petal_width_input] -merged = keras.layers.concatenate(inputs) -dense1 = Dense(8, activation='relu')(merged) -output = Dense(3, activation='softmax', name="Species")(dense1) - -# Compile model -model = keras.Model(inputs=inputs, outputs=[output]) -optimizer = tf.keras.optimizers.Adam() -model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) - -``` - -Models built in this way will accept inputs in dictionary form mapping variable name to a numpy array of values. - -## Configuration file -The KerasModel can be instantiated using the utility function `lume_model.utils.model_from_yaml` method. - -KerasModel can be specified in the `model_class` of the model configuration. -```yaml -model: - model_class: lume_model.keras.KerasModel -``` - -Custom parsing will require a custom model class. diff --git a/lume_model/keras/__init__.py b/lume_model/keras/__init__.py deleted file mode 100644 index 08e68fd..0000000 --- a/lume_model/keras/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -try: - from lume_model.keras.model import KerasModel -except ModuleNotFoundError: - pass diff --git a/lume_model/models.py b/lume_model/models/__init__.py similarity index 89% rename from lume_model/models.py rename to lume_model/models/__init__.py index c058ee4..cce0457 100644 --- a/lume_model/models.py +++ b/lume_model/models/__init__.py @@ -6,14 +6,15 @@ # models requiring torch try: - from lume_model.torch import TorchModel, TorchModule + from lume_model.models.torch_model import TorchModel + from lume_model.models.torch_module import TorchModule registered_models += [TorchModel, TorchModule] except ModuleNotFoundError: pass # models requiring keras try: - from lume_model.keras import KerasModel + from lume_model.models.keras_model import KerasModel registered_models += [KerasModel] except ModuleNotFoundError: pass diff --git a/lume_model/keras/layers.py b/lume_model/models/keras_layers.py similarity index 100% rename from lume_model/keras/layers.py rename to lume_model/models/keras_layers.py diff --git a/lume_model/keras/model.py b/lume_model/models/keras_model.py similarity index 100% rename from lume_model/keras/model.py rename to lume_model/models/keras_model.py diff --git a/lume_model/torch/model.py b/lume_model/models/torch_model.py similarity index 100% rename from lume_model/torch/model.py rename to lume_model/models/torch_model.py diff --git a/lume_model/torch/module.py b/lume_model/models/torch_module.py similarity index 99% rename from lume_model/torch/module.py rename to lume_model/models/torch_module.py index b68e82b..aea71e4 100644 --- a/lume_model/torch/module.py +++ b/lume_model/models/torch_module.py @@ -7,7 +7,7 @@ import torch from lume_model.base import parse_config, recursive_serialize -from lume_model.torch.model import TorchModel +from lume_model.models.torch_model import TorchModel class TorchModule(torch.nn.Module): diff --git a/lume_model/torch/__init__.py b/lume_model/torch/__init__.py deleted file mode 100644 index 105e5ba..0000000 --- a/lume_model/torch/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -try: - from lume_model.torch.model import TorchModel - from lume_model.torch.module import TorchModule -except ModuleNotFoundError: - pass diff --git a/tests/conftest.py b/tests/conftest.py index 9711c90..399609d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ import os import json -from typing import Any, TextIO, Union +from typing import Any, Union import pytest import numpy as np @@ -11,13 +11,13 @@ try: import torch from botorch.models.transforms.input import AffineInputTransform - from lume_model.torch import TorchModel, TorchModule + from lume_model.models import TorchModel, TorchModule except ModuleNotFoundError: pass try: import keras - from lume_model.keras import KerasModel + from lume_model.models import KerasModel except ModuleNotFoundError: pass diff --git a/tests/test_keras/test_layers.py b/tests/models/test_keras_layers.py similarity index 83% rename from tests/test_keras/test_layers.py rename to tests/models/test_keras_layers.py index dca914a..6e77f38 100644 --- a/tests/test_keras/test_layers.py +++ b/tests/models/test_keras_layers.py @@ -1,5 +1,5 @@ import pytest -import sys + # test value and failed initialization with characters @pytest.mark.parametrize( @@ -11,7 +11,7 @@ ], ) def test_scale_layer(offset, scale, lower, upper): - layers = pytest.importorskip("lume_model.keras.layers") + layers = pytest.importorskip("lume_model.models.keras_layers") scale_layer = layers.ScaleLayer(offset, scale, lower, upper) @@ -25,7 +25,7 @@ def test_scale_layer(offset, scale, lower, upper): ], ) def test_unscale_layer(offset, scale, lower, upper): - layers = pytest.importorskip("lume_model.keras.layers") + layers = pytest.importorskip("lume_model.models.keras_layers") unscale_layer = layers.UnscaleLayer(offset, scale, lower, upper) @@ -34,5 +34,5 @@ def test_unscale_layer(offset, scale, lower, upper): "offset,scale", [(1, 2), (5, 4), pytest.param("t", "e", marks=pytest.mark.xfail),], ) def test_unscale_image_layer(offset, scale): - layers = pytest.importorskip("lume_model.keras.layers") + layers = pytest.importorskip("lume_model.models.keras_layers") unscale_layer = layers.UnscaleImgLayer(offset, scale) diff --git a/tests/test_keras/test_model.py b/tests/models/test_keras_model.py similarity index 99% rename from tests/test_keras/test_model.py rename to tests/models/test_keras_model.py index 37971ff..45c3783 100644 --- a/tests/test_keras/test_model.py +++ b/tests/models/test_keras_model.py @@ -7,7 +7,7 @@ import numpy as np try: - from lume_model.keras import KerasModel + from lume_model.models import KerasModel from lume_model.variables import InputVariable, OutputVariable, ScalarOutputVariable except ImportError: pass diff --git a/tests/test_models.py b/tests/models/test_models.py similarity index 83% rename from tests/test_models.py rename to tests/models/test_models.py index 771b0df..e5e1fce 100644 --- a/tests/test_models.py +++ b/tests/models/test_models.py @@ -1,8 +1,7 @@ import pytest try: - from lume_model.torch import TorchModel, TorchModule - from lume_model.keras import KerasModel + from lume_model.models import TorchModel, TorchModule, KerasModel from lume_model.models import model_from_yaml except ImportError: pass diff --git a/tests/test_torch/test_model.py b/tests/models/test_torch_model.py similarity index 99% rename from tests/test_torch/test_model.py rename to tests/models/test_torch_model.py index d121b84..42968f1 100644 --- a/tests/test_torch/test_model.py +++ b/tests/models/test_torch_model.py @@ -8,7 +8,7 @@ try: import torch from botorch.models.transforms.input import AffineInputTransform - from lume_model.torch import TorchModel + from lume_model.models import TorchModel from lume_model.variables import InputVariable, OutputVariable, ScalarOutputVariable except ImportError: pass @@ -80,6 +80,7 @@ def test_model_from_yaml(self, rootdir: str, california_model): rootdir, "test_files", "california_regression", "torch_model.yml" ) yaml_model = TorchModel.from_file(file) + assert_model_equality(yaml_model, california_model) def test_model_as_yaml(self, rootdir: str, california_model): diff --git a/tests/test_torch/test_module.py b/tests/models/test_torch_module.py similarity index 98% rename from tests/test_torch/test_module.py rename to tests/models/test_torch_module.py index 472fb9b..801f74a 100644 --- a/tests/test_torch/test_module.py +++ b/tests/models/test_torch_module.py @@ -8,7 +8,7 @@ try: import torch from botorch.models import SingleTaskGP - from lume_model.torch import TorchModel, TorchModule + from lume_model.models import TorchModel, TorchModule except ImportError: pass @@ -93,7 +93,9 @@ def test_module_differentiability(self, california_test_input_tensor, california loss.backward() def test_module_from_yaml(self, rootdir: str, california_module): - file = f"{rootdir}/test_files/california_regression/torch_module.yml" + file = os.path.join( + rootdir, "test_files", "california_regression", "torch_module.yml" + ) yaml_module = TorchModule(file) assert_module_equality(yaml_module, california_module) diff --git a/tests/test_keras/__init__.py b/tests/test_keras/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_torch/__init__.py b/tests/test_torch/__init__.py deleted file mode 100644 index e69de29..0000000 From d0246f6cd2965c75b29c09c427930faf22c49fbd Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 24 Oct 2023 13:04:46 -0700 Subject: [PATCH 10/10] Update conda-recipe to pydantic>2.3 --- conda-recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 3215844..7a23356 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -16,10 +16,10 @@ requirements: - python - setuptools - pip - - pydantic==1.10.9 + - pydantic>2.3 run: - python - - pydantic==1.10.9 + - pydantic>2.3 - numpy - pyyaml