Update pre-commit (#603)

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -18,7 +18,7 @@ repos:
       - id: sort-simple-yaml
       - id: trailing-whitespace
   - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
       - id: flake8
         args:
@@ -29,15 +29,15 @@ repos:
           - flake8-debugger
           - flake8-string-format
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 23.1.0
     hooks:
       - id: black
   - repo: 'https://github.com/PyCQA/isort'
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.982
+    rev: v0.991
     hooks:
       - id: mypy
         additional_dependencies:

mlem/cli/main.py

@@ -503,7 +503,7 @@ def inner(*iargs, **ikwargs):
                             and (o not in ikwargs or ikwargs[o] is None)
                         }
                     )
-                with (cli_echo() if not ctx.obj["quiet"] else no_echo()):
+                with cli_echo() if not ctx.obj["quiet"] else no_echo():
                     f(*iargs, **ikwargs)
             except (ClickException, Exit, Abort) as e:
                 error = f"{e.__class__.__module__}.{e.__class__.__name__}"

mlem/cli/utils.py

@@ -200,7 +200,8 @@ def _get_type_name_alias(type_):
 
 def anything(type_):
     """Creates special type that is named as original type or collection type
-    It returns original object on creation and is needed for nice typename in cli option help"""
+    It returns original object on creation and is needed for nice typename in cli option help
+    """
     return type(
         _get_type_name_alias(type_), (), {"__new__": lambda cls, value: value}
     )
@@ -570,7 +571,6 @@ def _iter_errors(
 ):
     for error in errors:
         if isinstance(error, ErrorWrapper):
-
             if loc:
                 error_loc = loc + error.loc_tuple()
             else:

mlem/contrib/bitbucketfs.py

@@ -22,7 +22,6 @@
 
 
 class BitbucketWrapper:
-
     tree_endpoint = "/api/internal/repositories/{repo}/tree/{rev}/{path}"
     repo_endpoint = "/api/2.0/repositories/{repo}"
     refs_endpoint = "/api/2.0/repositories/{repo}/refs"

mlem/contrib/docker/utils.py

@@ -175,7 +175,6 @@ def container_logs(
     until=None,
     **kwargs,
 ) -> Generator[str, None, None]:
-
     log = container.logs(
         stdout=stdout,
         stderr=stderr,

mlem/contrib/pandas.py

@@ -373,7 +373,7 @@ class DataFrameSerializer(PandasSerializer, DataSerializer[DataFrameType]):
 
     def get_model(self, data_type, prefix: str = "") -> Type[BaseModel]:
         # TODO: https://github.com/iterative/mlem/issues/33
-        row_type = List[data_type.row_type()]  # type: ignore[index]
+        row_type = List[data_type.row_type()]  # type: ignore[valid-type,index]
         return create_model(prefix + "DataFrame", values=(row_type, ...))
 
 

mlem/contrib/sagemaker/meta.py

@@ -373,7 +373,6 @@ def remove(self):
             if state.model_location is not None:
                 delete_model_file_from_s3(session, state.model_location)
             if state.endpoint_name is not None:
-
                 client = session.sagemaker_client
                 endpoint_conf = session.sagemaker_client.describe_endpoint(
                     EndpointName=state.endpoint_name

mlem/contrib/scipy.py

@@ -40,7 +40,7 @@ class ScipySparseMatrix(
 
     type: ClassVar[str] = "csr_matrix"
     valid_types: ClassVar = (spmatrix,)
-    shape: Optional[Tuple]
+    shape: Tuple[int, ...]
     """Shape of `sparse.csr_matrix` object in data"""
     dtype: str
     """Dtype of `sparse.csr_matrix` object in data"""
@@ -115,7 +115,8 @@ class ScipySparseMatrixSerializer(DataSerializer[ScipySparseMatrix]):
     def get_model(
         self, data_type: ScipySparseMatrix, prefix: str = ""
     ) -> Union[Type[BaseModel], type]:
-        item_type = List[data_type.subtype(data_type.shape[1:])]  # type: ignore[index]
+        subtype = data_type.subtype(data_type.shape[1:])
+        item_type = List[subtype]  # type: ignore[valid-type,index]
         return create_model(
             prefix + "ScipySparse",
             __root__=(item_type, ...),
@@ -136,7 +137,6 @@ def serialize(self, data_type: ScipySparseMatrix, instance: spmatrix):
         return data, (row, col)
 
     def deserialize(self, data_type, obj) -> sparse.csr_matrix:
-
         try:
             mat = sparse.csr_matrix(
                 obj, dtype=data_type.dtype, shape=data_type.shape

mlem/contrib/sklearn.py

@@ -41,7 +41,6 @@ def process(
         methods_sample_data: Optional[Dict[str, Any]] = None,
         **kwargs
     ) -> ModelType:
-
         predict_sample_data = (methods_sample_data or {}).get(
             "predict", sample_data
         )

mlem/contrib/tensorflow.py

@@ -97,7 +97,8 @@ class TFTensorSerializer(DataSerializer[TFTensorDataType]):
     data_class: ClassVar = TFTensorDataType
 
     def get_model(self, data_type, prefix: str = ""):
-        item_type = List[data_type.subtype(data_type.shape[1:])]  # type: ignore[index]
+        subtype = data_type.subtype(data_type.shape[1:])
+        item_type = List[subtype]  # type: ignore[valid-type]
         return create_model(
             prefix + "TFTensor",
             __root__=(item_type, ...),

mlem/core/artifacts.py

@@ -319,7 +319,6 @@ class LocalArtifact(FSSpecArtifact):
     type: ClassVar = "local"
 
     def relative(self, fs: AbstractFileSystem, path: str) -> "FSSpecArtifact":
-
         if isinstance(fs, LocalFileSystem):
             return LocalArtifact(
                 uri=posixpath.join(path, self.uri),

mlem/core/data_type.py

@@ -451,9 +451,8 @@ class ArraySerializer(DataSerializer[ArrayType]):
 
     def get_model(self, data_type, prefix: str = "") -> Type[BaseModel]:
         subname = prefix + "__root__"
-        item_type = List[
-            data_type.dtype.get_serializer().get_model(subname)
-        ]  # type: ignore[index]
+        model = data_type.dtype.get_serializer().get_model(subname)
+        item_type = List[model]  # type: ignore[valid-type]
         return create_model(
             prefix + "Array",
             __root__=(item_type, ...),
@@ -823,7 +822,7 @@ def write(
             )
         res = {}
         readers = {}
-        for (key, dtype) in data.item_types.items():
+        for key, dtype in data.item_types.items():
             dtype_reader, art = dtype.get_writer().write(
                 dtype.copy().bind(data.data[key]),
                 storage,
@@ -849,7 +848,7 @@ class DictReader(DataReader[DictType]):
     def read(self, artifacts: Artifacts) -> DictType:
         artifacts = flatdict.FlatterDict(artifacts, delimiter="/")
         data_dict = {}
-        for (key, dtype_reader) in self.item_readers.items():
+        for key, dtype_reader in self.item_readers.items():
             v_data_type = dtype_reader.read(artifacts.get(str(key), {}))  # type: ignore[arg-type]
             data_dict[key] = v_data_type.data
         return self.data_type.copy().bind(data_dict)

mlem/core/errors.py

@@ -25,7 +25,6 @@ class MlemProjectNotFound(MlemError):
     _message = "{MLEM_CONFIG_FILE_NAME} file wasn't found when searching through the path. Search has started from here: path={path}, fs={fs}, rev={rev}"
 
     def __init__(self, path, fs=None, rev=None) -> None:
-
         self.path = path
         self.fs = fs
         self.rev = rev
@@ -55,7 +54,6 @@ def __init__(
         path,
         fs=None,
     ) -> None:
-
         self.path = path
         self.fs = fs
         self.rev = rev
@@ -88,7 +86,6 @@ def __init__(
         self,
         data_type,
     ) -> None:
-
         self.data_type = data_type
         self.message = self._message.format(data_type=data_type)
         super().__init__(self.message)

mlem/core/meta_io.py

@@ -151,7 +151,6 @@ def find_resolver(
         rev: Optional[str],
         fs: Optional[AbstractFileSystem],
     ) -> Type["UriResolver"]:
-
         for i in cls.impls:
             if i.check(path, project, rev, fs):
                 return i
@@ -290,7 +289,9 @@ def get_fs(
             fs, _, (path,) = get_fs_token_paths(
                 path, protocol=cls.PROTOCOL, storage_options=options
             )
-        except FileNotFoundError as e:  # TODO catch HTTPError for wrong orgrepo
+        except (
+            FileNotFoundError
+        ) as e:  # TODO catch HTTPError for wrong orgrepo
             if options["sha"] is not None and not cls.check_rev(options):
                 raise RevisionNotFound(options["sha"], uri) from e
             raise LocationNotFound(f"Could not resolve location {uri}") from e
@@ -378,7 +379,8 @@ def get_path_by_fs_path(fs: AbstractFileSystem, path: str):
     """Restore full uri from fs and path
 
     Not ideal, but alternative to this is to save uri on MlemObject level and pass it everywhere
-    Another alternative is to support this on fsspec level, but we need to contribute it ourselves"""
+    Another alternative is to support this on fsspec level, but we need to contribute it ourselves
+    """
     return UriResolver.find_resolver(path, None, None, fs=fs).get_uri(
         path, None, None, fs=fs
     )

mlem/core/model.py

@@ -316,7 +316,8 @@ def resolve_method(self, method_name: str = None):
         """Checks if method with this name exists
 
         :param method_name: name of the method.
-        If not provided, this model must have only one method and it will be used"""
+        If not provided, this model must have only one method and it will be used
+        """
         if method_name is None:
             if len(self.methods) > 1:
                 raise WrongMethodError(

mlem/core/objects.py

@@ -1088,7 +1088,14 @@ def purge_state(self, deployment: "MlemDeployment"):
 
     @abstractmethod
     def lock_state(self, deployment: "MlemDeployment") -> ContextManager:
-        raise NotImplementedError
+        class Dummy(ContextManager):
+            def __enter__(self):
+                pass
+
+            def __exit__(self, exc_type, exc_val, exc_tb):
+                pass
+
+        return Dummy()
 
 
 class LocalFileStateManager(StateManager):
@@ -1207,8 +1214,8 @@ def lock_state(self, deployment: "MlemDeployment"):
         return super().lock_state(deployment)
 
 
-EnvLink: TypeAlias = MlemLink.typed_link(MlemEnv)
-ModelLink: TypeAlias = MlemLink.typed_link(MlemModel)
+EnvLink: TypeAlias = MlemLink.typed_link(MlemEnv)  # type: ignore[valid-type]
+ModelLink: TypeAlias = MlemLink.typed_link(MlemModel)  # type: ignore[valid-type]
 
 ET = TypeVar("ET", bound=MlemEnv)
 

mlem/polydantic/core.py

@@ -46,7 +46,7 @@ class PolyModel(LazyModel, metaclass=PolyModelMetaclass):
     __parent__: ClassVar[Optional[Type["PolyModel"]]] = None
     __is_root__: ClassVar[bool]
 
-    class Config:
+    class Config(BaseConfig):
         """
         Attributes:
             type_root: set to True for root of your hierarchy (parent model)
@@ -60,7 +60,7 @@ class Config:
         default_type: Optional[str] = None
 
     if TYPE_CHECKING:
-        __config__: ClassVar[Config, BaseConfig]
+        __config__: ClassVar[Type[Config]] = Config
 
     @classmethod
     def validate(cls, value):

mlem/utils/backport.py

@@ -35,7 +35,9 @@ def __get__(self, instance, owner=None):
                 )
             try:
                 cache = instance.__dict__
-            except AttributeError:  # not all objects have __dict__ (e.g. class defines slots)
+            except (
+                AttributeError
+            ):  # not all objects have __dict__ (e.g. class defines slots)
                 msg = (
                     f"No '__dict__' attribute on {type(instance).__name__!r} "
                     f"instance to cache {self.attrname!r} property."

mlem/utils/entrypoints.py

@@ -134,7 +134,6 @@ def find_implementations(
             continue
 
         for obj in module.__dict__.values():
-
             # pylint: disable=too-many-boolean-expressions
             if (
                 isinstance(obj, type)

setup.cfg

@@ -1,14 +1,23 @@
 [flake8]
 ignore =
-    E501, # too long lines, for now. TODO: https://github.com/iterative/mlem/issues/3
-    E203, # Whitespace before ':'
-    E266, # Too many leading '#' for block comment
-    W503, # Line break occurred before a binary operator
-    B008, # Do not perform function calls in argument defaults: conflicts with typer
-    P1,   # unindexed parameters in the str.format, see:
-    B902, # Invalid first argument 'cls' used for instance method.
-    B024, # ABCs without methods
-    B028, # Use f"{obj!r}" instead of f"'{obj}'"
+    # too long lines, for now. TODO: https://github.com/iterative/mlem/issues/3
+    E501,
+    # Whitespace before ':'
+    E203,
+    # Too many leading '#' for block comment
+    E266,
+    # Line break occurred before a binary operator
+    W503,
+    # Do not perform function calls in argument defaults: conflicts with typer
+    B008,
+    # unindexed parameters in the str.format, see:
+    P1,
+    # Invalid first argument 'cls' used for instance method.
+    B902,
+    # ABCs without methods
+    B024,
+    # Use f"{obj!r}" instead of f"'{obj}'"
+    B028,
     # https://pypi.org/project/flake8-string-format/
 max_line_length = 79
 max-complexity = 15
@@ -40,12 +49,15 @@ show_error_context = True
 show_traceback = True
 pretty = True
 exclude = mlem/deploy/*
-disable_error_code = misc
+disable_error_code = misc, type-abstract, annotation-unchecked
+# TODO: enable no_implicit_optional with
+# https://github.com/hauntsaninja/no_implicit_optional
+no_implicit_optional = False
+check_untyped_defs = False
 # plugins = pydantic.mypy
 
 # See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports.
 ignore_missing_imports = True
-check_untyped_defs = False
 
 # Warnings
 warn_no_return = True

tests/api/test_migrations.py

@@ -22,7 +22,6 @@
 
 @pytest.mark.parametrize("old_data", [model_03])
 def test_single(tmpdir, old_data):
-
     path = tmpdir / "model.mlem"
     old_payload, new_object = old_data
     path.write_text(safe_dump(old_payload), encoding="utf8")

tests/cli/test_deployment.py

@@ -157,7 +157,6 @@ def test_deploy_meta_link_env_project(mlem_project, mock_env_path):
 
 
 def test_deploy_meta_link_env_no_project(tmpdir, mock_env_path):
-
     deployment = MlemDeploymentMock(
         env=MlemLink(path=mock_env_path, link_type="env"),
     )
@@ -291,7 +290,6 @@ def test_deploy_apply(
 
 
 def add_mock_declare(type_: Type[MlemObject]):
-
     typer = [
         g.typer_instance
         for g in declare.registered_groups
@@ -320,7 +318,6 @@ def _deploy_and_check(
     add_args="",
     env_param_value: Optional[str] = "env_val",
 ):
-
     if load_deploy:
         status_res = runner.invoke(
             f"deploy status {deploy_path}", raise_on_error=True

tests/contrib/test_rabbitmq.py

@@ -63,7 +63,6 @@ def rmq_server(model_meta_saved_single, rmq_instance):
         queue_prefix="aaa",
     )
     for _ in range(10):
-
         t = ServeThread(model_meta_saved_single, server)
         t.start()
         time.sleep(0.5)