diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49cfcf5b..2356a3c3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -18,7 +18,7 @@ repos: - id: sort-simple-yaml - id: trailing-whitespace - repo: https://github.com/pycqa/flake8 - rev: 5.0.4 + rev: 6.0.0 hooks: - id: flake8 args: @@ -29,15 +29,15 @@ repos: - flake8-debugger - flake8-string-format - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 23.1.0 hooks: - id: black - repo: 'https://github.com/PyCQA/isort' - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.982 + rev: v0.991 hooks: - id: mypy additional_dependencies: diff --git a/mlem/cli/main.py b/mlem/cli/main.py index 232a4947..b8f5e89d 100644 --- a/mlem/cli/main.py +++ b/mlem/cli/main.py @@ -503,7 +503,7 @@ def inner(*iargs, **ikwargs): and (o not in ikwargs or ikwargs[o] is None) } ) - with (cli_echo() if not ctx.obj["quiet"] else no_echo()): + with cli_echo() if not ctx.obj["quiet"] else no_echo(): f(*iargs, **ikwargs) except (ClickException, Exit, Abort) as e: error = f"{e.__class__.__module__}.{e.__class__.__name__}" diff --git a/mlem/cli/utils.py b/mlem/cli/utils.py index 4447dead..81355bec 100644 --- a/mlem/cli/utils.py +++ b/mlem/cli/utils.py @@ -200,7 +200,8 @@ def _get_type_name_alias(type_): def anything(type_): """Creates special type that is named as original type or collection type - It returns original object on creation and is needed for nice typename in cli option help""" + It returns original object on creation and is needed for nice typename in cli option help + """ return type( _get_type_name_alias(type_), (), {"__new__": lambda cls, value: value} ) @@ -570,7 +571,6 @@ def _iter_errors( ): for error in errors: if isinstance(error, ErrorWrapper): - if loc: error_loc = loc + error.loc_tuple() else: diff --git a/mlem/contrib/bitbucketfs.py b/mlem/contrib/bitbucketfs.py index 849dd533..ac5e104e 100644 --- a/mlem/contrib/bitbucketfs.py +++ b/mlem/contrib/bitbucketfs.py @@ -22,7 +22,6 @@ class BitbucketWrapper: - tree_endpoint = "/api/internal/repositories/{repo}/tree/{rev}/{path}" repo_endpoint = "/api/2.0/repositories/{repo}" refs_endpoint = "/api/2.0/repositories/{repo}/refs" diff --git a/mlem/contrib/docker/utils.py b/mlem/contrib/docker/utils.py index c0469772..679f3459 100644 --- a/mlem/contrib/docker/utils.py +++ b/mlem/contrib/docker/utils.py @@ -175,7 +175,6 @@ def container_logs( until=None, **kwargs, ) -> Generator[str, None, None]: - log = container.logs( stdout=stdout, stderr=stderr, diff --git a/mlem/contrib/pandas.py b/mlem/contrib/pandas.py index efacf5c2..2b685bcc 100644 --- a/mlem/contrib/pandas.py +++ b/mlem/contrib/pandas.py @@ -373,7 +373,7 @@ class DataFrameSerializer(PandasSerializer, DataSerializer[DataFrameType]): def get_model(self, data_type, prefix: str = "") -> Type[BaseModel]: # TODO: https://github.com/iterative/mlem/issues/33 - row_type = List[data_type.row_type()] # type: ignore[index] + row_type = List[data_type.row_type()] # type: ignore[valid-type,index] return create_model(prefix + "DataFrame", values=(row_type, ...)) diff --git a/mlem/contrib/sagemaker/meta.py b/mlem/contrib/sagemaker/meta.py index 437d5762..f7a2fb6b 100644 --- a/mlem/contrib/sagemaker/meta.py +++ b/mlem/contrib/sagemaker/meta.py @@ -373,7 +373,6 @@ def remove(self): if state.model_location is not None: delete_model_file_from_s3(session, state.model_location) if state.endpoint_name is not None: - client = session.sagemaker_client endpoint_conf = session.sagemaker_client.describe_endpoint( EndpointName=state.endpoint_name diff --git a/mlem/contrib/scipy.py b/mlem/contrib/scipy.py index 75c95336..a35eb9b4 100644 --- a/mlem/contrib/scipy.py +++ b/mlem/contrib/scipy.py @@ -40,7 +40,7 @@ class ScipySparseMatrix( type: ClassVar[str] = "csr_matrix" valid_types: ClassVar = (spmatrix,) - shape: Optional[Tuple] + shape: Tuple[int, ...] """Shape of `sparse.csr_matrix` object in data""" dtype: str """Dtype of `sparse.csr_matrix` object in data""" @@ -115,7 +115,8 @@ class ScipySparseMatrixSerializer(DataSerializer[ScipySparseMatrix]): def get_model( self, data_type: ScipySparseMatrix, prefix: str = "" ) -> Union[Type[BaseModel], type]: - item_type = List[data_type.subtype(data_type.shape[1:])] # type: ignore[index] + subtype = data_type.subtype(data_type.shape[1:]) + item_type = List[subtype] # type: ignore[valid-type,index] return create_model( prefix + "ScipySparse", __root__=(item_type, ...), @@ -136,7 +137,6 @@ def serialize(self, data_type: ScipySparseMatrix, instance: spmatrix): return data, (row, col) def deserialize(self, data_type, obj) -> sparse.csr_matrix: - try: mat = sparse.csr_matrix( obj, dtype=data_type.dtype, shape=data_type.shape diff --git a/mlem/contrib/sklearn.py b/mlem/contrib/sklearn.py index 99a81102..37e6348d 100644 --- a/mlem/contrib/sklearn.py +++ b/mlem/contrib/sklearn.py @@ -41,7 +41,6 @@ def process( methods_sample_data: Optional[Dict[str, Any]] = None, **kwargs ) -> ModelType: - predict_sample_data = (methods_sample_data or {}).get( "predict", sample_data ) diff --git a/mlem/contrib/tensorflow.py b/mlem/contrib/tensorflow.py index 6458eb98..c97a5a28 100644 --- a/mlem/contrib/tensorflow.py +++ b/mlem/contrib/tensorflow.py @@ -97,7 +97,8 @@ class TFTensorSerializer(DataSerializer[TFTensorDataType]): data_class: ClassVar = TFTensorDataType def get_model(self, data_type, prefix: str = ""): - item_type = List[data_type.subtype(data_type.shape[1:])] # type: ignore[index] + subtype = data_type.subtype(data_type.shape[1:]) + item_type = List[subtype] # type: ignore[valid-type] return create_model( prefix + "TFTensor", __root__=(item_type, ...), diff --git a/mlem/core/artifacts.py b/mlem/core/artifacts.py index 03eeec9a..991148e9 100644 --- a/mlem/core/artifacts.py +++ b/mlem/core/artifacts.py @@ -319,7 +319,6 @@ class LocalArtifact(FSSpecArtifact): type: ClassVar = "local" def relative(self, fs: AbstractFileSystem, path: str) -> "FSSpecArtifact": - if isinstance(fs, LocalFileSystem): return LocalArtifact( uri=posixpath.join(path, self.uri), diff --git a/mlem/core/data_type.py b/mlem/core/data_type.py index 6a16c97f..e413957f 100644 --- a/mlem/core/data_type.py +++ b/mlem/core/data_type.py @@ -451,9 +451,8 @@ class ArraySerializer(DataSerializer[ArrayType]): def get_model(self, data_type, prefix: str = "") -> Type[BaseModel]: subname = prefix + "__root__" - item_type = List[ - data_type.dtype.get_serializer().get_model(subname) - ] # type: ignore[index] + model = data_type.dtype.get_serializer().get_model(subname) + item_type = List[model] # type: ignore[valid-type] return create_model( prefix + "Array", __root__=(item_type, ...), @@ -823,7 +822,7 @@ def write( ) res = {} readers = {} - for (key, dtype) in data.item_types.items(): + for key, dtype in data.item_types.items(): dtype_reader, art = dtype.get_writer().write( dtype.copy().bind(data.data[key]), storage, @@ -849,7 +848,7 @@ class DictReader(DataReader[DictType]): def read(self, artifacts: Artifacts) -> DictType: artifacts = flatdict.FlatterDict(artifacts, delimiter="/") data_dict = {} - for (key, dtype_reader) in self.item_readers.items(): + for key, dtype_reader in self.item_readers.items(): v_data_type = dtype_reader.read(artifacts.get(str(key), {})) # type: ignore[arg-type] data_dict[key] = v_data_type.data return self.data_type.copy().bind(data_dict) diff --git a/mlem/core/errors.py b/mlem/core/errors.py index 4c1e6dab..b0d86702 100644 --- a/mlem/core/errors.py +++ b/mlem/core/errors.py @@ -25,7 +25,6 @@ class MlemProjectNotFound(MlemError): _message = "{MLEM_CONFIG_FILE_NAME} file wasn't found when searching through the path. Search has started from here: path={path}, fs={fs}, rev={rev}" def __init__(self, path, fs=None, rev=None) -> None: - self.path = path self.fs = fs self.rev = rev @@ -55,7 +54,6 @@ def __init__( path, fs=None, ) -> None: - self.path = path self.fs = fs self.rev = rev @@ -88,7 +86,6 @@ def __init__( self, data_type, ) -> None: - self.data_type = data_type self.message = self._message.format(data_type=data_type) super().__init__(self.message) diff --git a/mlem/core/meta_io.py b/mlem/core/meta_io.py index 80445a57..fd25014f 100644 --- a/mlem/core/meta_io.py +++ b/mlem/core/meta_io.py @@ -151,7 +151,6 @@ def find_resolver( rev: Optional[str], fs: Optional[AbstractFileSystem], ) -> Type["UriResolver"]: - for i in cls.impls: if i.check(path, project, rev, fs): return i @@ -290,7 +289,9 @@ def get_fs( fs, _, (path,) = get_fs_token_paths( path, protocol=cls.PROTOCOL, storage_options=options ) - except FileNotFoundError as e: # TODO catch HTTPError for wrong orgrepo + except ( + FileNotFoundError + ) as e: # TODO catch HTTPError for wrong orgrepo if options["sha"] is not None and not cls.check_rev(options): raise RevisionNotFound(options["sha"], uri) from e raise LocationNotFound(f"Could not resolve location {uri}") from e @@ -378,7 +379,8 @@ def get_path_by_fs_path(fs: AbstractFileSystem, path: str): """Restore full uri from fs and path Not ideal, but alternative to this is to save uri on MlemObject level and pass it everywhere - Another alternative is to support this on fsspec level, but we need to contribute it ourselves""" + Another alternative is to support this on fsspec level, but we need to contribute it ourselves + """ return UriResolver.find_resolver(path, None, None, fs=fs).get_uri( path, None, None, fs=fs ) diff --git a/mlem/core/model.py b/mlem/core/model.py index ee6e3a97..dfc12341 100644 --- a/mlem/core/model.py +++ b/mlem/core/model.py @@ -316,7 +316,8 @@ def resolve_method(self, method_name: str = None): """Checks if method with this name exists :param method_name: name of the method. - If not provided, this model must have only one method and it will be used""" + If not provided, this model must have only one method and it will be used + """ if method_name is None: if len(self.methods) > 1: raise WrongMethodError( diff --git a/mlem/core/objects.py b/mlem/core/objects.py index 73fd4abf..0cf03162 100644 --- a/mlem/core/objects.py +++ b/mlem/core/objects.py @@ -1088,7 +1088,14 @@ def purge_state(self, deployment: "MlemDeployment"): @abstractmethod def lock_state(self, deployment: "MlemDeployment") -> ContextManager: - raise NotImplementedError + class Dummy(ContextManager): + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + return Dummy() class LocalFileStateManager(StateManager): @@ -1207,8 +1214,8 @@ def lock_state(self, deployment: "MlemDeployment"): return super().lock_state(deployment) -EnvLink: TypeAlias = MlemLink.typed_link(MlemEnv) -ModelLink: TypeAlias = MlemLink.typed_link(MlemModel) +EnvLink: TypeAlias = MlemLink.typed_link(MlemEnv) # type: ignore[valid-type] +ModelLink: TypeAlias = MlemLink.typed_link(MlemModel) # type: ignore[valid-type] ET = TypeVar("ET", bound=MlemEnv) diff --git a/mlem/polydantic/core.py b/mlem/polydantic/core.py index 811aa8ed..23fc77ef 100644 --- a/mlem/polydantic/core.py +++ b/mlem/polydantic/core.py @@ -46,7 +46,7 @@ class PolyModel(LazyModel, metaclass=PolyModelMetaclass): __parent__: ClassVar[Optional[Type["PolyModel"]]] = None __is_root__: ClassVar[bool] - class Config: + class Config(BaseConfig): """ Attributes: type_root: set to True for root of your hierarchy (parent model) @@ -60,7 +60,7 @@ class Config: default_type: Optional[str] = None if TYPE_CHECKING: - __config__: ClassVar[Config, BaseConfig] + __config__: ClassVar[Type[Config]] = Config @classmethod def validate(cls, value): diff --git a/mlem/utils/backport.py b/mlem/utils/backport.py index a2d2631a..a8ff66e1 100644 --- a/mlem/utils/backport.py +++ b/mlem/utils/backport.py @@ -35,7 +35,9 @@ def __get__(self, instance, owner=None): ) try: cache = instance.__dict__ - except AttributeError: # not all objects have __dict__ (e.g. class defines slots) + except ( + AttributeError + ): # not all objects have __dict__ (e.g. class defines slots) msg = ( f"No '__dict__' attribute on {type(instance).__name__!r} " f"instance to cache {self.attrname!r} property." diff --git a/mlem/utils/entrypoints.py b/mlem/utils/entrypoints.py index 7c4e0604..55d36cf9 100644 --- a/mlem/utils/entrypoints.py +++ b/mlem/utils/entrypoints.py @@ -134,7 +134,6 @@ def find_implementations( continue for obj in module.__dict__.values(): - # pylint: disable=too-many-boolean-expressions if ( isinstance(obj, type) diff --git a/setup.cfg b/setup.cfg index e8146186..20af526e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,14 +1,23 @@ [flake8] ignore = - E501, # too long lines, for now. TODO: https://github.com/iterative/mlem/issues/3 - E203, # Whitespace before ':' - E266, # Too many leading '#' for block comment - W503, # Line break occurred before a binary operator - B008, # Do not perform function calls in argument defaults: conflicts with typer - P1, # unindexed parameters in the str.format, see: - B902, # Invalid first argument 'cls' used for instance method. - B024, # ABCs without methods - B028, # Use f"{obj!r}" instead of f"'{obj}'" + # too long lines, for now. TODO: https://github.com/iterative/mlem/issues/3 + E501, + # Whitespace before ':' + E203, + # Too many leading '#' for block comment + E266, + # Line break occurred before a binary operator + W503, + # Do not perform function calls in argument defaults: conflicts with typer + B008, + # unindexed parameters in the str.format, see: + P1, + # Invalid first argument 'cls' used for instance method. + B902, + # ABCs without methods + B024, + # Use f"{obj!r}" instead of f"'{obj}'" + B028, # https://pypi.org/project/flake8-string-format/ max_line_length = 79 max-complexity = 15 @@ -40,12 +49,15 @@ show_error_context = True show_traceback = True pretty = True exclude = mlem/deploy/* -disable_error_code = misc +disable_error_code = misc, type-abstract, annotation-unchecked +# TODO: enable no_implicit_optional with +# https://github.com/hauntsaninja/no_implicit_optional +no_implicit_optional = False +check_untyped_defs = False # plugins = pydantic.mypy # See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports. ignore_missing_imports = True -check_untyped_defs = False # Warnings warn_no_return = True diff --git a/tests/api/test_migrations.py b/tests/api/test_migrations.py index e466e010..51e76117 100644 --- a/tests/api/test_migrations.py +++ b/tests/api/test_migrations.py @@ -22,7 +22,6 @@ @pytest.mark.parametrize("old_data", [model_03]) def test_single(tmpdir, old_data): - path = tmpdir / "model.mlem" old_payload, new_object = old_data path.write_text(safe_dump(old_payload), encoding="utf8") diff --git a/tests/cli/test_deployment.py b/tests/cli/test_deployment.py index 8918f86c..74244c0c 100644 --- a/tests/cli/test_deployment.py +++ b/tests/cli/test_deployment.py @@ -157,7 +157,6 @@ def test_deploy_meta_link_env_project(mlem_project, mock_env_path): def test_deploy_meta_link_env_no_project(tmpdir, mock_env_path): - deployment = MlemDeploymentMock( env=MlemLink(path=mock_env_path, link_type="env"), ) @@ -291,7 +290,6 @@ def test_deploy_apply( def add_mock_declare(type_: Type[MlemObject]): - typer = [ g.typer_instance for g in declare.registered_groups @@ -320,7 +318,6 @@ def _deploy_and_check( add_args="", env_param_value: Optional[str] = "env_val", ): - if load_deploy: status_res = runner.invoke( f"deploy status {deploy_path}", raise_on_error=True diff --git a/tests/contrib/test_rabbitmq.py b/tests/contrib/test_rabbitmq.py index feedd671..abc45c8b 100644 --- a/tests/contrib/test_rabbitmq.py +++ b/tests/contrib/test_rabbitmq.py @@ -63,7 +63,6 @@ def rmq_server(model_meta_saved_single, rmq_instance): queue_prefix="aaa", ) for _ in range(10): - t = ServeThread(model_meta_saved_single, server) t.start() time.sleep(0.5) diff --git a/tests/core/custom_requirements/pack_1/model.py b/tests/core/custom_requirements/pack_1/model.py index 04a15afd..7d07c65e 100644 --- a/tests/core/custom_requirements/pack_1/model.py +++ b/tests/core/custom_requirements/pack_1/model.py @@ -2,7 +2,6 @@ class TestM: - name = name def _init_(self, alpha: float, max_lag: int): diff --git a/tests/core/test_objects.py b/tests/core/test_objects.py index 0fb39cfa..8a56fc9c 100644 --- a/tests/core/test_objects.py +++ b/tests/core/test_objects.py @@ -61,7 +61,7 @@ def remove(self): pass def get_status(self, raise_on_error=True) -> DeployStatus: - pass + return DeployStatus.NOT_DEPLOYED def _get_client(self, state): pass diff --git a/tests/polydantic/test_lazy.py b/tests/polydantic/test_lazy.py index ac4d3998..e987187e 100644 --- a/tests/polydantic/test_lazy.py +++ b/tests/polydantic/test_lazy.py @@ -34,7 +34,6 @@ def test_deserialization_and_cache(): def test_laziness(): - payload = {"field": {"value": "string"}} obj = parse_obj_as(Model, payload) diff --git a/tests/utils/test_module_tools.py b/tests/utils/test_module_tools.py index c4faede4..f971f493 100644 --- a/tests/utils/test_module_tools.py +++ b/tests/utils/test_module_tools.py @@ -216,7 +216,6 @@ def test_get_object_requirements__classvar_in_model(): def test_get_requirements_notebook(): - from nbloader import Notebook loaded_notebook = Notebook(TEST_NOTEBOOK_NAME)