Add -c help WIP (#363)

* Add -c help to declare WIP * extrapolate for other commands * some field docs and little improvements * cli utils * add simple_parsing * fix tests * fix tests * lazy help * ooopsie * class and fields docstrings * reparsing cli params for nested complex objects rewrite get_field_docstring for 9000x speed fix build_model bug * fix for py37 * support lists in build_model * support lists in cli * nested options WIP * very nested options WIP * all but flat nested WIP * lil refactoring * flat nested stuff DONE * Update mlem/contrib/heroku/build.py Co-authored-by: Alexander Guschin <[email protected]> * Update mlem/cli/declare.py Co-authored-by: Alexander Guschin <[email protected]> * get rid of --conf, add mlem abc to declare * fix tests * fix lazyness * fix serialization * fix tests * fix tests * Update bitbucketfs.py * Apply suggestions from code review * fix comments and disable failfast for gh actions * backport docs from mlem.ai * sort import choices * make run_cmd optional instead of bool * docs for torch import * allow --load for groups add server config into docker build * fix windows bugs * suddenly fix dockerhub requests * suddenly fix dockerhub requests Co-authored-by: Alexander Guschin <[email protected]>
iterative · Sep 8, 2022 · 1f7bc29 · 1f7bc29
1 parent 5cef637
commit 1f7bc29
Show file tree

Hide file tree

Showing 70 changed files with 2,844 additions and 649 deletions.
diff --git a/.github/workflows/check-test-release.yml b/.github/workflows/check-test-release.yml
@@ -58,7 +58,7 @@ jobs:
         # no HDF5 support installed for tables
         - os: windows-latest
           python: "3.9"
-      fail-fast: true
+      fail-fast: false
     steps:
     - uses: actions/checkout@v3
       with:

diff --git a/.pylintrc b/.pylintrc
@@ -389,7 +389,7 @@ ignore-comments=yes
 ignore-docstrings=yes
 
 # Ignore imports when computing similarities.
-ignore-imports=no
+ignore-imports=yes
 
 # Ignore function signatures when computing similarities.
 ignore-signatures=no

diff --git a/mlem/api/commands.py b/mlem/api/commands.py
@@ -25,7 +25,7 @@
     WrongMethodError,
 )
 from mlem.core.import_objects import ImportAnalyzer, ImportHook
-from mlem.core.meta_io import MLEM_DIR, Location, UriResolver, get_fs
+from mlem.core.meta_io import MLEM_DIR, Location, get_fs
 from mlem.core.metadata import load_meta, save
 from mlem.core.objects import (
     MlemBuilder,
@@ -367,7 +367,7 @@ def ls(  # pylint: disable=too-many-locals
     include_links: bool = True,
     ignore_errors: bool = False,
 ) -> Dict[Type[MlemObject], List[MlemObject]]:
-    loc = UriResolver.resolve(
+    loc = Location.resolve(
         "", project=project, rev=rev, fs=fs, find_project=True
     )
     _validate_ls_project(loc, project)
@@ -392,7 +392,7 @@ def import_object(
     """Try to load an object as MLEM model (or data) and return it,
     optionally saving to the specified target location
     """
-    loc = UriResolver.resolve(path, project, rev, fs)
+    loc = Location.resolve(path, project, rev, fs)
     echo(EMOJI_LOAD + f"Importing object from {loc.uri_repr}")
     if type_ is not None:
         type_, modifier = parse_import_type_modifier(type_)

diff --git a/mlem/cli/apply.py b/mlem/cli/apply.py
@@ -1,13 +1,14 @@
 from json import dumps
 from typing import List, Optional
 
-from typer import Argument, Option
+from typer import Argument, Option, Typer
 
 from mlem.api import import_object
 from mlem.cli.main import (
-    config_arg,
+    app,
     mlem_command,
-    option_conf,
+    mlem_group,
+    mlem_group_callback,
     option_data_project,
     option_data_rev,
     option_external,
@@ -20,6 +21,12 @@
     option_rev,
     option_target_project,
 )
+from mlem.cli.utils import (
+    abc_fields_parameters,
+    config_arg,
+    for_each_impl,
+    lazy_class_docstring,
+)
 from mlem.core.data_type import DataAnalyzer
 from mlem.core.errors import UnsupportedDataBatchLoading
 from mlem.core.import_objects import ImportHook
@@ -32,7 +39,7 @@
 
 @mlem_command("apply", section="runtime")
 def apply(
-    model: str = Argument(..., help="Path to model object"),
+    model: str = Argument(..., metavar="model", help="Path to model object"),
     data_path: str = Argument(..., metavar="data", help="Path to data object"),
     project: Optional[str] = option_project,
     rev: Optional[str] = option_rev,
@@ -65,7 +72,8 @@ def apply(
     external: bool = option_external,
     json: bool = option_json,
 ):
-    """Apply a model to data. Resulting data will be saved as MLEM object to `output` if it is provided, otherwise will be printed
+    """Apply a model to data. The result will be saved as a MLEM object to `output` if
+    provided. Otherwise, it will be printed to `stdout`.
 
     Examples:
         Apply local mlem model to local mlem data
@@ -120,38 +128,55 @@ def apply(
         )
 
 
-@mlem_command("apply-remote", section="runtime")
-def apply_remote(
-    subtype: str = Argument(
-        "",
-        help=f"Type of client. Choices: {list_implementations(Client)}",
-        show_default=False,
-    ),
-    data: str = Argument(..., help="Path to data object"),
-    project: Optional[str] = option_project,
-    rev: Optional[str] = option_rev,
-    output: Optional[str] = Option(
-        None, "-o", "--output", help="Where to store the outputs."
-    ),
-    target_project: Optional[str] = option_target_project,
-    method: str = option_method,
-    index: bool = option_index,
-    json: bool = option_json,
-    load: Optional[str] = option_load("client"),
-    conf: List[str] = option_conf("client"),
-    file_conf: List[str] = option_file_conf("client"),
-):
-    """Apply a model (deployed somewhere remotely) to data. Resulting data will be saved as MLEM object to `output` if it is provided, otherwise will be printed
+apply_remote = Typer(
+    name="apply-remote",
+    help="""Apply a deployed-model (possibly remotely) to data. The results will be saved as
+a MLEM object to `output` if provided. Otherwise, it will be printed to
+`stdout`.
 
     Examples:
         Apply hosted mlem model to local mlem data
         $ mlem apply-remote http mydata -c host="0.0.0.0" -c port=8080 --output myprediction
-    """
-    client = config_arg(Client, load, subtype, conf, file_conf)
+    """,
+    cls=mlem_group("runtime"),
+    subcommand_metavar="client",
+)
+app.add_typer(apply_remote)
+
+
+def _apply_remote(
+    data,
+    project,
+    rev,
+    index,
+    method,
+    output,
+    target_project,
+    json,
+    type_name,
+    load,
+    file_conf,
+    kwargs,
+):
+    client = config_arg(
+        Client,
+        load,
+        type_name,
+        conf=None,
+        file_conf=file_conf,
+        **(kwargs or {}),
+    )
 
     with set_echo(None if json else ...):
         result = run_apply_remote(
-            client, data, project, rev, index, method, output, target_project
+            client,
+            data,
+            project,
+            rev,
+            index,
+            method,
+            output,
+            target_project,
         )
     if output is None and json:
         print(
@@ -161,6 +186,79 @@ def apply_remote(
         )
 
 
+option_output = Option(
+    None, "-o", "--output", help="Where to store the outputs."
+)
+
+
+@mlem_group_callback(apply_remote, required=["data", "load"])
+def apply_remote_load(
+    data: str = Option(None, "-d", "--data", help="Path to data object"),
+    project: Optional[str] = option_project,
+    rev: Optional[str] = option_rev,
+    output: Optional[str] = option_output,
+    target_project: Optional[str] = option_target_project,
+    method: str = option_method,
+    index: bool = option_index,
+    json: bool = option_json,
+    load: Optional[str] = option_load("client"),
+):
+    return _apply_remote(
+        data,
+        project,
+        rev,
+        index,
+        method,
+        output,
+        target_project,
+        json,
+        None,
+        load,
+        None,
+        None,
+    )
+
+
+@for_each_impl(Client)
+def create_apply_remote(type_name):
+    @mlem_command(
+        type_name,
+        section="clients",
+        parent=apply_remote,
+        dynamic_metavar="__kwargs__",
+        dynamic_options_generator=abc_fields_parameters(type_name, Client),
+        hidden=type_name.startswith("_"),
+        lazy_help=lazy_class_docstring(Client.abs_name, type_name),
+        no_pass_from_parent=["file_conf"],
+    )
+    def apply_remote_func(
+        data: str = Option(..., "-d", "--data", help="Path to data object"),
+        project: Optional[str] = option_project,
+        rev: Optional[str] = option_rev,
+        output: Optional[str] = option_output,
+        target_project: Optional[str] = option_target_project,
+        method: str = option_method,
+        index: bool = option_index,
+        json: bool = option_json,
+        file_conf: List[str] = option_file_conf("client"),
+        **__kwargs__,
+    ):
+        return _apply_remote(
+            data,
+            project,
+            rev,
+            index,
+            method,
+            output,
+            target_project,
+            json,
+            type_name,
+            None,
+            file_conf,
+            __kwargs__,
+        )
+
+
 def run_apply_remote(
     client: Client,
     data_path: str,

diff --git a/mlem/cli/build.py b/mlem/cli/build.py
@@ -1,49 +1,98 @@
 from typing import List, Optional
 
-from typer import Argument
+from typer import Option, Typer
 
 from mlem.cli.main import (
-    config_arg,
+    app,
     mlem_command,
-    option_conf,
+    mlem_group,
+    mlem_group_callback,
     option_file_conf,
     option_load,
     option_project,
     option_rev,
 )
+from mlem.cli.utils import (
+    abc_fields_parameters,
+    config_arg,
+    for_each_impl,
+    lazy_class_docstring,
+)
 from mlem.core.metadata import load_meta
 from mlem.core.objects import MlemBuilder, MlemModel
-from mlem.utils.entrypoints import list_implementations
+
+build = Typer(
+    name="build",
+    help="""
+        Build models to create re-usable, ship-able entities such as a Docker image or
+Python package.
+
+        Examples:
+            Build docker image from model
+            $ mlem build mymodel docker -c server.type=fastapi -c image.name=myimage
+
+            Create build docker_dir declaration and build it
+            $ mlem declare builder docker_dir -c server=fastapi -c target=build build_dock
+            $ mlem build mymodel --load build_dock
+        """,
+    cls=mlem_group("runtime", aliases=["export"]),
+    subcommand_metavar="builder",
+)
+app.add_typer(build)
 
 
-@mlem_command("build", section="runtime", aliases=["export"])
-def build(
-    model: str = Argument(..., help="Path to model"),
-    subtype: str = Argument(
-        "",
-        help=f"Type of build. Choices: {list_implementations(MlemBuilder)}",
-        show_default=False,
-    ),
+@mlem_group_callback(build, required=["model", "load"])
+def build_load(
+    model: str = Option(None, "-m", "--model", help="Path to model"),
     project: Optional[str] = option_project,
     rev: Optional[str] = option_rev,
-    load: Optional[str] = option_load("builder"),
-    conf: List[str] = option_conf("builder"),
-    file_conf: List[str] = option_file_conf("builder"),
+    load: str = option_load("builder"),
 ):
-    """
-    Build/export model
-
-    Examples:
-        Build docker image from model
-        $ mlem build mymodel docker -c server.type=fastapi -c image.name=myimage
-
-        Create build docker_dir declaration and build it
-        $ mlem declare builder docker_dir -c server=fastapi -c target=build build_dock
-        $ mlem build mymodel --load build_dock
-    """
     from mlem.api.commands import build
 
     build(
-        config_arg(MlemBuilder, load, subtype, conf, file_conf),
+        config_arg(
+            MlemBuilder,
+            load,
+            None,
+            conf=None,
+            file_conf=None,
+        ),
         load_meta(model, project, rev, force_type=MlemModel),
     )
+
+
+@for_each_impl(MlemBuilder)
+def create_build_command(type_name):
+    @mlem_command(
+        type_name,
+        section="builders",
+        parent=build,
+        dynamic_metavar="__kwargs__",
+        dynamic_options_generator=abc_fields_parameters(
+            type_name, MlemBuilder
+        ),
+        hidden=type_name.startswith("_"),
+        lazy_help=lazy_class_docstring(MlemBuilder.abs_name, type_name),
+        no_pass_from_parent=["file_conf"],
+    )
+    def build_type(
+        model: str = Option(..., "-m", "--model", help="Path to model"),
+        project: Optional[str] = option_project,
+        rev: Optional[str] = option_rev,
+        file_conf: List[str] = option_file_conf("builder"),
+        **__kwargs__
+    ):
+        from mlem.api.commands import build
+
+        build(
+            config_arg(
+                MlemBuilder,
+                None,
+                type_name,
+                conf=None,
+                file_conf=file_conf,
+                **__kwargs__
+            ),
+            load_meta(model, project, rev, force_type=MlemModel),
+        )