From f9d84555c69cf920241aa197af0de2685975b6a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 30 Aug 2022 22:30:52 -0500 Subject: [PATCH] refactor: Use inheritance for plugins' CLI --- ...dk.authenticators.APIAuthenticatorBase.rst | 3 +- docs/cli_commands.md | 4 +- docs/implementation/cli.md | 18 +- docs/porting.md | 2 +- singer_sdk/cli/__init__.py | 1 - singer_sdk/cli/common_options.py | 37 --- singer_sdk/mapper_base.py | 102 +++----- singer_sdk/plugin_base.py | 131 +++++++++- singer_sdk/tap_base.py | 242 +++++++++--------- singer_sdk/target_base.py | 112 +++----- 10 files changed, 345 insertions(+), 307 deletions(-) delete mode 100644 singer_sdk/cli/__init__.py delete mode 100644 singer_sdk/cli/common_options.py diff --git a/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst b/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst index b0d129a50..e88561e46 100644 --- a/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst +++ b/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst @@ -4,4 +4,5 @@ .. currentmodule:: singer_sdk.authenticators .. autoclass:: APIAuthenticatorBase - :members: \ No newline at end of file + :members: + :special-members: __init__ \ No newline at end of file diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 97bf0df9a..716626ce3 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -82,7 +82,7 @@ Settings: {'type': 'object', 'properties': {}} This information can also be printed in JSON format for consumption by other applications ```console -$ poetry run sdk-tap-countries-sample --about --format json +$ poetry run sdk-tap-countries-sample --about=json { "name": "sample-tap-countries", "version": "[could not be detected]", @@ -179,7 +179,7 @@ plugins: | ------------------- | :-----------------------------------------------------------------------------------------: | :------------------------------------------------------------------: | | Configuration store | Config JSON file (`--config=path/to/config.json`) or environment variables (`--config=ENV`) | `meltano.yml`, `.env`, environment variables, or Meltano's system db | | Simple invocation | `my-tap --config=...` | `meltano invoke my-tap` | -| Other CLI options | `my-tap --about --format=json` | `meltano invoke my-tap --about --format=json` | +| Other CLI options | `my-tap --about=json` | `meltano invoke my-tap --about=json` | | ELT | `my-tap --config=... \| path/to/target-jsonl --config=...` | `meltano elt my-tap target-jsonl` | [Meltano]: https://www.meltano.com diff --git a/docs/implementation/cli.md b/docs/implementation/cli.md index e4f78c26b..49f95c65d 100644 --- a/docs/implementation/cli.md +++ b/docs/implementation/cli.md @@ -14,7 +14,9 @@ This page describes how SDK-based taps and targets can be invoked via the comman - [`--help`](#--help) - [`--version`](#--version) - [`--about`](#--about) - - [`--format`](#--format) + - [`--about=plain`](#--about-plain) + - [`--about=json`](#--about-json) + - [`--about=markdown`](#--about-markdown) - [`--config`](#--config) - [`--config=ENV`](#--config-env) - [Tap-Specific CLI Options](#tap-specific-cli-options) @@ -43,13 +45,19 @@ Prints the version of the tap or target along with the SDK version and then exit Prints important information about the tap or target, including the list of supported CLI commands, the `--version` metadata, and list of supported capabilities. -_Note: By default, the format of `--about` is plain text. You can invoke `--about` in combination with the `--format` option described below to have the output printed in different formats._ +_Note: By default, the format of `--about` is plain text. You can pass a value to `--about` from one of the options described below._ -#### `--format` +#### `--about plain` -When `--format=json` is specified, the `--about` information will be printed as `json` in order to easily process the metadata in automated workflows. +Prints the plain text version of the `--about` output. This is the default. -When `--format=markdown` is specified, the `--about` information will be printed as Markdown, optimized for copy-pasting into the maintainer's `README.md` file. Among other helpful guidance, this automatically creates a markdown table of all settings, their descriptions, and their default values. +#### `--about json` + +Information will be printed as `json` in order to easily process the metadata in automated workflows. + +#### `--about markdown` + +Information will be printed as Markdown, optimized for copy-pasting into the maintainer's `README.md` file. Among other helpful guidance, this automatically creates a markdown table of all settings, their descriptions, and their default values. ### `--config` diff --git a/docs/porting.md b/docs/porting.md index 05b88a582..7b9122d00 100644 --- a/docs/porting.md +++ b/docs/porting.md @@ -225,7 +225,7 @@ To handle the conversion operation, you'll override [`Tap.load_state()`](singer_ The SDK provides autogenerated markdown you can paste into your README: ```console -poetry run tap-mysource --about --format=markdown +poetry run tap-mysource --about=markdown ``` This text will automatically document all settings, including setting descriptions. Optionally, paste this into your existing `README.md` file. diff --git a/singer_sdk/cli/__init__.py b/singer_sdk/cli/__init__.py deleted file mode 100644 index 11bf18ab7..000000000 --- a/singer_sdk/cli/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Helpers for the tap, target and mapper CLIs.""" diff --git a/singer_sdk/cli/common_options.py b/singer_sdk/cli/common_options.py deleted file mode 100644 index c9d4fef16..000000000 --- a/singer_sdk/cli/common_options.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Common CLI options for plugins.""" - -import click - -PLUGIN_VERSION = click.option( - "--version", - is_flag=True, - help="Display the package version.", -) - -PLUGIN_ABOUT = click.option( - "--about", - is_flag=True, - help="Display package metadata and settings.", -) - -PLUGIN_ABOUT_FORMAT = click.option( - "--format", - help="Specify output style for --about", - type=click.Choice(["json", "markdown"], case_sensitive=False), - default=None, -) - -PLUGIN_CONFIG = click.option( - "--config", - multiple=True, - help="Configuration file location or 'ENV' to use environment variables.", - type=click.STRING, - default=(), -) - -PLUGIN_FILE_INPUT = click.option( - "--input", - "file_input", - help="A path to read messages from instead of from standard in.", - type=click.File("r"), -) diff --git a/singer_sdk/mapper_base.py b/singer_sdk/mapper_base.py index c09d39255..d932ad5fc 100644 --- a/singer_sdk/mapper_base.py +++ b/singer_sdk/mapper_base.py @@ -2,13 +2,11 @@ import abc from io import FileIO -from typing import Callable, Iterable, List, Tuple +from typing import Iterable, List, Tuple, Type import click import singer -from singer_sdk.cli import common_options -from singer_sdk.configuration._dict_config import merge_config_sources from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers.capabilities import CapabilitiesEnum, PluginCapabilities from singer_sdk.io_base import SingerReader @@ -89,68 +87,48 @@ def map_activate_version_message( """ ... + # CLI handler + + @classmethod + def invoke( + cls: Type["InlineMapper"], + config: Tuple[str, ...] = (), + file_input: FileIO = None, + ) -> None: + """Invoke the mapper. + + Args: + config: Configuration file location or 'ENV' to use environment + variables. Accepts multiple inputs as a tuple. + file_input: Optional file to read input from. + """ + cls.print_version(print_fn=cls.logger.info) + config_files, parse_env_config = cls.config_from_cli_args(*config) + + mapper = cls( + config=config_files, + validate_config=True, + parse_env_config=parse_env_config, + ) + mapper.listen(file_input) + @classproperty - def cli(cls) -> Callable: + def cli(cls) -> click.Command: """Execute standard CLI handler for inline mappers. Returns: - A callable CLI object. + A click.Command object. """ - - @common_options.PLUGIN_VERSION - @common_options.PLUGIN_ABOUT - @common_options.PLUGIN_ABOUT_FORMAT - @common_options.PLUGIN_CONFIG - @common_options.PLUGIN_FILE_INPUT - @click.command( - help="Execute the Singer mapper.", - context_settings={"help_option_names": ["--help"]}, + command = super().cli + command.help = "Execute the Singer mapper." + command.params.extend( + [ + click.Option( + ["--input", "file_input"], + help="A path to read messages from instead of from standard in.", + type=click.File("r"), + ), + ], ) - def cli( - version: bool = False, - about: bool = False, - config: Tuple[str, ...] = (), - format: str = None, - file_input: FileIO = None, - ) -> None: - """Handle command line execution. - - Args: - version: Display the package version. - about: Display package metadata and settings. - format: Specify output style for `--about`. - config: Configuration file location or 'ENV' to use environment - variables. Accepts multiple inputs as a tuple. - file_input: Specify a path to an input file to read messages from. - Defaults to standard in if unspecified. - """ - if version: - cls.print_version() - return - - if not about: - cls.print_version(print_fn=cls.logger.info) - - validate_config: bool = True - if about: - validate_config = False - - cls.print_version(print_fn=cls.logger.info) - - config_dict = merge_config_sources( - config, - cls.config_jsonschema, - cls._env_prefix, - ) - - mapper = cls( # type: ignore # Ignore 'type not callable' - config=config_dict, - validate_config=validate_config, - ) - - if about: - mapper.print_about(format) - else: - mapper.listen(file_input) - - return cli + + return command diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 7ef4d4857..68bee956d 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -5,7 +5,7 @@ import logging import os from collections import OrderedDict -from pathlib import PurePath +from pathlib import Path, PurePath from types import MappingProxyType from typing import ( Any, @@ -397,16 +397,131 @@ def print_about(cls: Type["PluginBase"], format: Optional[str] = None) -> None: formatted = "\n".join([f"{k.title()}: {v}" for k, v in info.items()]) print(formatted) + @staticmethod + def config_from_cli_args(*args: str) -> Tuple[List[str], bool]: + """Parse CLI arguments into a config dictionary. + + Args: + args: CLI arguments. + + Raises: + FileNotFoundError: If the config file does not exist. + + Returns: + A tuple containing the config dictionary and a boolean indicating whether + the config file was found. + """ + config_files = [] + parse_env_config = False + + for config_path in args: + if config_path == "ENV": + # Allow parse from env vars: + parse_env_config = True + continue + + # Validate config file paths before adding to list + if not Path(config_path).is_file(): + raise FileNotFoundError( + f"Could not locate config file at '{config_path}'." + "Please check that the file exists." + ) + + config_files.append(Path(config_path)) + + return config_files, parse_env_config + + @abc.abstractclassmethod + def invoke(cls: Type["PluginBase"], *args: Any, **kwargs: Any) -> None: + """Invoke the plugin. + + Args: + args: Plugin arguments. + kwargs: Plugin keyword arguments. + """ + ... + + @classmethod + def cb_version( + cls: Type["PluginBase"], + ctx: click.Context, + param: click.Option, + value: bool, + ) -> None: + """CLI callback to print the plugin version and exit. + + Args: + ctx: Click context. + param: Click parameter. + value: Boolean indicating whether to print the version. + """ + if not value: + return + cls.print_version(print_fn=click.echo) + ctx.exit() + + @classmethod + def cb_about( + cls: Type["PluginBase"], + ctx: click.Context, + param: click.Option, + value: str, + ) -> None: + """CLI callback to print the plugin information and exit. + + Args: + ctx: Click context. + param: Click parameter. + value: String indicating the format of the information to print. + """ + if not value: + return + cls.print_about(format=value) + ctx.exit() + @classproperty - def cli(cls) -> Callable: + def cli(cls) -> click.Command: """Handle command line execution. Returns: A callable CLI object. """ - - @click.command() - def cli() -> None: - pass - - return cli + return click.Command( + name=cls.name, + callback=cls.invoke, + context_settings={"help_option_names": ["--help"]}, + params=[ + click.Option( + ["--version"], + is_flag=True, + help="Display the package version.", + is_eager=True, + expose_value=False, + callback=cls.cb_version, + ), + click.Option( + ["--about"], + type=click.Choice( + ["plain", "json", "markdown"], + case_sensitive=False, + ), + help="Display package metadata and settings.", + is_flag=False, + is_eager=True, + expose_value=False, + callback=cls.cb_about, + flag_value="plain", + ), + click.Option( + ["--config"], + multiple=True, + help=( + "Configuration file location or 'ENV' to use environment " + + "variables." + ), + type=click.STRING, + default=(), + is_eager=True, + ), + ], + ) diff --git a/singer_sdk/tap_base.py b/singer_sdk/tap_base.py index 75f1b481e..38f63039f 100644 --- a/singer_sdk/tap_base.py +++ b/singer_sdk/tap_base.py @@ -3,12 +3,11 @@ import abc import json from enum import Enum -from pathlib import Path, PurePath -from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union, cast +from pathlib import PurePath +from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast import click -from singer_sdk.cli import common_options from singer_sdk.exceptions import MaxRecordsLimitException from singer_sdk.helpers import _state from singer_sdk.helpers._classproperty import classproperty @@ -33,7 +32,7 @@ class CliTestOptionValue(Enum): All = "all" Schema = "schema" - Disabled = False + Disabled = "disabled" class Tap(PluginBase, metaclass=abc.ABCMeta): @@ -386,126 +385,137 @@ def sync_all(self) -> None: # Command Line Execution - @classproperty - def cli(cls) -> Callable: - """Execute standard CLI handler for taps. + @classmethod + def invoke( + cls: Type["Tap"], + config: Tuple[str, ...] = (), + state: str = None, + catalog: str = None, + ) -> None: + """Invoke the tap's command line interface. - Returns: - A callable CLI object. + Args: + config: Configuration file location or 'ENV' to use environment + variables. Accepts multiple inputs as a tuple. + catalog: Use a Singer catalog file with the tap.", + state: Use a bookmarks file for incremental replication. """ + cls.print_version(print_fn=cls.logger.info) + config_files, parse_env_config = cls.config_from_cli_args(*config) - @common_options.PLUGIN_VERSION - @common_options.PLUGIN_ABOUT - @common_options.PLUGIN_ABOUT_FORMAT - @common_options.PLUGIN_CONFIG - @click.option( - "--discover", - is_flag=True, - help="Run the tap in discovery mode.", - ) - @click.option( - "--test", - is_flag=False, - flag_value=CliTestOptionValue.All.value, - default=CliTestOptionValue.Disabled, - help=( - "Use --test to sync a single record for each stream. " - + "Use --test=schema to test schema output without syncing " - + "records." - ), + tap = cls( + config=config_files or None, + state=state, + catalog=catalog, + parse_env_config=parse_env_config, + validate_config=True, ) - @click.option( - "--catalog", - help="Use a Singer catalog file with the tap.", - type=click.Path(), + tap.sync_all() + + @classmethod + def cb_discover( + cls: Type["Tap"], + ctx: click.Context, + param: click.Option, + value: bool, + ) -> None: + """CLI callback to run the tap in discovery mode. + + Args: + ctx: Click context. + param: Click option. + value: Whether to run in discovery mode. + """ + if not value: + return + + config_args = ctx.params.get("config", ()) + config_files, parse_env_config = cls.config_from_cli_args(*config_args) + tap = cls( + config=config_files, + parse_env_config=parse_env_config, + validate_config=False, ) - @click.option( - "--state", - help="Use a bookmarks file for incremental replication.", - type=click.Path(), + tap.run_discovery() + ctx.exit() + + @classmethod + def cb_test( + cls: Type["Tap"], + ctx: click.Context, + param: click.Option, + value: bool, + ) -> None: + """CLI callback to run the tap in test mode. + + Args: + ctx: Click context. + param: Click option. + value: Whether to run in test mode. + """ + if value == CliTestOptionValue.Disabled.value: + return + + config_args = ctx.params.get("config", ()) + config_files, parse_env_config = cls.config_from_cli_args(*config_args) + tap = cls( + config=config_files, + parse_env_config=parse_env_config, + validate_config=True, ) - @click.command( - help="Execute the Singer tap.", - context_settings={"help_option_names": ["--help"]}, + + if value == CliTestOptionValue.Schema.value: + tap.write_schemas() + else: + tap.run_connection_test() + + ctx.exit() + + @classproperty + def cli(cls) -> click.Command: + """Execute standard CLI handler for taps. + + Returns: + A click.Command object. + """ + command = super().cli + command.help = "Execute the Singer tap." + command.params.extend( + [ + click.Option( + ["--discover"], + is_flag=True, + help="Run the tap in discovery mode.", + callback=cls.cb_discover, + expose_value=False, + ), + click.Option( + ["--test"], + is_flag=False, + flag_value=CliTestOptionValue.All.value, + default=CliTestOptionValue.Disabled.value, + help=( + "Use --test to sync a single record for each stream. " + + "Use --test=schema to test schema output without syncing " + + "records." + ), + callback=cls.cb_test, + expose_value=False, + ), + click.Option( + ["--catalog"], + help="Use a Singer catalog file with the tap.", + type=click.Path(), + ), + click.Option( + ["--state"], + help="Use a bookmarks file for incremental replication.", + type=click.Path(), + ), + ], ) - def cli( - version: bool = False, - about: bool = False, - discover: bool = False, - test: CliTestOptionValue = CliTestOptionValue.Disabled, - config: Tuple[str, ...] = (), - state: str = None, - catalog: str = None, - format: str = None, - ) -> None: - """Handle command line execution. - - Args: - version: Display the package version. - about: Display package metadata and settings. - discover: Run the tap in discovery mode. - test: Test connectivity by syncing a single record and exiting. - format: Specify output style for `--about`. - config: Configuration file location or 'ENV' to use environment - variables. Accepts multiple inputs as a tuple. - catalog: Use a Singer catalog file with the tap.", - state: Use a bookmarks file for incremental replication. - - Raises: - FileNotFoundError: If the config file does not exist. - """ - if version: - cls.print_version() - return - - if not about: - cls.print_version(print_fn=cls.logger.info) - else: - cls.print_about(format=format) - return - - validate_config: bool = True - if discover: - # Don't abort on validation failures - validate_config = False - - parse_env_config = False - config_files: List[PurePath] = [] - for config_path in config: - if config_path == "ENV": - # Allow parse from env vars: - parse_env_config = True - continue - - # Validate config file paths before adding to list - if not Path(config_path).is_file(): - raise FileNotFoundError( - f"Could not locate config file at '{config_path}'." - "Please check that the file exists." - ) - config_files.append(Path(config_path)) - - tap = cls( # type: ignore # Ignore 'type not callable' - config=config_files or None, - state=state, - catalog=catalog, - parse_env_config=parse_env_config, - validate_config=validate_config, - ) - - if discover: - tap.run_discovery() - if test == CliTestOptionValue.All.value: - tap.run_connection_test() - elif test == CliTestOptionValue.All.value: - tap.run_connection_test() - elif test == CliTestOptionValue.Schema.value: - tap.write_schemas() - else: - tap.sync_all() - - return cli + return command class SQLTap(Tap): diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index 1b58e88d6..130e922c5 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -6,13 +6,12 @@ import sys import time from io import FileIO -from pathlib import Path, PurePath -from typing import IO, Callable, Counter, Dict, List, Optional, Tuple, Type, Union +from pathlib import PurePath +from typing import IO, Counter, Dict, List, Optional, Tuple, Type, Union import click from joblib import Parallel, delayed, parallel_backend -from singer_sdk.cli import common_options from singer_sdk.exceptions import RecordsWithoutSchemaException from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers._compat import final @@ -469,84 +468,49 @@ def _write_state_message(self, state: dict) -> None: # CLI handler + @classmethod + def invoke( + cls: Type["Target"], + config: Tuple[str, ...] = (), + file_input: FileIO = None, + ) -> None: + """Invoke the target. + + Args: + config: Configuration file location or 'ENV' to use environment + variables. Accepts multiple inputs as a tuple. + file_input: Optional file to read input from. + """ + cls.print_version(print_fn=cls.logger.info) + config_files, parse_env_config = cls.config_from_cli_args(*config) + + target = cls( + config=config_files, + validate_config=True, + parse_env_config=parse_env_config, + ) + target.listen(file_input) + @classproperty - def cli(cls) -> Callable: + def cli(cls) -> click.Command: """Execute standard CLI handler for taps. Returns: - A callable CLI object. + A click.Command object. """ - - @common_options.PLUGIN_VERSION - @common_options.PLUGIN_ABOUT - @common_options.PLUGIN_ABOUT_FORMAT - @common_options.PLUGIN_CONFIG - @common_options.PLUGIN_FILE_INPUT - @click.command( - help="Execute the Singer target.", - context_settings={"help_option_names": ["--help"]}, + command = super().cli + command.help = "Execute the Singer target." + command.params.extend( + [ + click.Option( + ["--input", "file_input"], + help="A path to read messages from instead of from standard in.", + type=click.File("r"), + ), + ], ) - def cli( - version: bool = False, - about: bool = False, - config: Tuple[str, ...] = (), - format: str = None, - file_input: FileIO = None, - ) -> None: - """Handle command line execution. - - Args: - version: Display the package version. - about: Display package metadata and settings. - format: Specify output style for `--about`. - config: Configuration file location or 'ENV' to use environment - variables. Accepts multiple inputs as a tuple. - file_input: Specify a path to an input file to read messages from. - Defaults to standard in if unspecified. - - Raises: - FileNotFoundError: If the config file does not exist. - """ - if version: - cls.print_version() - return - - if not about: - cls.print_version(print_fn=cls.logger.info) - else: - cls.print_about(format=format) - return - - validate_config: bool = True - - cls.print_version(print_fn=cls.logger.info) - - parse_env_config = False - config_files: List[PurePath] = [] - for config_path in config: - if config_path == "ENV": - # Allow parse from env vars: - parse_env_config = True - continue - - # Validate config file paths before adding to list - if not Path(config_path).is_file(): - raise FileNotFoundError( - f"Could not locate config file at '{config_path}'." - "Please check that the file exists." - ) - - config_files.append(Path(config_path)) - - target = cls( # type: ignore # Ignore 'type not callable' - config=config_files or None, - parse_env_config=parse_env_config, - validate_config=validate_config, - ) - - target.listen(file_input) - return cli + return command class SQLTarget(Target):