From a05660e8976a9ab8954bc29c433ed70667d2d7f9 Mon Sep 17 00:00:00 2001 From: jordi Date: Thu, 19 Sep 2024 17:23:49 +0200 Subject: [PATCH 1/4] Fix retry docs generate by marking args as False by default. --- core/dbt/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/dbt/utils.py b/core/dbt/utils.py index 05416d43344..eee3b90e675 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -355,6 +355,8 @@ def args_to_dict(args): "log_cache_events", "store_failures", "use_experimental_parser", + "static", + "empty_catalog", ) default_empty_yaml_dict_keys = ("vars", "warn_error_options") if key in default_false_keys and var_args[key] is False: From 346421bec5cd2d825bd9cabd587f9e2c954b114c Mon Sep 17 00:00:00 2001 From: jordi Date: Thu, 19 Sep 2024 18:02:51 +0200 Subject: [PATCH 2/4] add type hint --- core/dbt/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/utils.py b/core/dbt/utils.py index eee3b90e675..089b6627278 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -329,7 +329,7 @@ def __contains__(self, name) -> bool: # cli args and flags, which is more complete than just the cli args. # If new args are added that are false by default (particularly in the # global options) they should be added to the 'default_false_keys' list. -def args_to_dict(args): +def args_to_dict(args) -> dict: var_args = vars(args).copy() # update the args with the flags, which could also come from environment # variables or project_flags From 4e074877e0d4b5617ee2ae5eb00140106a7a8207 Mon Sep 17 00:00:00 2001 From: jordi Date: Thu, 19 Sep 2024 18:08:58 +0200 Subject: [PATCH 3/4] add changelog item --- .changes/unreleased/Fixes-20240919-180837.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240919-180837.yaml diff --git a/.changes/unreleased/Fixes-20240919-180837.yaml b/.changes/unreleased/Fixes-20240919-180837.yaml new file mode 100644 index 00000000000..b8a1a379889 --- /dev/null +++ b/.changes/unreleased/Fixes-20240919-180837.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix NoSuchOption errors when running `dbt retry` after `dbt docs generate`. +time: 2024-09-19T18:08:37.450919604+02:00 +custom: + Author: jordivandooren + Issue: "10741" From 59df29b39ea21300fc4b594992e3c4922f25dd63 Mon Sep 17 00:00:00 2001 From: jordi Date: Fri, 20 Sep 2024 13:44:57 +0200 Subject: [PATCH 4/4] add test for correct handling of default False args and handle --show --- core/dbt/task/retry.py | 1 - core/dbt/utils.py | 1 + tests/unit/test_flag_translation.py | 55 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_flag_translation.py diff --git a/core/dbt/task/retry.py b/core/dbt/task/retry.py index fd943b1151f..7db7ab2ca52 100644 --- a/core/dbt/task/retry.py +++ b/core/dbt/task/retry.py @@ -84,7 +84,6 @@ def __init__(self, args: Flags, config: RuntimeConfig) -> None: cli_command = CMD_DICT.get(self.previous_command_name) # type: ignore # Remove these args when their default values are present, otherwise they'll raise an exception args_to_remove = { - "show": lambda x: True, "resource_types": lambda x: x == [], "warn_error_options": lambda x: x == {"exclude": [], "include": []}, } diff --git a/core/dbt/utils.py b/core/dbt/utils.py index 089b6627278..4726230d9b9 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -357,6 +357,7 @@ def args_to_dict(args) -> dict: "use_experimental_parser", "static", "empty_catalog", + "show", ) default_empty_yaml_dict_keys = ("vars", "warn_error_options") if key in default_false_keys and var_args[key] is False: diff --git a/tests/unit/test_flag_translation.py b/tests/unit/test_flag_translation.py new file mode 100644 index 00000000000..7f4eba6db6c --- /dev/null +++ b/tests/unit/test_flag_translation.py @@ -0,0 +1,55 @@ +import click +import pytest + +from dbt.cli import main as cli_main +from dbt.cli.types import Command +from dbt.task.retry import CMD_DICT +from dbt.utils import args_to_dict + + +def is_problematic_option(option: click.Option) -> bool: + return ( + option.is_flag and not option.default and not option.secondary_opts and option.expose_value + ) + + +def get_problemetic_options_for_command(command: click.Command) -> list[str]: + """ + Get boolean flags of a ClickCommand that are False by default, do not + have a secondary option (--no-*), and expose their value. + Why do we care? If not dealt with, these arguments are stored in run_results.json + and converted to non-existent --no-* options when running dbt retry. + """ + return [ + option.name + for option in command.params + if isinstance(option, click.Option) and is_problematic_option(option) + ] + + +def get_commands_supported_by_retry() -> list[click.Command]: + command_names = [convert_enum_to_command_function_name(value) for value in CMD_DICT.values()] + return [getattr(cli_main, name) for name in command_names] + + +def convert_enum_to_command_function_name(enum: Command) -> str: + return "_".join(enum.to_list()).replace("-", "_") + + +class FlagsDummy: + def __init__(self, args: dict[str, bool]): + self.__dict__ = args + + +@pytest.mark.parametrize("command", get_commands_supported_by_retry()) +def test_flags_problematic_for_retry_are_dealt_with(command: click.Command): + """ + For each command supported by retry, get a list of flags that should + not be converted to --no-* when False, and assert if args_to_dict correctly + skips it. + """ + flag_names = get_problemetic_options_for_command(command) + flags = FlagsDummy({name: False for name in flag_names}) + args_dict = args_to_dict(flags) + for flag_name in flag_names: + assert flag_name not in args_dict, f"add {flag_name} to default_false_keys in args_to_dict"