From c91b2a85221e521692cc8f0fc51b15ff97a0bc13 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 03:19:31 +0000 Subject: [PATCH 01/15] refactor the mapping to a constant Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 7626d8907e..0e7004576a 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -125,6 +125,15 @@ class KedroStarterSpec: # noqa: too-few-public-methods "6": "Pyspark", } +ADD_ONS_SHORTNAME_TO_NUMBER = { + "lint": "1", + "test": "2", + "log": "3", + "docs": "4", + "data": "5", + "pyspark": "6", +} + NAME_ARG_HELP = "The name of your new Kedro project." # noqa: unused-argument @@ -428,21 +437,13 @@ def _get_addons_from_cli_input(selected_addons: str) -> str: String with the numbers corresponding to the desired add_ons, or None in case the --addons flag was not used. """ - string_to_number = { - "lint": "1", - "test": "2", - "log": "3", - "docs": "4", - "data": "5", - "pyspark": "6", - } if selected_addons is not None: addons = selected_addons.split(",") for i in range(len(addons)): addon = addons[i].strip() - if addon in string_to_number: - addons[i] = string_to_number[addon] + if addon in ADD_ONS_SHORTNAME_TO_NUMBER: + addons[i] = ADD_ONS_SHORTNAME_TO_NUMBER[addon] return ",".join(addons) return None From 62e838f5b9e35e1b587d8ad340025d070ad4a13d Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 03:26:02 +0000 Subject: [PATCH 02/15] add a new mapping to convert short name to long name easily Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 0e7004576a..f598b1df0c 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -116,15 +116,6 @@ class KedroStarterSpec: # noqa: too-few-public-methods kedro new --addons=none """ -ADD_ONS_DICT = { - "1": "Linting", - "2": "Testing", - "3": "Custom Logging", - "4": "Documentation", - "5": "Data Structure", - "6": "Pyspark", -} - ADD_ONS_SHORTNAME_TO_NUMBER = { "lint": "1", "test": "2", @@ -133,6 +124,19 @@ class KedroStarterSpec: # noqa: too-few-public-methods "data": "5", "pyspark": "6", } +NUMBER_TO_ADD_ONS_NAME = { + "1": "Linting", + "2": "Testing", + "3": "Custom Logging", + "4": "Documentation", + "5": "Data Structure", + "6": "Pyspark", +} +ADD_ONS_SHORTNAME_TO_NAME = { + short_name: NUMBER_TO_ADD_ONS_NAME[num] + for short_name, num in ADD_ONS_SHORTNAME_TO_NUMBER.items() +} + NAME_ARG_HELP = "The name of your new Kedro project." @@ -222,13 +226,13 @@ def _validate_range(start, end): def _validate_selection(add_ons: list[str]): for add_on in add_ons: - if int(add_on) < 1 or int(add_on) > len(ADD_ONS_DICT): + if int(add_on) < 1 or int(add_on) > len(NUMBER_TO_ADD_ONS_NAME): message = f"'{add_on}' is not a valid selection.\nPlease select from the available add-ons: 1, 2, 3, 4, 5, 6." # nosec click.secho(message, fg="red", err=True) sys.exit(1) if add_ons_str == "all": - return list(ADD_ONS_DICT) + return list(NUMBER_TO_ADD_ONS_NAME) if add_ons_str == "none": return [] # Guard clause if add_ons_str is None, which can happen if prompts.yml is removed @@ -559,7 +563,7 @@ def _make_cookiecutter_args( add_ons = config.get("add_ons") if add_ons: config["add_ons"] = [ - ADD_ONS_DICT[add_on] for add_on in _parse_add_ons_input(add_ons) # type: ignore + NUMBER_TO_ADD_ONS_NAME[add_on] for add_on in _parse_add_ons_input(add_ons) # type: ignore ] config["add_ons"] = str(config["add_ons"]) From 3c6206b39f0da364e1990a48c1a9c3c29d46a1c9 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 03:31:20 +0000 Subject: [PATCH 03/15] shortcut return if add_ons is None, refactor the logic with a helper dictionary Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index f598b1df0c..eba57afeb5 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -441,16 +441,16 @@ def _get_addons_from_cli_input(selected_addons: str) -> str: String with the numbers corresponding to the desired add_ons, or None in case the --addons flag was not used. """ + if selected_addons is None: + return None + + addons = [] + for addon in selected_addons.split(","): + addon = addon.strip() + if addon in ADD_ONS_SHORTNAME_TO_NUMBER: + addons.append(ADD_ONS_SHORTNAME_TO_NUMBER[addon]) + return ",".join(addons) - if selected_addons is not None: - addons = selected_addons.split(",") - for i in range(len(addons)): - addon = addons[i].strip() - if addon in ADD_ONS_SHORTNAME_TO_NUMBER: - addons[i] = ADD_ONS_SHORTNAME_TO_NUMBER[addon] - return ",".join(addons) - - return None def _select_prompts_to_display( From bf9e6395b747943bf38944d572fbb521250db0df Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 03:44:07 +0000 Subject: [PATCH 04/15] refactor variable name to reflect what it does Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 53 +++++++++++++++------------------ 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index eba57afeb5..7f17155ac7 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -116,15 +116,7 @@ class KedroStarterSpec: # noqa: too-few-public-methods kedro new --addons=none """ -ADD_ONS_SHORTNAME_TO_NUMBER = { - "lint": "1", - "test": "2", - "log": "3", - "docs": "4", - "data": "5", - "pyspark": "6", -} -NUMBER_TO_ADD_ONS_NAME = { +ADD_ONS_DICT = { "1": "Linting", "2": "Testing", "3": "Custom Logging", @@ -132,11 +124,6 @@ class KedroStarterSpec: # noqa: too-few-public-methods "5": "Data Structure", "6": "Pyspark", } -ADD_ONS_SHORTNAME_TO_NAME = { - short_name: NUMBER_TO_ADD_ONS_NAME[num] - for short_name, num in ADD_ONS_SHORTNAME_TO_NUMBER.items() -} - NAME_ARG_HELP = "The name of your new Kedro project." @@ -226,13 +213,13 @@ def _validate_range(start, end): def _validate_selection(add_ons: list[str]): for add_on in add_ons: - if int(add_on) < 1 or int(add_on) > len(NUMBER_TO_ADD_ONS_NAME): + if int(add_on) < 1 or int(add_on) > len(ADD_ONS_DICT): message = f"'{add_on}' is not a valid selection.\nPlease select from the available add-ons: 1, 2, 3, 4, 5, 6." # nosec click.secho(message, fg="red", err=True) sys.exit(1) if add_ons_str == "all": - return list(NUMBER_TO_ADD_ONS_NAME) + return list(ADD_ONS_DICT) if add_ons_str == "none": return [] # Guard clause if add_ons_str is None, which can happen if prompts.yml is removed @@ -335,7 +322,7 @@ def new( # noqa: too-many-arguments shutil.rmtree(tmpdir, onerror=_remove_readonly) # Obtain config, either from a file or from interactive user prompts. - config = _get_config( + extra_context = _get_extra_context( prompts_required=prompts_required, config_path=config_path, cookiecutter_context=cookiecutter_context, @@ -343,7 +330,7 @@ def new( # noqa: too-many-arguments project_name=project_name, ) - cookiecutter_args = _make_cookiecutter_args(config, checkout, directory) + cookiecutter_args = _make_cookiecutter_args(extra_context, checkout, directory) project_template = fetch_template_based_on_add_ons(template_path, cookiecutter_args) @@ -380,14 +367,14 @@ def list_starters(): ) -def _get_config( +def _get_extra_context( prompts_required: dict, config_path: str, cookiecutter_context: OrderedDict, selected_addons: str, project_name: str, ) -> dict[str, str]: - """Generates a config dictionary to be used to generate cookiecutter args, based + """Generates a config dictionary to will be passed to cookiecutter as `extra_context`, based on CLI flags, user prompts, or a configuration file. Args: @@ -441,16 +428,24 @@ def _get_addons_from_cli_input(selected_addons: str) -> str: String with the numbers corresponding to the desired add_ons, or None in case the --addons flag was not used. """ - if selected_addons is None: - return None + string_to_number = { + "lint": "1", + "test": "2", + "log": "3", + "docs": "4", + "data": "5", + "pyspark": "6", + } - addons = [] - for addon in selected_addons.split(","): - addon = addon.strip() - if addon in ADD_ONS_SHORTNAME_TO_NUMBER: - addons.append(ADD_ONS_SHORTNAME_TO_NUMBER[addon]) - return ",".join(addons) + if selected_addons is not None: + addons = selected_addons.split(",") + for i in range(len(addons)): + addon = addons[i].strip() + if addon in string_to_number: + addons[i] = string_to_number[addon] + return ",".join(addons) + return None def _select_prompts_to_display( @@ -563,7 +558,7 @@ def _make_cookiecutter_args( add_ons = config.get("add_ons") if add_ons: config["add_ons"] = [ - NUMBER_TO_ADD_ONS_NAME[add_on] for add_on in _parse_add_ons_input(add_ons) # type: ignore + ADD_ONS_DICT[add_on] for add_on in _parse_add_ons_input(add_ons) # type: ignore ] config["add_ons"] = str(config["add_ons"]) From ba57dfbbf62c37783a87e5283837a4c300d4a7a4 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 03:44:54 +0000 Subject: [PATCH 05/15] rename config to extra_context Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 7f17155ac7..efe47e63f7 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -392,27 +392,27 @@ def _get_extra_context( the prompts_required dictionary, with all the redundant information removed. """ if not prompts_required: - config = {} + extra_context = {} if config_path: - config = _fetch_config_from_file(config_path) - _validate_config_file_inputs(config) + extra_context = _fetch_config_from_file(config_path) + _validate_config_file_inputs(extra_context) elif config_path: - config = _fetch_config_from_file(config_path) - _validate_config_file_against_prompts(config, prompts_required) - _validate_config_file_inputs(config) + extra_context = _fetch_config_from_file(config_path) + _validate_config_file_against_prompts(extra_context, prompts_required) + _validate_config_file_inputs(extra_context) else: - config = _fetch_config_from_user_prompts(prompts_required, cookiecutter_context) + extra_context = _fetch_config_from_user_prompts(prompts_required, cookiecutter_context) add_ons = _get_addons_from_cli_input(selected_addons) if add_ons is not None: - config["add_ons"] = add_ons + extra_context["add_ons"] = add_ons if project_name is not None: - config["project_name"] = project_name + extra_context["project_name"] = project_name - return config + return extra_context def _get_addons_from_cli_input(selected_addons: str) -> str: From ad2318966c8b6839d2350dbe2720294dc9955692 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 03:50:02 +0000 Subject: [PATCH 06/15] more refactor Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index efe47e63f7..822080beab 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -374,7 +374,7 @@ def _get_extra_context( selected_addons: str, project_name: str, ) -> dict[str, str]: - """Generates a config dictionary to will be passed to cookiecutter as `extra_context`, based + """Generates a config dictionary to will be passed, based on CLI flags, user prompts, or a configuration file. Args: @@ -392,27 +392,27 @@ def _get_extra_context( the prompts_required dictionary, with all the redundant information removed. """ if not prompts_required: - extra_context = {} + config = {} if config_path: - extra_context = _fetch_config_from_file(config_path) - _validate_config_file_inputs(extra_context) + config = _fetch_config_from_file(config_path) + _validate_config_file_inputs(config) elif config_path: - extra_context = _fetch_config_from_file(config_path) - _validate_config_file_against_prompts(extra_context, prompts_required) - _validate_config_file_inputs(extra_context) + config = _fetch_config_from_file(config_path) + _validate_config_file_against_prompts(config, prompts_required) + _validate_config_file_inputs(config) else: - extra_context = _fetch_config_from_user_prompts(prompts_required, cookiecutter_context) + config = _fetch_config_from_user_prompts(prompts_required, cookiecutter_context) add_ons = _get_addons_from_cli_input(selected_addons) if add_ons is not None: - extra_context["add_ons"] = add_ons + config["add_ons"] = add_ons if project_name is not None: - extra_context["project_name"] = project_name + config["project_name"] = project_name - return extra_context + return config def _get_addons_from_cli_input(selected_addons: str) -> str: From dc54eb9efebff175ba16143ac1b7ed147e3c1c68 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 03:50:51 +0000 Subject: [PATCH 07/15] Revert "refactor variable name to reflect what it does" This reverts commit bf9e6395b747943bf38944d572fbb521250db0df. Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 51 ++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 822080beab..87f24638bd 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -116,7 +116,15 @@ class KedroStarterSpec: # noqa: too-few-public-methods kedro new --addons=none """ -ADD_ONS_DICT = { +ADD_ONS_SHORTNAME_TO_NUMBER = { + "lint": "1", + "test": "2", + "log": "3", + "docs": "4", + "data": "5", + "pyspark": "6", +} +NUMBER_TO_ADD_ONS_NAME = { "1": "Linting", "2": "Testing", "3": "Custom Logging", @@ -124,6 +132,11 @@ class KedroStarterSpec: # noqa: too-few-public-methods "5": "Data Structure", "6": "Pyspark", } +ADD_ONS_SHORTNAME_TO_NAME = { + short_name: NUMBER_TO_ADD_ONS_NAME[num] + for short_name, num in ADD_ONS_SHORTNAME_TO_NUMBER.items() +} + NAME_ARG_HELP = "The name of your new Kedro project." @@ -213,13 +226,13 @@ def _validate_range(start, end): def _validate_selection(add_ons: list[str]): for add_on in add_ons: - if int(add_on) < 1 or int(add_on) > len(ADD_ONS_DICT): + if int(add_on) < 1 or int(add_on) > len(NUMBER_TO_ADD_ONS_NAME): message = f"'{add_on}' is not a valid selection.\nPlease select from the available add-ons: 1, 2, 3, 4, 5, 6." # nosec click.secho(message, fg="red", err=True) sys.exit(1) if add_ons_str == "all": - return list(ADD_ONS_DICT) + return list(NUMBER_TO_ADD_ONS_NAME) if add_ons_str == "none": return [] # Guard clause if add_ons_str is None, which can happen if prompts.yml is removed @@ -322,7 +335,7 @@ def new( # noqa: too-many-arguments shutil.rmtree(tmpdir, onerror=_remove_readonly) # Obtain config, either from a file or from interactive user prompts. - extra_context = _get_extra_context( + config = _get_config( prompts_required=prompts_required, config_path=config_path, cookiecutter_context=cookiecutter_context, @@ -330,7 +343,7 @@ def new( # noqa: too-many-arguments project_name=project_name, ) - cookiecutter_args = _make_cookiecutter_args(extra_context, checkout, directory) + cookiecutter_args = _make_cookiecutter_args(config, checkout, directory) project_template = fetch_template_based_on_add_ons(template_path, cookiecutter_args) @@ -367,7 +380,7 @@ def list_starters(): ) -def _get_extra_context( +def _get_config( prompts_required: dict, config_path: str, cookiecutter_context: OrderedDict, @@ -428,24 +441,16 @@ def _get_addons_from_cli_input(selected_addons: str) -> str: String with the numbers corresponding to the desired add_ons, or None in case the --addons flag was not used. """ - string_to_number = { - "lint": "1", - "test": "2", - "log": "3", - "docs": "4", - "data": "5", - "pyspark": "6", - } + if selected_addons is None: + return None - if selected_addons is not None: - addons = selected_addons.split(",") - for i in range(len(addons)): - addon = addons[i].strip() - if addon in string_to_number: - addons[i] = string_to_number[addon] - return ",".join(addons) + addons = [] + for addon in selected_addons.split(","): + addon = addon.strip() + if addon in ADD_ONS_SHORTNAME_TO_NUMBER: + addons.append(ADD_ONS_SHORTNAME_TO_NUMBER[addon]) + return ",".join(addons) - return None def _select_prompts_to_display( @@ -558,7 +563,7 @@ def _make_cookiecutter_args( add_ons = config.get("add_ons") if add_ons: config["add_ons"] = [ - ADD_ONS_DICT[add_on] for add_on in _parse_add_ons_input(add_ons) # type: ignore + NUMBER_TO_ADD_ONS_NAME[add_on] for add_on in _parse_add_ons_input(add_ons) # type: ignore ] config["add_ons"] = str(config["add_ons"]) From 9d4c6686147e3a322ec95499bb1c2b6f34b5b42a Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 03:54:06 +0000 Subject: [PATCH 08/15] refactor the list with the constants dict Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 87f24638bd..e33ceb5325 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -435,7 +435,7 @@ def _get_addons_from_cli_input(selected_addons: str) -> str: Args: selected_addons: a string containing the value for the --addons flag, - or None in case the flag wasn't used. + or None in case the flag wasn't used, i.e. lint,docs. Returns: String with the numbers corresponding to the desired add_ons, or @@ -471,7 +471,7 @@ def _select_prompts_to_display( Returns: the prompts_required dictionary, with all the redundant information removed. """ - valid_addons = ["lint", "test", "log", "docs", "data", "pyspark", "all", "none"] + valid_addons = list(ADD_ONS_SHORTNAME_TO_NAME) + ["all", "none"] if selected_addons is not None: addons = re.sub(r"\s", "", selected_addons).split(",") From e6f1154b55e9f4ba94748d67e3448569122aac2b Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 03:59:15 +0000 Subject: [PATCH 09/15] remove unncessary expression Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index e33ceb5325..e028f54dbd 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -493,7 +493,7 @@ def _select_prompts_to_display( del prompts_required["add_ons"] if project_name is not None: - if bool(re.match(r"^[\w -]{2,}$", project_name)) is False: + if not re.match(r"^[\w -]{2,}$", project_name): click.secho( "Kedro project names must contain only alphanumeric symbols, spaces, underscores and hyphens and be at least 2 characters long", fg="red", From 20a6204c86167ec8e8ca318caf8b0d20e7ab9bcc Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 04:01:56 +0000 Subject: [PATCH 10/15] rename config -> extra_context Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index e028f54dbd..3469081da2 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -335,7 +335,7 @@ def new( # noqa: too-many-arguments shutil.rmtree(tmpdir, onerror=_remove_readonly) # Obtain config, either from a file or from interactive user prompts. - config = _get_config( + extra_context = _get_config( prompts_required=prompts_required, config_path=config_path, cookiecutter_context=cookiecutter_context, @@ -343,7 +343,7 @@ def new( # noqa: too-many-arguments project_name=project_name, ) - cookiecutter_args = _make_cookiecutter_args(config, checkout, directory) + cookiecutter_args = _make_cookiecutter_args(extra_context, checkout, directory) project_template = fetch_template_based_on_add_ons(template_path, cookiecutter_args) From 14103a8d3a7c3543120397b50706053c6c2a466f Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 04:02:52 +0000 Subject: [PATCH 11/15] _get_config -> _get_extra_context Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 3469081da2..ee1470110c 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -335,7 +335,7 @@ def new( # noqa: too-many-arguments shutil.rmtree(tmpdir, onerror=_remove_readonly) # Obtain config, either from a file or from interactive user prompts. - extra_context = _get_config( + extra_context = _get_extra_context( prompts_required=prompts_required, config_path=config_path, cookiecutter_context=cookiecutter_context, @@ -380,14 +380,14 @@ def list_starters(): ) -def _get_config( +def _get_extra_context( prompts_required: dict, config_path: str, cookiecutter_context: OrderedDict, selected_addons: str, project_name: str, ) -> dict[str, str]: - """Generates a config dictionary to will be passed, based + """Generates a config dictionary that will be passed to cookiecutters as `extra_context`, based on CLI flags, user prompts, or a configuration file. Args: From 39789e66d73cdcb6be0f02d2a420f4739695f898 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 04:03:54 +0000 Subject: [PATCH 12/15] rename all config reference to extra_context Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index ee1470110c..05cdc761e9 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -405,27 +405,27 @@ def _get_extra_context( the prompts_required dictionary, with all the redundant information removed. """ if not prompts_required: - config = {} + extra_context = {} if config_path: - config = _fetch_config_from_file(config_path) - _validate_config_file_inputs(config) + extra_context = _fetch_config_from_file(config_path) + _validate_config_file_inputs(extra_context) elif config_path: - config = _fetch_config_from_file(config_path) - _validate_config_file_against_prompts(config, prompts_required) - _validate_config_file_inputs(config) + extra_context = _fetch_config_from_file(config_path) + _validate_config_file_against_prompts(extra_context, prompts_required) + _validate_config_file_inputs(extra_context) else: - config = _fetch_config_from_user_prompts(prompts_required, cookiecutter_context) + extra_context = _fetch_config_from_user_prompts(prompts_required, cookiecutter_context) add_ons = _get_addons_from_cli_input(selected_addons) if add_ons is not None: - config["add_ons"] = add_ons + extra_context["add_ons"] = add_ons if project_name is not None: - config["project_name"] = project_name + extra_context["project_name"] = project_name - return config + return extra_context def _get_addons_from_cli_input(selected_addons: str) -> str: From 0a4bdbfe74981c8fa4689dc4502363640fd60e6e Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 04:12:05 +0000 Subject: [PATCH 13/15] fix lint Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 9 +++++---- tests/framework/cli/test_starters.py | 20 +------------------- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 05cdc761e9..a600a86ef5 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -415,9 +415,11 @@ def _get_extra_context( _validate_config_file_against_prompts(extra_context, prompts_required) _validate_config_file_inputs(extra_context) else: - extra_context = _fetch_config_from_user_prompts(prompts_required, cookiecutter_context) + extra_context = _fetch_config_from_user_prompts( + prompts_required, cookiecutter_context + ) - add_ons = _get_addons_from_cli_input(selected_addons) + add_ons = _convert_addon_names_to_numbers(selected_addons) if add_ons is not None: extra_context["add_ons"] = add_ons @@ -428,7 +430,7 @@ def _get_extra_context( return extra_context -def _get_addons_from_cli_input(selected_addons: str) -> str: +def _convert_addon_names_to_numbers(selected_addons: str) -> str: """Prepares add-on selection from the CLI input to the correct format to be put in the project configuration, if it exists. Replaces add-on strings with the corresponding prompt number. @@ -452,7 +454,6 @@ def _get_addons_from_cli_input(selected_addons: str) -> str: return ",".join(addons) - def _select_prompts_to_display( prompts_required: dict, selected_addons: str, project_name: str ) -> dict: diff --git a/tests/framework/cli/test_starters.py b/tests/framework/cli/test_starters.py index e7f5c6beab..b1893d1d8a 100644 --- a/tests/framework/cli/test_starters.py +++ b/tests/framework/cli/test_starters.py @@ -15,6 +15,7 @@ _OFFICIAL_STARTER_SPECS, TEMPLATE_PATH, KedroStarterSpec, + _convert_addon_names_to_numbers, _parse_add_ons_input, ) @@ -68,25 +69,6 @@ def _make_cli_prompt_input_without_name( return "\n".join([add_ons, repo_name, python_package]) -def _convert_addon_names_to_numbers(selected_addons: str): - string_to_number = { - "lint": "1", - "test": "2", - "log": "3", - "docs": "4", - "data": "5", - "pyspark": "6", - } - - addons = selected_addons.split(",") - for i in range(len(addons)): - addon = addons[i].strip() - if addon in string_to_number: - addons[i] = string_to_number[addon] - - return ",".join(addons) - - def _get_expected_files(add_ons: str): add_ons_template_files = { "1": 0, From a9bfa650640457c5b27fe112e7ff9e424f9f5a91 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 04:22:20 +0000 Subject: [PATCH 14/15] rename to avoid overwriting loop variable - ruff suggestion Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index a600a86ef5..9febd93fe7 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -448,9 +448,9 @@ def _convert_addon_names_to_numbers(selected_addons: str) -> str: addons = [] for addon in selected_addons.split(","): - addon = addon.strip() - if addon in ADD_ONS_SHORTNAME_TO_NUMBER: - addons.append(ADD_ONS_SHORTNAME_TO_NUMBER[addon]) + addon_short_name = addon.strip() + if addon_short_name in ADD_ONS_SHORTNAME_TO_NUMBER: + addons.append(ADD_ONS_SHORTNAME_TO_NUMBER[addon_short_name]) return ",".join(addons) From 9bdc3b6addc3b966df6deaf8998c1afb7ffd4934 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 3 Nov 2023 16:31:14 +0000 Subject: [PATCH 15/15] refactor based on reviews suggention Signed-off-by: Nok Chan --- kedro/framework/cli/starters.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 9febd93fe7..1442a86224 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -132,10 +132,6 @@ class KedroStarterSpec: # noqa: too-few-public-methods "5": "Data Structure", "6": "Pyspark", } -ADD_ONS_SHORTNAME_TO_NAME = { - short_name: NUMBER_TO_ADD_ONS_NAME[num] - for short_name, num in ADD_ONS_SHORTNAME_TO_NUMBER.items() -} NAME_ARG_HELP = "The name of your new Kedro project." @@ -226,7 +222,7 @@ def _validate_range(start, end): def _validate_selection(add_ons: list[str]): for add_on in add_ons: - if int(add_on) < 1 or int(add_on) > len(NUMBER_TO_ADD_ONS_NAME): + if add_on not in NUMBER_TO_ADD_ONS_NAME: message = f"'{add_on}' is not a valid selection.\nPlease select from the available add-ons: 1, 2, 3, 4, 5, 6." # nosec click.secho(message, fg="red", err=True) sys.exit(1) @@ -387,7 +383,7 @@ def _get_extra_context( selected_addons: str, project_name: str, ) -> dict[str, str]: - """Generates a config dictionary that will be passed to cookiecutters as `extra_context`, based + """Generates a config dictionary that will be passed to cookiecutter as `extra_context`, based on CLI flags, user prompts, or a configuration file. Args: @@ -472,7 +468,7 @@ def _select_prompts_to_display( Returns: the prompts_required dictionary, with all the redundant information removed. """ - valid_addons = list(ADD_ONS_SHORTNAME_TO_NAME) + ["all", "none"] + valid_addons = list(ADD_ONS_SHORTNAME_TO_NUMBER) + ["all", "none"] if selected_addons is not None: addons = re.sub(r"\s", "", selected_addons).split(",")