From 6e8fe8cf1dd43b8a7cf7ef1b509d314a2e59a7be Mon Sep 17 00:00:00 2001 From: Vitaly Markov Date: Thu, 12 Oct 2023 19:05:52 +0100 Subject: [PATCH] Implement ability to execute custom code while initializing config; Add tests for custom code; Add remove_blueprint() function; Adjust regexp for placeholders to allow skipping extra space before and after curly braces; Make certain attributes of TableColumn, ViewColumn optional --- .../_config/sample02_01/__custom/01_test.py | 23 ++++++++++++++ .../_config/sample02_01/__custom/02_test.py | 17 ++++++++++ snowddl/app/base.py | 17 ++++++++++ snowddl/blueprint/column.py | 16 +++++----- snowddl/config.py | 4 +++ snowddl/parser/_parsed_file.py | 6 ++-- test/_config/step1/__custom/cu001.py | 22 +++++++++++++ test/_config/step1/__custom/cu002.py | 6 ++++ .../step1/db1/sc1/table/cu002_tb1.yaml | 3 ++ .../step1/db1/sc1/table/cu002_tb2.yaml | 3 ++ test/_config/step2/__custom/cu001.py | 31 +++++++++++++++++++ test/_config/step2/__custom/cu002.py | 6 ++++ .../step2/db1/sc1/table/cu002_tb1.yaml | 3 ++ .../step2/db1/sc1/table/cu002_tb3.yaml | 3 ++ test/custom/cu001.py | 26 ++++++++++++++++ test/custom/cu002.py | 28 +++++++++++++++++ 16 files changed, 203 insertions(+), 11 deletions(-) create mode 100644 snowddl/_config/sample02_01/__custom/01_test.py create mode 100644 snowddl/_config/sample02_01/__custom/02_test.py create mode 100644 test/_config/step1/__custom/cu001.py create mode 100644 test/_config/step1/__custom/cu002.py create mode 100644 test/_config/step1/db1/sc1/table/cu002_tb1.yaml create mode 100644 test/_config/step1/db1/sc1/table/cu002_tb2.yaml create mode 100644 test/_config/step2/__custom/cu001.py create mode 100644 test/_config/step2/__custom/cu002.py create mode 100644 test/_config/step2/db1/sc1/table/cu002_tb1.yaml create mode 100644 test/_config/step2/db1/sc1/table/cu002_tb3.yaml create mode 100644 test/custom/cu001.py create mode 100644 test/custom/cu002.py diff --git a/snowddl/_config/sample02_01/__custom/01_test.py b/snowddl/_config/sample02_01/__custom/01_test.py new file mode 100644 index 0000000..4cb5c03 --- /dev/null +++ b/snowddl/_config/sample02_01/__custom/01_test.py @@ -0,0 +1,23 @@ +from snowddl import DataType, Ident, TableBlueprint, TableColumn, SchemaObjectIdent, SnowDDLConfig + + +def handler(config: SnowDDLConfig): + # Add custom tables + for i in range(1,5): + bp = TableBlueprint( + full_name=SchemaObjectIdent(config.env_prefix, "test_db", "test_schema", f"custom_table_{i}"), + columns=[ + TableColumn( + name=Ident("id"), + type=DataType("NUMBER(38,0)"), + ), + TableColumn( + name=Ident("name"), + type=DataType("VARCHAR(255)"), + ), + ], + is_transient=True, + comment=f"This table was created programmatically", + ) + + config.add_blueprint(bp) diff --git a/snowddl/_config/sample02_01/__custom/02_test.py b/snowddl/_config/sample02_01/__custom/02_test.py new file mode 100644 index 0000000..68ee38c --- /dev/null +++ b/snowddl/_config/sample02_01/__custom/02_test.py @@ -0,0 +1,17 @@ +from snowddl import SchemaObjectIdent, SnowDDLConfig, TableBlueprint, ViewBlueprint + + +def handler(config: SnowDDLConfig): + # Add view combining all custom tables + parts = [] + + for full_name, bp in config.get_blueprints_by_type_and_pattern(TableBlueprint, "test_db.test_schema.custom_table_*"): + parts.append(f"SELECT id, name FROM {full_name}") + + bp = ViewBlueprint( + full_name=SchemaObjectIdent(config.env_prefix, "test_db", "test_schema", "custom_view"), + text="\nUNION ALL\n".join(parts), + comment=f"This view was created programmatically", + ) + + config.add_blueprint(bp) diff --git a/snowddl/app/base.py b/snowddl/app/base.py index d925bb3..13ce490 100644 --- a/snowddl/app/base.py +++ b/snowddl/app/base.py @@ -1,4 +1,5 @@ from argparse import ArgumentParser, HelpFormatter +from importlib.util import module_from_spec, spec_from_file_location from json import loads as json_loads from json.decoder import JSONDecodeError from logging import getLogger, Formatter, StreamHandler @@ -263,6 +264,22 @@ def init_config(self): self.output_config_errors(config) exit(1) + # Custom programmatically generated blueprints and config adjustments + for module_path in sorted(self.config_path.glob("__custom/*.py")): + try: + spec = spec_from_file_location(module_path.name, module_path) + + module = module_from_spec(spec) + spec.loader.exec_module(module) + + module.handler(config) + except Exception as e: + config.add_error(module_path, e) + + if config.errors: + self.output_config_errors(config) + exit(1) + return config def init_settings(self): diff --git a/snowddl/blueprint/column.py b/snowddl/blueprint/column.py index 0648cf9..666fb9d 100644 --- a/snowddl/blueprint/column.py +++ b/snowddl/blueprint/column.py @@ -9,23 +9,23 @@ class ExternalTableColumn(BaseModelWithConfig): name: Ident type: DataType expr: str - not_null: bool - comment: Optional[str] + not_null: bool = False + comment: Optional[str] = None class TableColumn(BaseModelWithConfig): name: Ident type: DataType - not_null: bool - default: Optional[Union[SchemaObjectIdent, str]] - expression: Optional[str] - collate: Optional[str] - comment: Optional[str] + not_null: bool = False + default: Optional[Union[SchemaObjectIdent, str]] = None + expression: Optional[str] = None + collate: Optional[str] = None + comment: Optional[str] = None class ViewColumn(BaseModelWithConfig): name: Ident - comment: Optional[str] + comment: Optional[str] = None class NameWithType(BaseModelWithConfig): diff --git a/snowddl/config.py b/snowddl/config.py index d3580f3..40e1e53 100644 --- a/snowddl/config.py +++ b/snowddl/config.py @@ -55,6 +55,10 @@ def get_placeholder(self, name: str) -> Union[bool, float, int, str]: def add_blueprint(self, bp: AbstractBlueprint): self.blueprints[bp.__class__][str(bp.full_name)] = bp + def remove_blueprint(self, bp: AbstractBlueprint): + if str(bp.full_name) not in self.blueprints.get(bp.__class__, {}): + raise ValueError(f"Blueprint with type [{bp.__class__.__name__}] and name [{bp.full_name}] does not exist in config") + def add_error(self, path: Path, e: Exception): self.errors.append( { diff --git a/snowddl/parser/_parsed_file.py b/snowddl/parser/_parsed_file.py index f8460be..5ea3af8 100644 --- a/snowddl/parser/_parsed_file.py +++ b/snowddl/parser/_parsed_file.py @@ -11,9 +11,9 @@ class ParsedFile: - placeholder_start = "${{ " - placeholder_end = " }}" - placeholder_re = compile(r"\${{\s([a-z0-9._-]+)\s}}", IGNORECASE) + placeholder_start = "${{" + placeholder_end = "}}" + placeholder_re = compile(r"\${{\s?([a-z0-9._-]+)\s?}}", IGNORECASE) def __init__(self, parser: "AbstractParser", path: Path, json_schema: dict): self.parser = parser diff --git a/test/_config/step1/__custom/cu001.py b/test/_config/step1/__custom/cu001.py new file mode 100644 index 0000000..5afe62e --- /dev/null +++ b/test/_config/step1/__custom/cu001.py @@ -0,0 +1,22 @@ +from snowddl import DataType, Ident, SchemaObjectIdent, SnowDDLConfig, TableBlueprint, TableColumn + +def handler(config: SnowDDLConfig): + # Add some custom tables + for i in range(1,5): + bp = TableBlueprint( + full_name=SchemaObjectIdent(config.env_prefix, "db1", "sc1", f"cu001_tb{i}"), + columns=[ + TableColumn( + name=Ident("id"), + type=DataType("NUMBER(38,0)"), + ), + TableColumn( + name=Ident("name"), + type=DataType("VARCHAR(255)"), + ), + ], + is_transient=True, + comment=f"This table was created programmatically", + ) + + config.add_blueprint(bp) diff --git a/test/_config/step1/__custom/cu002.py b/test/_config/step1/__custom/cu002.py new file mode 100644 index 0000000..7522815 --- /dev/null +++ b/test/_config/step1/__custom/cu002.py @@ -0,0 +1,6 @@ +from snowddl import SnowDDLConfig + + +def handler(config: SnowDDLConfig): + # Empty handler, nothing happens here + pass diff --git a/test/_config/step1/db1/sc1/table/cu002_tb1.yaml b/test/_config/step1/db1/sc1/table/cu002_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step1/db1/sc1/table/cu002_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step1/db1/sc1/table/cu002_tb2.yaml b/test/_config/step1/db1/sc1/table/cu002_tb2.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step1/db1/sc1/table/cu002_tb2.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step2/__custom/cu001.py b/test/_config/step2/__custom/cu001.py new file mode 100644 index 0000000..54f08f0 --- /dev/null +++ b/test/_config/step2/__custom/cu001.py @@ -0,0 +1,31 @@ +from snowddl import DataType, Ident, SchemaObjectIdent, SnowDDLConfig, TableBlueprint, TableColumn, ViewBlueprint + + +def handler(config: SnowDDLConfig): + # Add some custom tables and corresponding views + for i in range(1,4): + bp = TableBlueprint( + full_name=SchemaObjectIdent(config.env_prefix, "db1", "sc1", f"cu001_tb{i}"), + columns=[ + TableColumn( + name=Ident("id"), + type=DataType("NUMBER(38,0)"), + ), + TableColumn( + name=Ident("name"), + type=DataType("VARCHAR(255)"), + ), + ], + is_transient=True, + comment=f"This table was created programmatically", + ) + + config.add_blueprint(bp) + + bp = ViewBlueprint( + full_name=SchemaObjectIdent(config.env_prefix, "db1", "sc1", f"cu001_vw{i}"), + text=f"SELECT id, name FROM cu001_tb{i}", + comment=f"This view was created programmatically", + ) + + config.add_blueprint(bp) diff --git a/test/_config/step2/__custom/cu002.py b/test/_config/step2/__custom/cu002.py new file mode 100644 index 0000000..f1bb67b --- /dev/null +++ b/test/_config/step2/__custom/cu002.py @@ -0,0 +1,6 @@ +from snowddl import SnowDDLConfig, TableBlueprint + + +def handler(config: SnowDDLConfig): + for full_name, bp in config.get_blueprints_by_type_and_pattern(TableBlueprint, "db1.sc1.cu002*").items(): + config.remove_blueprint(bp) diff --git a/test/_config/step2/db1/sc1/table/cu002_tb1.yaml b/test/_config/step2/db1/sc1/table/cu002_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step2/db1/sc1/table/cu002_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step2/db1/sc1/table/cu002_tb3.yaml b/test/_config/step2/db1/sc1/table/cu002_tb3.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step2/db1/sc1/table/cu002_tb3.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/custom/cu001.py b/test/custom/cu001.py new file mode 100644 index 0000000..ab326ef --- /dev/null +++ b/test/custom/cu001.py @@ -0,0 +1,26 @@ +def test_step1(helper): + for i in range(1,5): + table = helper.show_table("db1", "sc1", f"cu001_tb{i}") + assert table is not None + + +def test_step2(helper): + for i in range(1,5): + table = helper.show_table("db1", "sc1", f"cu001_tb{i}") + view = helper.show_view("db1", "sc1", f"cu001_vw{i}") + + if i <= 3: + assert table is not None + assert view is not None + else: + assert table is None + assert view is None + + +def test_step3(helper): + for i in range(1,5): + table = helper.show_table("db1", "sc1", f"cu001_tb{i}") + view = helper.show_view("db1", "sc1", f"cu001_vw{i}") + + assert table is None + assert view is None diff --git a/test/custom/cu002.py b/test/custom/cu002.py new file mode 100644 index 0000000..27b0045 --- /dev/null +++ b/test/custom/cu002.py @@ -0,0 +1,28 @@ +def test_step1(helper): + table1 = helper.show_table("db1", "sc1", f"cu002_tb1") + table2 = helper.show_table("db1", "sc1", f"cu002_tb2") + table3 = helper.show_table("db1", "sc1", f"cu002_tb3") + + assert table1 is not None + assert table2 is not None + assert table3 is None + + +def test_step2(helper): + table1 = helper.show_table("db1", "sc1", f"cu002_tb1") + table2 = helper.show_table("db1", "sc1", f"cu002_tb2") + table3 = helper.show_table("db1", "sc1", f"cu002_tb3") + + assert table1 is not None + assert table2 is None + assert table3 is not None + + +def test_step3(helper): + table1 = helper.show_table("db1", "sc1", f"cu002_tb1") + table2 = helper.show_table("db1", "sc1", f"cu002_tb2") + table3 = helper.show_table("db1", "sc1", f"cu002_tb3") + + assert table1 is None + assert table2 is None + assert table3 is None