Implement ability to execute custom code while initializing config; A…

…dd tests for custom code; Add remove_blueprint() function; Adjust regexp for placeholders to allow skipping extra space before and after curly braces; Make certain attributes of TableColumn, ViewColumn optional

snowddl/_config/sample02_01/__custom/01_test.py

@@ -0,0 +1,23 @@
+from snowddl import DataType, Ident, TableBlueprint, TableColumn, SchemaObjectIdent, SnowDDLConfig
+
+
+def handler(config: SnowDDLConfig):
+    # Add custom tables
+    for i in range(1,5):
+        bp = TableBlueprint(
+            full_name=SchemaObjectIdent(config.env_prefix, "test_db", "test_schema", f"custom_table_{i}"),
+            columns=[
+                TableColumn(
+                    name=Ident("id"),
+                    type=DataType("NUMBER(38,0)"),
+                ),
+                TableColumn(
+                    name=Ident("name"),
+                    type=DataType("VARCHAR(255)"),
+                ),
+            ],
+            is_transient=True,
+            comment=f"This table was created programmatically",
+        )
+
+        config.add_blueprint(bp)

snowddl/_config/sample02_01/__custom/02_test.py

@@ -0,0 +1,17 @@
+from snowddl import SchemaObjectIdent, SnowDDLConfig, TableBlueprint, ViewBlueprint
+
+
+def handler(config: SnowDDLConfig):
+    # Add view combining all custom tables
+    parts = []
+
+    for full_name, bp in config.get_blueprints_by_type_and_pattern(TableBlueprint, "test_db.test_schema.custom_table_*"):
+        parts.append(f"SELECT id, name FROM {full_name}")
+
+    bp = ViewBlueprint(
+        full_name=SchemaObjectIdent(config.env_prefix, "test_db", "test_schema", "custom_view"),
+        text="\nUNION ALL\n".join(parts),
+        comment=f"This view was created programmatically",
+    )
+
+    config.add_blueprint(bp)

snowddl/app/base.py

@@ -1,4 +1,5 @@
 from argparse import ArgumentParser, HelpFormatter
+from importlib.util import module_from_spec, spec_from_file_location
 from json import loads as json_loads
 from json.decoder import JSONDecodeError
 from logging import getLogger, Formatter, StreamHandler
@@ -263,6 +264,22 @@ def init_config(self):
             self.output_config_errors(config)
             exit(1)
 
+        # Custom programmatically generated blueprints and config adjustments
+        for module_path in sorted(self.config_path.glob("__custom/*.py")):
+            try:
+                spec = spec_from_file_location(module_path.name, module_path)
+
+                module = module_from_spec(spec)
+                spec.loader.exec_module(module)
+
+                module.handler(config)
+            except Exception as e:
+                config.add_error(module_path, e)
+
+        if config.errors:
+            self.output_config_errors(config)
+            exit(1)
+
         return config
 
     def init_settings(self):

snowddl/blueprint/column.py

@@ -9,23 +9,23 @@ class ExternalTableColumn(BaseModelWithConfig):
     name: Ident
     type: DataType
     expr: str
-    not_null: bool
-    comment: Optional[str]
+    not_null: bool = False
+    comment: Optional[str] = None
 
 
 class TableColumn(BaseModelWithConfig):
     name: Ident
     type: DataType
-    not_null: bool
-    default: Optional[Union[SchemaObjectIdent, str]]
-    expression: Optional[str]
-    collate: Optional[str]
-    comment: Optional[str]
+    not_null: bool = False
+    default: Optional[Union[SchemaObjectIdent, str]] = None
+    expression: Optional[str] = None
+    collate: Optional[str] = None
+    comment: Optional[str] = None
 
 
 class ViewColumn(BaseModelWithConfig):
     name: Ident
-    comment: Optional[str]
+    comment: Optional[str] = None
 
 
 class NameWithType(BaseModelWithConfig):

snowddl/config.py

@@ -55,6 +55,10 @@ def get_placeholder(self, name: str) -> Union[bool, float, int, str]:
     def add_blueprint(self, bp: AbstractBlueprint):
         self.blueprints[bp.__class__][str(bp.full_name)] = bp
 
+    def remove_blueprint(self, bp: AbstractBlueprint):
+        if str(bp.full_name) not in self.blueprints.get(bp.__class__, {}):
+            raise ValueError(f"Blueprint with type [{bp.__class__.__name__}] and name [{bp.full_name}] does not exist in config")
+
     def add_error(self, path: Path, e: Exception):
         self.errors.append(
             {

snowddl/parser/_parsed_file.py

@@ -11,9 +11,9 @@
 
 
 class ParsedFile:
-    placeholder_start = "${{ "
-    placeholder_end = " }}"
-    placeholder_re = compile(r"\${{\s([a-z0-9._-]+)\s}}", IGNORECASE)
+    placeholder_start = "${{"
+    placeholder_end = "}}"
+    placeholder_re = compile(r"\${{\s?([a-z0-9._-]+)\s?}}", IGNORECASE)
 
     def __init__(self, parser: "AbstractParser", path: Path, json_schema: dict):
         self.parser = parser

test/_config/step1/__custom/cu001.py

@@ -0,0 +1,22 @@
+from snowddl import DataType, Ident,  SchemaObjectIdent, SnowDDLConfig, TableBlueprint, TableColumn
+
+def handler(config: SnowDDLConfig):
+    # Add some custom tables
+    for i in range(1,5):
+        bp = TableBlueprint(
+            full_name=SchemaObjectIdent(config.env_prefix, "db1", "sc1", f"cu001_tb{i}"),
+            columns=[
+                TableColumn(
+                    name=Ident("id"),
+                    type=DataType("NUMBER(38,0)"),
+                ),
+                TableColumn(
+                    name=Ident("name"),
+                    type=DataType("VARCHAR(255)"),
+                ),
+            ],
+            is_transient=True,
+            comment=f"This table was created programmatically",
+        )
+
+        config.add_blueprint(bp)

test/_config/step1/__custom/cu002.py

@@ -0,0 +1,6 @@
+from snowddl import SnowDDLConfig
+
+
+def handler(config: SnowDDLConfig):
+    # Empty handler, nothing happens here
+    pass

test/_config/step1/db1/sc1/table/cu002_tb1.yaml

@@ -0,0 +1,3 @@
+columns:
+  id: NUMBER(38,0)
+  name: VARCHAR(255)

test/_config/step1/db1/sc1/table/cu002_tb2.yaml

@@ -0,0 +1,3 @@
+columns:
+  id: NUMBER(38,0)
+  name: VARCHAR(255)

test/_config/step2/__custom/cu001.py

@@ -0,0 +1,31 @@
+from snowddl import DataType, Ident, SchemaObjectIdent, SnowDDLConfig, TableBlueprint, TableColumn, ViewBlueprint
+
+
+def handler(config: SnowDDLConfig):
+    # Add some custom tables and corresponding views
+    for i in range(1,4):
+        bp = TableBlueprint(
+            full_name=SchemaObjectIdent(config.env_prefix, "db1", "sc1", f"cu001_tb{i}"),
+            columns=[
+                TableColumn(
+                    name=Ident("id"),
+                    type=DataType("NUMBER(38,0)"),
+                ),
+                TableColumn(
+                    name=Ident("name"),
+                    type=DataType("VARCHAR(255)"),
+                ),
+            ],
+            is_transient=True,
+            comment=f"This table was created programmatically",
+        )
+
+        config.add_blueprint(bp)
+
+        bp = ViewBlueprint(
+            full_name=SchemaObjectIdent(config.env_prefix, "db1", "sc1", f"cu001_vw{i}"),
+            text=f"SELECT id, name FROM cu001_tb{i}",
+            comment=f"This view was created programmatically",
+        )
+
+        config.add_blueprint(bp)

test/_config/step2/__custom/cu002.py

@@ -0,0 +1,6 @@
+from snowddl import SnowDDLConfig, TableBlueprint
+
+
+def handler(config: SnowDDLConfig):
+    for full_name, bp in config.get_blueprints_by_type_and_pattern(TableBlueprint, "db1.sc1.cu002*").items():
+        config.remove_blueprint(bp)

test/_config/step2/db1/sc1/table/cu002_tb1.yaml

@@ -0,0 +1,3 @@
+columns:
+  id: NUMBER(38,0)
+  name: VARCHAR(255)

test/_config/step2/db1/sc1/table/cu002_tb3.yaml

@@ -0,0 +1,3 @@
+columns:
+  id: NUMBER(38,0)
+  name: VARCHAR(255)

test/custom/cu001.py

@@ -0,0 +1,26 @@
+def test_step1(helper):
+    for i in range(1,5):
+        table = helper.show_table("db1", "sc1", f"cu001_tb{i}")
+        assert table is not None
+
+
+def test_step2(helper):
+    for i in range(1,5):
+        table = helper.show_table("db1", "sc1", f"cu001_tb{i}")
+        view = helper.show_view("db1", "sc1", f"cu001_vw{i}")
+
+        if i <= 3:
+            assert table is not None
+            assert view is not None
+        else:
+            assert table is None
+            assert view is None
+
+
+def test_step3(helper):
+    for i in range(1,5):
+        table = helper.show_table("db1", "sc1", f"cu001_tb{i}")
+        view = helper.show_view("db1", "sc1", f"cu001_vw{i}")
+
+        assert table is None
+        assert view is None

test/custom/cu002.py

@@ -0,0 +1,28 @@
+def test_step1(helper):
+    table1 = helper.show_table("db1", "sc1", f"cu002_tb1")
+    table2 = helper.show_table("db1", "sc1", f"cu002_tb2")
+    table3 = helper.show_table("db1", "sc1", f"cu002_tb3")
+
+    assert table1 is not None
+    assert table2 is not None
+    assert table3 is None
+
+
+def test_step2(helper):
+    table1 = helper.show_table("db1", "sc1", f"cu002_tb1")
+    table2 = helper.show_table("db1", "sc1", f"cu002_tb2")
+    table3 = helper.show_table("db1", "sc1", f"cu002_tb3")
+
+    assert table1 is not None
+    assert table2 is None
+    assert table3 is not None
+
+
+def test_step3(helper):
+    table1 = helper.show_table("db1", "sc1", f"cu002_tb1")
+    table2 = helper.show_table("db1", "sc1", f"cu002_tb2")
+    table3 = helper.show_table("db1", "sc1", f"cu002_tb3")
+
+    assert table1 is None
+    assert table2 is None
+    assert table3 is None