Rework FUNCTION converter, refactor other converters

snowddl/app/convert.py

@@ -79,11 +79,6 @@ def init_arguments_parser(self):
             "--max-workers", help="Maximum number of workers to resolve objects in parallel", default=None, type=int
         )
         parser.add_argument("--clean", help="Delete existing config files before conversion", default=False, action="store_true")
-        parser.add_argument(
-            "--add-include-files", 
-            help="Write out any source code text (Tasks, Functions, Procedures, Views) to separate files", 
-            default=False, 
-            action="store_true")
 
         # Logging
         parser.add_argument(
@@ -110,6 +105,20 @@ def init_arguments_parser(self):
             action="store_true",
         )
 
+        parser.add_argument(
+            "--convert-function-body-to-file",
+            help="Dump out FUNCTION body to separate files",
+            default=False,
+            action="store_true"
+        )
+
+        parser.add_argument(
+            "--convert-view-text-to-file",
+            help="Dump VIEW text to separate files",
+            default=False,
+            action="store_true"
+        )
+
         return parser
 
     def init_config_path(self):
@@ -162,6 +171,12 @@ def init_settings(self):
         if self.args.get("max_workers"):
             settings.max_workers = int(self.args.get("max_workers"))
 
+        if self.args.get("convert_function_body_to_file"):
+            settings.convert_function_body_to_file = True
+
+        if self.args.get("convert_view_text_to_file"):
+            settings.convert_view_text_to_file = True
+
         return settings
 
     def execute(self):

snowddl/blueprint/data_type.py

@@ -1,86 +1,106 @@
 from enum import Enum
 from re import compile, IGNORECASE
+from typing import List
 
 
 class BaseDataType(Enum):
     NUMBER = {
         "base_name": "NUMBER",
         "number_of_properties": 2,
+        "default_properties": [38, 0],
     }
 
     FLOAT = {
         "base_name": "FLOAT",
         "number_of_properties": 0,
+        "default_properties": [],
     }
 
     BINARY = {
         "base_name": "BINARY",
         "number_of_properties": 1,
+        "default_properties": [8_388_608],
     }
 
     BOOLEAN = {
         "base_name": "BOOLEAN",
         "number_of_properties": 0,
+        "default_properties": [],
     }
 
     VARCHAR = {
         "base_name": "VARCHAR",
         "number_of_properties": 1,
+        "default_properties": [16_777_216],
     }
 
     DATE = {
         "base_name": "DATE",
         "number_of_properties": 0,
+        "default_properties": [],
     }
 
     TIME = {
         "base_name": "TIME",
         "number_of_properties": 1,
+        "default_properties": [9],
     }
 
     TIMESTAMP_LTZ = {
         "base_name": "TIMESTAMP_LTZ",
         "number_of_properties": 1,
+        "default_properties": [9],
     }
 
     TIMESTAMP_NTZ = {
         "base_name": "TIMESTAMP_NTZ",
         "number_of_properties": 1,
+        "default_properties": [9],
     }
 
     TIMESTAMP_TZ = {
         "base_name": "TIMESTAMP_TZ",
         "number_of_properties": 1,
+        "default_properties": [9],
     }
 
     VARIANT = {
         "base_name": "VARIANT",
         "number_of_properties": 0,
+        "default_properties": [],
     }
 
     OBJECT = {
         "base_name": "OBJECT",
         "number_of_properties": 0,
+        "default_properties": [],
     }
 
     ARRAY = {
         "base_name": "ARRAY",
         "number_of_properties": 0,
+        "default_properties": [],
     }
 
     GEOGRAPHY = {
         "base_name": "GEOGRAPHY",
         "number_of_properties": 0,
+        "default_properties": [],
     }
 
     GEOMETRY = {
         "base_name": "GEOMETRY",
         "number_of_properties": 0,
+        "default_properties": [],
     }
 
     @property
     def number_of_properties(self) -> int:
-        return self.value.get("number_of_properties", 0)
+        return self.value["number_of_properties"]
+
+    @property
+    def default_properties(self) -> List[int]:
+        return self.value["default_properties"]
 
     def __repr__(self):
         return f"<{self.__class__.__name__}.{self.name}>"
@@ -103,6 +123,13 @@ def __init__(self, data_type_str):
         self.val1 = int(m["val1"]) if self.base_type.number_of_properties >= 1 else None
         self.val2 = int(m["val2"]) if self.base_type.number_of_properties >= 2 else None
 
+    @staticmethod
+    def from_base_type(base_type: BaseDataType):
+        if base_type.default_properties:
+            return DataType(f"{base_type.name}({','.join(str(p) for p in base_type.default_properties)})")
+
+        return DataType(base_type.name)
+
     def __str__(self):
         if self.base_type.number_of_properties >= 2:
             return f"{self.base_type.name}({self.val1},{self.val2})"

snowddl/converter/_yaml.py

@@ -13,6 +13,10 @@ class YamlLiteralStr(str):
     pass
 
 
+class YamlIncludeStr(str):
+    pass
+
+
 def folded_str_presenter(dumper: SnowDDLDumper, data):
     return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=">")
 
@@ -21,5 +25,10 @@ def literal_str_presenter(dumper: SnowDDLDumper, data):
     return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
 
 
+def include_str_presenter(dumper: SnowDDLDumper, data):
+    return dumper.represent_scalar("!include", data)
+
+
 add_representer(YamlFoldedStr, folded_str_presenter, Dumper=SnowDDLDumper)
 add_representer(YamlLiteralStr, literal_str_presenter, Dumper=SnowDDLDumper)
+add_representer(YamlIncludeStr, include_str_presenter, Dumper=SnowDDLDumper)

snowddl/converter/abc_converter.py

@@ -101,10 +101,20 @@ def _dump_file(self, file_path: Path, data: Dict, json_schema: dict):
         # Validate JSON schema
         validate(data, json_schema)
 
+        # Create directory if not exist
+        file_path.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
+
         # (Over)write file
         with file_path.open("w", encoding="utf-8") as f:
             dump_all([data], f, Dumper=SnowDDLDumper, sort_keys=False)
 
+    def _dump_code(self, file_path: Path, content: str):
+        # Create directory if not exist
+        file_path.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
+
+        # Write data as plain text
+        file_path.write_text(content)
+
     def _normalise_name(self, name: str):
         return name.lower()
 

snowddl/converter/database.py

@@ -20,7 +20,6 @@ def dump_object(self, row):
             data["comment"] = row["comment"]
 
         object_path = self.base_path / self._normalise_name_with_prefix(row["database"])
-        object_path.mkdir(mode=0o755, parents=True, exist_ok=True)
 
         self._dump_file(object_path / "params.yaml", data, database_json_schema)
         return ConvertResult.DUMP