Ct 201 source config functionality

core/dbt/context/context_config.py

@@ -182,6 +182,7 @@ def initial_result(self, resource_type: NodeType, base: bool) -> C:
 
     def _update_from_config(self, result: C, partial: Dict[str, Any], validate: bool = False) -> C:
         translated = self._active_project.credentials.translate_aliases(partial)
+        # breakpoint()
         return result.update_from(
             translated, self._active_project.credentials.type, validate=validate
         )
@@ -203,6 +204,7 @@ def calculate_node_config_dict(
             base=base,
             patch_config_dict=patch_config_dict,
         )
+        # breakpoint()
         finalized = config.finalize_and_validate()
         return finalized.to_dict(omit_none=True)
 

core/dbt/contracts/graph/manifest.py

@@ -453,7 +453,7 @@ class Disabled(Generic[D]):
 
 def _update_into(dest: MutableMapping[str, T], new_item: T):
     """Update dest to overwrite whatever is at dest[new_item.unique_id] with
-    new_itme. There must be an existing value to overwrite, and the two nodes
+    new_item. There must be an existing value to overwrite, and the two nodes
     must have the same original file path.
     """
     unique_id = new_item.unique_id

core/dbt/contracts/graph/model_config.py

@@ -7,7 +7,7 @@
     ValidationError,
     register_pattern,
 )
-from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed
+from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed, FreshnessThreshold, Quoting
 from dbt.exceptions import InternalException, CompilationException
 from dbt.contracts.util import Replaceable, list_str
 from dbt import hooks
@@ -256,7 +256,7 @@ def same_contents(cls, unrendered: Dict[str, Any], other: Dict[str, Any]) -> boo
     # 'meta' moved here from node
     mergebehavior = {
         "append": ["pre-hook", "pre_hook", "post-hook", "post_hook", "tags"],
-        "update": ["quoting", "column_types", "meta"],
+        "update": ["quoting", "column_types", "meta", "freshness"],
     }
 
     @classmethod
@@ -332,9 +332,49 @@ def replace(self, **kwargs):
         return self.from_dict(dct)
 
 
+# TODO: determine what all the metadata should be?
 @dataclass
 class SourceConfig(BaseConfig):
     enabled: bool = True
+    # these fields are included in serialized output, but are not part of
+    # config comparison (they are part of database_representation)
+    # TODO: confirm these
+    loader: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    loaded_at_field: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    schema: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    database: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    identifer: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    tags: Union[List[str], str] = field(
+        default_factory=list_str,
+        metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude),
+    )
+    meta: Dict[str, Any] = field(  # TODO: blank?
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    quoting: Quoting = field(  # TODO: blank
+        default_factory=Quoting,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    freshness: FreshnessThreshold = field(  # TODO: blank
+        default_factory=FreshnessThreshold,
+        metadata=MergeBehavior.Update.meta(),
+    )
 
 
 @dataclass

core/dbt/contracts/graph/parsed.py

@@ -688,7 +688,7 @@ def same_contents(self, old: Optional["ParsedSourceDefinition"]) -> bool:
         return (
             self.same_database_representation(old)
             and self.same_fqn(old)
-            and self.same_config(old)
+            and self.same_config(old)  # TODO: ct-201 what actually constitues a change?
             and self.same_quoting(old)
             and self.same_freshness(old)
             and self.same_external(old)

core/dbt/contracts/graph/unparsed.py

@@ -241,7 +241,7 @@ class Quoting(dbtClassMixin, Mergeable):
 
 
 @dataclass
-class UnparsedSourceTableDefinition(HasColumnTests, HasTests):
+class UnparsedSourceTableDefinition(HasColumnTests, HasConfig, HasTests):
     loaded_at_field: Optional[str] = None
     identifier: Optional[str] = None
     quoting: Quoting = field(default_factory=Quoting)
@@ -257,8 +257,8 @@ def __post_serialize__(self, dct):
 
 
 @dataclass
-class UnparsedSourceDefinition(dbtClassMixin, Replaceable):
-    name: str
+class UnparsedSourceDefinition(dbtClassMixin, Replaceable, HasConfig):
+    name: str = ""
     description: str = ""
     meta: Dict[str, Any] = field(default_factory=dict)
     database: Optional[str] = None
@@ -269,7 +269,6 @@ class UnparsedSourceDefinition(dbtClassMixin, Replaceable):
     loaded_at_field: Optional[str] = None
     tables: List[UnparsedSourceTableDefinition] = field(default_factory=list)
     tags: List[str] = field(default_factory=list)
-    config: Dict[str, Any] = field(default_factory=dict)
 
     @property
     def yaml_key(self) -> "str":
@@ -284,7 +283,7 @@ def __post_serialize__(self, dct):
 
 @dataclass
 class SourceTablePatch(dbtClassMixin):
-    name: str
+    name: str = ""
     description: Optional[str] = None
     meta: Optional[Dict[str, Any]] = None
     data_type: Optional[str] = None

core/dbt/parser/schemas.py

@@ -685,9 +685,12 @@ def parse(self) -> List[TestBlock]:
         for data in self.get_key_dicts():
             data = self.project.credentials.translate_aliases(data, recurse=True)
 
+            path = self.yaml.path.original_file_path
+
+            self.normalize_configs_properties(data, path)
             is_override = "overrides" in data
             if is_override:
-                data["path"] = self.yaml.path.original_file_path
+                data["path"] = path
                 patch = self._target_from_dict(SourcePatch, data)
                 assert isinstance(self.yaml.file, SchemaSourceFile)
                 source_file = self.yaml.file
@@ -702,6 +705,23 @@ def parse(self) -> List[TestBlock]:
                 self.add_source_definitions(source)
         return []
 
+    # We want to raise an error if configs and/or properties are in two places, and copy
+    # to/from toplevel to config if necessary
+    def normalize_configs_properties(self, data, path):
+        # TODO: ct201 - update this for all configs
+        if "meta" in data:
+            if "config" in data and "meta" in data["config"]:
+                raise ParsingException(
+                    f"""
+                    In {path}: found meta dictionary in 'config' dictionary and as top-level key.
+                    Remove the top-level key and define it under 'config' dictionary only.
+                """.strip()
+                )
+            else:
+                if "config" not in data:
+                    data["config"] = {}
+                data["config"]["meta"] = data.pop("meta")
+
     def add_source_definitions(self, source: UnparsedSourceDefinition) -> None:
         original_file_path = self.yaml.path.original_file_path
         fqn_path = self.yaml.path.relative_path
@@ -724,6 +744,7 @@ def add_source_definitions(self, source: UnparsedSourceDefinition) -> None:
                 unique_id=unique_id,
                 resource_type=NodeType.Source,
                 fqn=fqn,
+                # config=config, #TODO ?
             )
             self.manifest.add_source(self.yaml.file, source_def)
 

core/dbt/parser/sources.py

@@ -140,12 +140,14 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> ParsedSourceDefinit
             fqn=target.fqn,
             rendered=True,
             project_name=target.package_name,
+            config_call_dict=table.config,
         )
 
         unrendered_config = self._generate_source_config(
             fqn=target.fqn,
             rendered=False,
             project_name=target.package_name,
+            config_call_dict=table.config,
         )
 
         if not isinstance(config, SourceConfig):
@@ -261,15 +263,17 @@ def parse_source_test(
         )
         return node
 
-    def _generate_source_config(self, fqn: List[str], rendered: bool, project_name: str):
+    def _generate_source_config(
+        self, fqn: List[str], rendered: bool, project_name: str, config_call_dict: Dict[str, Any]
+    ):
         generator: BaseContextConfigGenerator
         if rendered:
             generator = ContextConfigGenerator(self.root_project)
         else:
             generator = UnrenderedConfigGenerator(self.root_project)
 
         return generator.calculate_node_config(
-            config_call_dict={},
+            config_call_dict=config_call_dict,
             fqn=fqn,
             resource_type=NodeType.Source,
             project_name=project_name,

tests/functional/sources/test_source_configs.py

@@ -12,18 +12,18 @@ def setUp(self):
         pytest.expected_config = SourceConfig(
             enabled=True,
             # TODO: uncomment all this once it's added to SourceConfig, throws error right now
-            # quoting = Quoting(database=False, schema=False, identifier=False, column=False)
-            # freshness = FreshnessThreshold(
-            #     warn_after=Time(count=12, period=TimePeriod.hour),
-            #     error_after=Time(count=24, period=TimePeriod.hour),
-            #     filter=None
-            #     )
-            # loader = "a_loader"
-            # loaded_at_field = some_column
-            # database = custom_database
-            # schema = custom_schema
-            # meta = {'languages': ['python']}
-            # tags = ["important_tag"]
+            quoting=Quoting(database=False, schema=False, identifier=False, column=False),
+            freshness=FreshnessThreshold(
+                warn_after=Time(count=12, period=TimePeriod.hour),
+                error_after=Time(count=24, period=TimePeriod.hour),
+                filter=None,
+            ),
+            loader="a_loader",
+            loaded_at_field="some_column",
+            database="custom_database",
+            schema="custom_schema",
+            meta={"languages": ["python"]},
+            tags=["important_tag"],
         )
 
 
@@ -126,15 +126,14 @@ def test_source_config_yaml_source_level(self, project):
 
 
 # Test enabled config at source table level in yaml file
-# expect fail - not implemented
+# expect pass - implemented
 class TestConfigYamlSourceTable(SourceConfigTests):
     @pytest.fixture(scope="class")
     def models(self):
         return {
             "schema.yml": disabled_source_table__schema_yml,
         }
 
-    @pytest.mark.xfail
     def test_source_config_yaml_source_table(self, project):
         run_dbt(["parse"])
         manifest = get_manifest(project.project_root)
@@ -190,7 +189,7 @@ def test_source_all_configs_dbt_project(self, project):
 sources:
   - name: test_source
     config:
-        enabled: True
+        enabled: False
         quoting:
             database: False
             schema: False
@@ -212,13 +211,13 @@ def test_source_all_configs_dbt_project(self, project):
 
 
 # Test configs other than enabled at sources level in yaml file
-# **currently passes since enabled is all that ends up in the
-# node.config since it's the only thing implemented
+# expect fail - not implemented
 class TestAllConfigsSourceLevel(SourceConfigTests):
     @pytest.fixture(scope="class")
     def models(self):
         return {"schema.yml": configs_source_level__schema_yml}
 
+    @pytest.mark.xfail
     def test_source_all_configs_source_level(self, project):
         run_dbt(["parse"])
         manifest = get_manifest(project.project_root)
@@ -263,13 +262,13 @@ def test_source_all_configs_source_level(self, project):
 
 
 # Test configs other than enabled at source table level in yml file
-# expect fail - not implemented
+# only implemented feature so far!
 class TestSourceAllConfigsSourceTable(SourceConfigTests):
     @pytest.fixture(scope="class")
     def models(self):
         return {"schema.yml": configs_source_table__schema_yml}
 
-    @pytest.mark.xfail
+    # @pytest.mark.xfail
     def test_source_all_configs_source_table(self, project):
         run_dbt(["parse"])
         manifest = get_manifest(project.project_root)
@@ -524,13 +523,13 @@ def check_source_configs_and_properties(self, project):
 
 
 # Check backwards compatibility of setting configs as properties at top level
-# expect pass since the properties don't get copied to the node.config yet so
-# the values match since we haven't build the full SourceConfig yet
+# expect fail - not implemented
 class TestPropertiesAsConfigs(SourceBackwardsCompatibility):
     @pytest.fixture(scope="class")
     def models(self):
         return {"schema.yml": properties_as_configs__schema_yml}
 
+    @pytest.mark.xfail
     def test_source_configs_as_properties(self, project):
         self.check_source_configs_and_properties(project)