Semantic model configs - enable/disable + groups (dbt-labs#8502)

* WIP

* WIP

* get group and enabled added

* changelog

* cleanup

* getting measure lookup working

* missed file

* get project level working

* fix last test

* add groups to config tests

* more group tests

* fix path

* clean up manifest.py

* update error message

* fix test assert

* remove extra check

* resolve conflicts in manaifest

* update manifest

* resolve conflict

* add alias

.changes/unreleased/Features-20230828-092100.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support configuration of semantic models with the addition of enable/disable and group enablement.
+time: 2023-08-28T09:21:00.551633-05:00
+custom:
+  Author: emmyoop
+  Issue: "7968"

core/dbt/config/project.py

@@ -426,6 +426,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         sources: Dict[str, Any]
         tests: Dict[str, Any]
         metrics: Dict[str, Any]
+        semantic_models: Dict[str, Any]
         exposures: Dict[str, Any]
         vars_value: VarProvider
 
@@ -436,6 +437,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         sources = cfg.sources
         tests = cfg.tests
         metrics = cfg.metrics
+        semantic_models = cfg.semantic_models
         exposures = cfg.exposures
         if cfg.vars is None:
             vars_dict: Dict[str, Any] = {}
@@ -492,6 +494,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
             sources=sources,
             tests=tests,
             metrics=metrics,
+            semantic_models=semantic_models,
             exposures=exposures,
             vars=vars_value,
             config_version=cfg.config_version,
@@ -598,6 +601,7 @@ class Project:
     sources: Dict[str, Any]
     tests: Dict[str, Any]
     metrics: Dict[str, Any]
+    semantic_models: Dict[str, Any]
     exposures: Dict[str, Any]
     vars: VarProvider
     dbt_version: List[VersionSpecifier]
@@ -673,6 +677,7 @@ def to_project_config(self, with_packages=False):
                 "sources": self.sources,
                 "tests": self.tests,
                 "metrics": self.metrics,
+                "semantic-models": self.semantic_models,
                 "exposures": self.exposures,
                 "vars": self.vars.to_dict(),
                 "require-dbt-version": [v.to_version_string() for v in self.dbt_version],

core/dbt/config/runtime.py

@@ -167,6 +167,7 @@ def from_parts(
             sources=project.sources,
             tests=project.tests,
             metrics=project.metrics,
+            semantic_models=project.semantic_models,
             exposures=project.exposures,
             vars=project.vars,
             config_version=project.config_version,
@@ -322,6 +323,7 @@ def get_resource_config_paths(self) -> Dict[str, PathSet]:
             "sources": self._get_config_paths(self.sources),
             "tests": self._get_config_paths(self.tests),
             "metrics": self._get_config_paths(self.metrics),
+            "semantic_models": self._get_config_paths(self.semantic_models),
             "exposures": self._get_config_paths(self.exposures),
         }
 

core/dbt/context/context_config.py

@@ -45,6 +45,8 @@ def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]:
             model_configs = unrendered.get("tests")
         elif resource_type == NodeType.Metric:
             model_configs = unrendered.get("metrics")
+        elif resource_type == NodeType.SemanticModel:
+            model_configs = unrendered.get("semantic_models")
         elif resource_type == NodeType.Exposure:
             model_configs = unrendered.get("exposures")
         else:
@@ -70,6 +72,8 @@ def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]:
             model_configs = self.project.tests
         elif resource_type == NodeType.Metric:
             model_configs = self.project.metrics
+        elif resource_type == NodeType.SemanticModel:
+            model_configs = self.project.semantic_models
         elif resource_type == NodeType.Exposure:
             model_configs = self.project.exposures
         else:

core/dbt/contracts/graph/manifest.py

@@ -331,18 +331,29 @@ def populate(self, manifest: "Manifest"):
         """Populate storage with all the measure + package paths to the Manifest's SemanticModels"""
         for semantic_model in manifest.semantic_models.values():
             self.add(semantic_model=semantic_model)
+        for disabled in manifest.disabled.values():
+            for node in disabled:
+                if isinstance(node, SemanticModel):
+                    self.add(semantic_model=node)
 
     def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SemanticModel:
         """Tries to get a SemanticModel from the Manifest"""
-        semantic_model = manifest.semantic_models.get(unique_id)
-        if semantic_model is None:
+        enabled_semantic_model: Optional[SemanticModel] = manifest.semantic_models.get(unique_id)
+        disabled_semantic_model: Optional[List] = manifest.disabled.get(unique_id)
+
+        if isinstance(enabled_semantic_model, SemanticModel):
+            return enabled_semantic_model
+        elif disabled_semantic_model is not None and isinstance(
+            disabled_semantic_model[0], SemanticModel
+        ):
+            return disabled_semantic_model[0]
+        else:
             raise dbt.exceptions.DbtInternalError(
                 f"Semantic model `{unique_id}` found in cache but not found in manifest"
             )
-        return semantic_model
 
 
-# This handles both models/seeds/snapshots and sources/metrics/exposures
+# This handles both models/seeds/snapshots and sources/metrics/exposures/semantic_models
 class DisabledLookup(dbtClassMixin):
     def __init__(self, manifest: "Manifest"):
         self.storage: Dict[str, Dict[PackageName, List[Any]]] = {}
@@ -927,6 +938,7 @@ def build_group_map(self):
         groupable_nodes = list(
             chain(
                 self.nodes.values(),
+                self.semantic_models.values(),
                 self.metrics.values(),
             )
         )
@@ -1056,8 +1068,7 @@ def resolve_refs(
 
         return resolved_refs
 
-    # Called by dbt.parser.manifest._process_refs_for_exposure, _process_refs_for_metric,
-    # and dbt.parser.manifest._process_refs_for_node
+    # Called by dbt.parser.manifest._process_refs & ManifestLoader.check_for_model_deprecations
     def resolve_ref(
         self,
         source_node: GraphMemberNode,
@@ -1156,6 +1167,7 @@ def resolve_semantic_model_for_measure(
             semantic_model = self.semantic_model_by_measure_lookup.find(
                 target_measure_name, pkg, self
             )
+            # need to return it even if it's disabled so know it's not fully missing
             if semantic_model is not None:
                 return semantic_model
 
@@ -1359,6 +1371,8 @@ def add_disabled(self, source_file: AnySourceFile, node: ResultNode, test_from=N
                 source_file.add_test(node.unique_id, test_from)
             if isinstance(node, Metric):
                 source_file.metrics.append(node.unique_id)
+            if isinstance(node, SemanticModel):
+                source_file.semantic_models.append(node.unique_id)
             if isinstance(node, Exposure):
                 source_file.exposures.append(node.unique_id)
         else:

core/dbt/contracts/graph/model_config.py

@@ -378,12 +378,19 @@ def finalize_and_validate(self: T) -> T:
 @dataclass
 class SemanticModelConfig(BaseConfig):
     enabled: bool = True
+    group: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
 
 
 @dataclass
 class MetricConfig(BaseConfig):
     enabled: bool = True
-    group: Optional[str] = None
+    group: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
 
 
 @dataclass
@@ -635,6 +642,7 @@ def finalize_and_validate(self):
 
 RESOURCE_TYPES: Dict[NodeType, Type[BaseConfig]] = {
     NodeType.Metric: MetricConfig,
+    NodeType.SemanticModel: SemanticModelConfig,
     NodeType.Exposure: ExposureConfig,
     NodeType.Source: SourceConfig,
     NodeType.Seed: SeedConfig,

core/dbt/contracts/graph/nodes.py

@@ -1578,7 +1578,9 @@ class SemanticModel(GraphNode):
     refs: List[RefArgs] = field(default_factory=list)
     created_at: float = field(default_factory=lambda: time.time())
     config: SemanticModelConfig = field(default_factory=SemanticModelConfig)
+    unrendered_config: Dict[str, Any] = field(default_factory=dict)
     primary_entity: Optional[str] = None
+    group: Optional[str] = None
 
     @property
     def entity_references(self) -> List[LinkableElementReference]:

core/dbt/contracts/graph/unparsed.py

@@ -711,6 +711,7 @@ class UnparsedDimension(dbtClassMixin):
 class UnparsedSemanticModel(dbtClassMixin):
     name: str
     model: str  # looks like "ref(...)"
+    config: Dict[str, Any] = field(default_factory=dict)
     description: Optional[str] = None
     defaults: Optional[Defaults] = None
     entities: List[UnparsedEntity] = field(default_factory=list)

core/dbt/contracts/project.py

@@ -214,6 +214,7 @@ class Project(dbtClassMixin, Replaceable):
     sources: Dict[str, Any] = field(default_factory=dict)
     tests: Dict[str, Any] = field(default_factory=dict)
     metrics: Dict[str, Any] = field(default_factory=dict)
+    semantic_models: Dict[str, Any] = field(default_factory=dict)
     exposures: Dict[str, Any] = field(default_factory=dict)
     vars: Optional[Dict[str, Any]] = field(
         default=None,
@@ -249,6 +250,7 @@ class Config(dbtMashConfig):
             "require_dbt_version": "require-dbt-version",
             "query_comment": "query-comment",
             "restrict_access": "restrict-access",
+            "semantic_models": "semantic-models",
         }
 
     @classmethod

core/dbt/graph/selector.py

@@ -169,7 +169,8 @@ def _is_graph_member(self, unique_id: UniqueId) -> bool:
             metric = self.manifest.metrics[unique_id]
             return metric.config.enabled
         elif unique_id in self.manifest.semantic_models:
-            return True
+            semantic_model = self.manifest.semantic_models[unique_id]
+            return semantic_model.config.enabled
         node = self.manifest.nodes[unique_id]
 
         if self.include_empty_nodes:

core/dbt/parser/manifest.py

@@ -95,6 +95,7 @@
     Exposure,
     Metric,
     SeedNode,
+    SemanticModel,
     ManifestNode,
     ResultNode,
     ModelNode,
@@ -1220,11 +1221,16 @@ def check_valid_group_config(self):
         for metric in manifest.metrics.values():
             self.check_valid_group_config_node(metric, group_names)
 
+        for semantic_model in manifest.semantic_models.values():
+            self.check_valid_group_config_node(semantic_model, group_names)
+
         for node in manifest.nodes.values():
             self.check_valid_group_config_node(node, group_names)
 
     def check_valid_group_config_node(
-        self, groupable_node: Union[Metric, ManifestNode], valid_group_names: Set[str]
+        self,
+        groupable_node: Union[Metric, SemanticModel, ManifestNode],
+        valid_group_names: Set[str],
     ):
         groupable_node_group = groupable_node.group
         if groupable_node_group and groupable_node_group not in valid_group_names:
@@ -1493,6 +1499,11 @@ def _process_metric_node(
                 f"A semantic model having a measure `{metric.type_params.measure.name}` does not exist but was referenced.",
                 node=metric,
             )
+        if target_semantic_model.config.enabled is False:
+            raise dbt.exceptions.ParsingError(
+                f"The measure `{metric.type_params.measure.name}` is referenced on disabled semantic model `{target_semantic_model.name}`.",
+                node=metric,
+            )
 
         metric.depends_on.add_node(target_semantic_model.unique_id)
 

core/dbt/parser/schema_yaml_readers.py

@@ -522,6 +522,30 @@ def _create_metric(self, measure: UnparsedMeasure, enabled: bool) -> None:
         parser = MetricParser(self.schema_parser, yaml=self.yaml)
         parser.parse_metric(unparsed=unparsed_metric, generated=True)
 
+    def _generate_semantic_model_config(
+        self, target: UnparsedSemanticModel, fqn: List[str], package_name: str, rendered: bool
+    ):
+        generator: BaseContextConfigGenerator
+        if rendered:
+            generator = ContextConfigGenerator(self.root_project)
+        else:
+            generator = UnrenderedConfigGenerator(self.root_project)
+
+        # configs with precendence set
+        precedence_configs = dict()
+        # first apply semantic model configs
+        precedence_configs.update(target.config)
+
+        config = generator.calculate_node_config(
+            config_call_dict={},
+            fqn=fqn,
+            resource_type=NodeType.SemanticModel,
+            project_name=package_name,
+            base=False,
+            patch_config_dict=precedence_configs,
+        )
+        return config
+
     def parse_semantic_model(self, unparsed: UnparsedSemanticModel):
         package_name = self.project.project_name
         unique_id = f"{NodeType.SemanticModel}.{package_name}.{unparsed.name}"
@@ -530,6 +554,22 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel):
         fqn = self.schema_parser.get_fqn_prefix(path)
         fqn.append(unparsed.name)
 
+        config = self._generate_semantic_model_config(
+            target=unparsed,
+            fqn=fqn,
+            package_name=package_name,
+            rendered=True,
+        )
+
+        config = config.finalize_and_validate()
+
+        unrendered_config = self._generate_semantic_model_config(
+            target=unparsed,
+            fqn=fqn,
+            package_name=package_name,
+            rendered=False,
+        )
+
         parsed = SemanticModel(
             description=unparsed.description,
             fqn=fqn,
@@ -546,6 +586,9 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel):
             dimensions=self._get_dimensions(unparsed.dimensions),
             defaults=unparsed.defaults,
             primary_entity=unparsed.primary_entity,
+            config=config,
+            unrendered_config=unrendered_config,
+            group=config.group,
         )
 
         ctx = generate_parse_semantic_models(
@@ -557,11 +600,15 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel):
 
         if parsed.model is not None:
             model_ref = "{{ " + parsed.model + " }}"
-            # This sets the "refs" in the SemanticModel from the MetricRefResolver in context/providers.py
+            # This sets the "refs" in the SemanticModel from the SemanticModelRefResolver in context/providers.py
             get_rendered(model_ref, ctx, parsed)
 
-        # No ability to disable a semantic model at this time
-        self.manifest.add_semantic_model(self.yaml.file, parsed)
+        # if the semantic model is disabled we do not want it included in the manifest,
+        # only in the disabled dict
+        if parsed.config.enabled:
+            self.manifest.add_semantic_model(self.yaml.file, parsed)
+        else:
+            self.manifest.add_disabled(self.yaml.file, parsed)
 
         # Create a metric for each measure with `create_metric = True`
         for measure in unparsed.measures: