From 7ae3de1fa011b231b397073bac7dadb67a56ed87 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 30 Aug 2023 20:18:22 -0500 Subject: [PATCH] Semantic model configs - enable/disable + groups (#8502) * WIP * WIP * get group and enabled added * changelog * cleanup * getting measure lookup working * missed file * get project level working * fix last test * add groups to config tests * more group tests * fix path * clean up manifest.py * update error message * fix test assert * remove extra check * resolve conflicts in manaifest * update manifest * resolve conflict * add alias --- .../unreleased/Features-20230828-092100.yaml | 6 + core/dbt/config/project.py | 5 + core/dbt/config/runtime.py | 2 + core/dbt/context/context_config.py | 4 + core/dbt/contracts/graph/manifest.py | 26 ++- core/dbt/contracts/graph/model_config.py | 10 +- core/dbt/contracts/graph/nodes.py | 2 + core/dbt/contracts/graph/unparsed.py | 1 + core/dbt/contracts/project.py | 2 + core/dbt/graph/selector.py | 3 +- core/dbt/parser/manifest.py | 13 +- core/dbt/parser/schema_yaml_readers.py | 53 ++++- schemas/dbt/manifest/v11.json | 32 ++- tests/functional/semantic_models/fixtures.py | 188 +++++++++++++++- .../test_semantic_model_configs.py | 206 ++++++++++++++++++ .../test_semantic_model_parsing.py | 101 +-------- .../semantic_models/test_semantic_models.py | 6 +- 17 files changed, 546 insertions(+), 114 deletions(-) create mode 100644 .changes/unreleased/Features-20230828-092100.yaml create mode 100644 tests/functional/semantic_models/test_semantic_model_configs.py diff --git a/.changes/unreleased/Features-20230828-092100.yaml b/.changes/unreleased/Features-20230828-092100.yaml new file mode 100644 index 00000000000..e765217bd92 --- /dev/null +++ b/.changes/unreleased/Features-20230828-092100.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support configuration of semantic models with the addition of enable/disable and group enablement. +time: 2023-08-28T09:21:00.551633-05:00 +custom: + Author: emmyoop + Issue: "7968" diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index baa99239e99..ecaa9427603 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -426,6 +426,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": sources: Dict[str, Any] tests: Dict[str, Any] metrics: Dict[str, Any] + semantic_models: Dict[str, Any] exposures: Dict[str, Any] vars_value: VarProvider @@ -436,6 +437,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": sources = cfg.sources tests = cfg.tests metrics = cfg.metrics + semantic_models = cfg.semantic_models exposures = cfg.exposures if cfg.vars is None: vars_dict: Dict[str, Any] = {} @@ -492,6 +494,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": sources=sources, tests=tests, metrics=metrics, + semantic_models=semantic_models, exposures=exposures, vars=vars_value, config_version=cfg.config_version, @@ -598,6 +601,7 @@ class Project: sources: Dict[str, Any] tests: Dict[str, Any] metrics: Dict[str, Any] + semantic_models: Dict[str, Any] exposures: Dict[str, Any] vars: VarProvider dbt_version: List[VersionSpecifier] @@ -673,6 +677,7 @@ def to_project_config(self, with_packages=False): "sources": self.sources, "tests": self.tests, "metrics": self.metrics, + "semantic-models": self.semantic_models, "exposures": self.exposures, "vars": self.vars.to_dict(), "require-dbt-version": [v.to_version_string() for v in self.dbt_version], diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index 8e11b2cd43a..3156aa31878 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -167,6 +167,7 @@ def from_parts( sources=project.sources, tests=project.tests, metrics=project.metrics, + semantic_models=project.semantic_models, exposures=project.exposures, vars=project.vars, config_version=project.config_version, @@ -322,6 +323,7 @@ def get_resource_config_paths(self) -> Dict[str, PathSet]: "sources": self._get_config_paths(self.sources), "tests": self._get_config_paths(self.tests), "metrics": self._get_config_paths(self.metrics), + "semantic_models": self._get_config_paths(self.semantic_models), "exposures": self._get_config_paths(self.exposures), } diff --git a/core/dbt/context/context_config.py b/core/dbt/context/context_config.py index 3cdde7f3b00..f5c32ff0f81 100644 --- a/core/dbt/context/context_config.py +++ b/core/dbt/context/context_config.py @@ -45,6 +45,8 @@ def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]: model_configs = unrendered.get("tests") elif resource_type == NodeType.Metric: model_configs = unrendered.get("metrics") + elif resource_type == NodeType.SemanticModel: + model_configs = unrendered.get("semantic_models") elif resource_type == NodeType.Exposure: model_configs = unrendered.get("exposures") else: @@ -70,6 +72,8 @@ def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]: model_configs = self.project.tests elif resource_type == NodeType.Metric: model_configs = self.project.metrics + elif resource_type == NodeType.SemanticModel: + model_configs = self.project.semantic_models elif resource_type == NodeType.Exposure: model_configs = self.project.exposures else: diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index e3dfe976986..2786529dbb0 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -331,18 +331,29 @@ def populate(self, manifest: "Manifest"): """Populate storage with all the measure + package paths to the Manifest's SemanticModels""" for semantic_model in manifest.semantic_models.values(): self.add(semantic_model=semantic_model) + for disabled in manifest.disabled.values(): + for node in disabled: + if isinstance(node, SemanticModel): + self.add(semantic_model=node) def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SemanticModel: """Tries to get a SemanticModel from the Manifest""" - semantic_model = manifest.semantic_models.get(unique_id) - if semantic_model is None: + enabled_semantic_model: Optional[SemanticModel] = manifest.semantic_models.get(unique_id) + disabled_semantic_model: Optional[List] = manifest.disabled.get(unique_id) + + if isinstance(enabled_semantic_model, SemanticModel): + return enabled_semantic_model + elif disabled_semantic_model is not None and isinstance( + disabled_semantic_model[0], SemanticModel + ): + return disabled_semantic_model[0] + else: raise dbt.exceptions.DbtInternalError( f"Semantic model `{unique_id}` found in cache but not found in manifest" ) - return semantic_model -# This handles both models/seeds/snapshots and sources/metrics/exposures +# This handles both models/seeds/snapshots and sources/metrics/exposures/semantic_models class DisabledLookup(dbtClassMixin): def __init__(self, manifest: "Manifest"): self.storage: Dict[str, Dict[PackageName, List[Any]]] = {} @@ -927,6 +938,7 @@ def build_group_map(self): groupable_nodes = list( chain( self.nodes.values(), + self.semantic_models.values(), self.metrics.values(), ) ) @@ -1056,8 +1068,7 @@ def resolve_refs( return resolved_refs - # Called by dbt.parser.manifest._process_refs_for_exposure, _process_refs_for_metric, - # and dbt.parser.manifest._process_refs_for_node + # Called by dbt.parser.manifest._process_refs & ManifestLoader.check_for_model_deprecations def resolve_ref( self, source_node: GraphMemberNode, @@ -1156,6 +1167,7 @@ def resolve_semantic_model_for_measure( semantic_model = self.semantic_model_by_measure_lookup.find( target_measure_name, pkg, self ) + # need to return it even if it's disabled so know it's not fully missing if semantic_model is not None: return semantic_model @@ -1359,6 +1371,8 @@ def add_disabled(self, source_file: AnySourceFile, node: ResultNode, test_from=N source_file.add_test(node.unique_id, test_from) if isinstance(node, Metric): source_file.metrics.append(node.unique_id) + if isinstance(node, SemanticModel): + source_file.semantic_models.append(node.unique_id) if isinstance(node, Exposure): source_file.exposures.append(node.unique_id) else: diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index abf4951714a..f30ddd8fcc3 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -378,12 +378,19 @@ def finalize_and_validate(self: T) -> T: @dataclass class SemanticModelConfig(BaseConfig): enabled: bool = True + group: Optional[str] = field( + default=None, + metadata=CompareBehavior.Exclude.meta(), + ) @dataclass class MetricConfig(BaseConfig): enabled: bool = True - group: Optional[str] = None + group: Optional[str] = field( + default=None, + metadata=CompareBehavior.Exclude.meta(), + ) @dataclass @@ -635,6 +642,7 @@ def finalize_and_validate(self): RESOURCE_TYPES: Dict[NodeType, Type[BaseConfig]] = { NodeType.Metric: MetricConfig, + NodeType.SemanticModel: SemanticModelConfig, NodeType.Exposure: ExposureConfig, NodeType.Source: SourceConfig, NodeType.Seed: SeedConfig, diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index b25be7c09b8..b6e685d985f 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1578,7 +1578,9 @@ class SemanticModel(GraphNode): refs: List[RefArgs] = field(default_factory=list) created_at: float = field(default_factory=lambda: time.time()) config: SemanticModelConfig = field(default_factory=SemanticModelConfig) + unrendered_config: Dict[str, Any] = field(default_factory=dict) primary_entity: Optional[str] = None + group: Optional[str] = None @property def entity_references(self) -> List[LinkableElementReference]: diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 59eeaadf8fb..e8c5f9d58cf 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -711,6 +711,7 @@ class UnparsedDimension(dbtClassMixin): class UnparsedSemanticModel(dbtClassMixin): name: str model: str # looks like "ref(...)" + config: Dict[str, Any] = field(default_factory=dict) description: Optional[str] = None defaults: Optional[Defaults] = None entities: List[UnparsedEntity] = field(default_factory=list) diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index 137c00d4f51..9e09fd56692 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -214,6 +214,7 @@ class Project(dbtClassMixin, Replaceable): sources: Dict[str, Any] = field(default_factory=dict) tests: Dict[str, Any] = field(default_factory=dict) metrics: Dict[str, Any] = field(default_factory=dict) + semantic_models: Dict[str, Any] = field(default_factory=dict) exposures: Dict[str, Any] = field(default_factory=dict) vars: Optional[Dict[str, Any]] = field( default=None, @@ -249,6 +250,7 @@ class Config(dbtMashConfig): "require_dbt_version": "require-dbt-version", "query_comment": "query-comment", "restrict_access": "restrict-access", + "semantic_models": "semantic-models", } @classmethod diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index edb865ce7fd..f205fa0fb7a 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -169,7 +169,8 @@ def _is_graph_member(self, unique_id: UniqueId) -> bool: metric = self.manifest.metrics[unique_id] return metric.config.enabled elif unique_id in self.manifest.semantic_models: - return True + semantic_model = self.manifest.semantic_models[unique_id] + return semantic_model.config.enabled node = self.manifest.nodes[unique_id] if self.include_empty_nodes: diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 748f32f607c..4398fa75bc7 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -95,6 +95,7 @@ Exposure, Metric, SeedNode, + SemanticModel, ManifestNode, ResultNode, ModelNode, @@ -1220,11 +1221,16 @@ def check_valid_group_config(self): for metric in manifest.metrics.values(): self.check_valid_group_config_node(metric, group_names) + for semantic_model in manifest.semantic_models.values(): + self.check_valid_group_config_node(semantic_model, group_names) + for node in manifest.nodes.values(): self.check_valid_group_config_node(node, group_names) def check_valid_group_config_node( - self, groupable_node: Union[Metric, ManifestNode], valid_group_names: Set[str] + self, + groupable_node: Union[Metric, SemanticModel, ManifestNode], + valid_group_names: Set[str], ): groupable_node_group = groupable_node.group if groupable_node_group and groupable_node_group not in valid_group_names: @@ -1493,6 +1499,11 @@ def _process_metric_node( f"A semantic model having a measure `{metric.type_params.measure.name}` does not exist but was referenced.", node=metric, ) + if target_semantic_model.config.enabled is False: + raise dbt.exceptions.ParsingError( + f"The measure `{metric.type_params.measure.name}` is referenced on disabled semantic model `{target_semantic_model.name}`.", + node=metric, + ) metric.depends_on.add_node(target_semantic_model.unique_id) diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 927bfdb9a9f..032b3a8b99d 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -522,6 +522,30 @@ def _create_metric(self, measure: UnparsedMeasure, enabled: bool) -> None: parser = MetricParser(self.schema_parser, yaml=self.yaml) parser.parse_metric(unparsed=unparsed_metric, generated=True) + def _generate_semantic_model_config( + self, target: UnparsedSemanticModel, fqn: List[str], package_name: str, rendered: bool + ): + generator: BaseContextConfigGenerator + if rendered: + generator = ContextConfigGenerator(self.root_project) + else: + generator = UnrenderedConfigGenerator(self.root_project) + + # configs with precendence set + precedence_configs = dict() + # first apply semantic model configs + precedence_configs.update(target.config) + + config = generator.calculate_node_config( + config_call_dict={}, + fqn=fqn, + resource_type=NodeType.SemanticModel, + project_name=package_name, + base=False, + patch_config_dict=precedence_configs, + ) + return config + def parse_semantic_model(self, unparsed: UnparsedSemanticModel): package_name = self.project.project_name unique_id = f"{NodeType.SemanticModel}.{package_name}.{unparsed.name}" @@ -530,6 +554,22 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel): fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) + config = self._generate_semantic_model_config( + target=unparsed, + fqn=fqn, + package_name=package_name, + rendered=True, + ) + + config = config.finalize_and_validate() + + unrendered_config = self._generate_semantic_model_config( + target=unparsed, + fqn=fqn, + package_name=package_name, + rendered=False, + ) + parsed = SemanticModel( description=unparsed.description, fqn=fqn, @@ -546,6 +586,9 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel): dimensions=self._get_dimensions(unparsed.dimensions), defaults=unparsed.defaults, primary_entity=unparsed.primary_entity, + config=config, + unrendered_config=unrendered_config, + group=config.group, ) ctx = generate_parse_semantic_models( @@ -557,11 +600,15 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel): if parsed.model is not None: model_ref = "{{ " + parsed.model + " }}" - # This sets the "refs" in the SemanticModel from the MetricRefResolver in context/providers.py + # This sets the "refs" in the SemanticModel from the SemanticModelRefResolver in context/providers.py get_rendered(model_ref, ctx, parsed) - # No ability to disable a semantic model at this time - self.manifest.add_semantic_model(self.yaml.file, parsed) + # if the semantic model is disabled we do not want it included in the manifest, + # only in the disabled dict + if parsed.config.enabled: + self.manifest.add_semantic_model(self.yaml.file, parsed) + else: + self.manifest.add_disabled(self.yaml.file, parsed) # Create a metric for each measure with `create_metric = True` for measure in unparsed.measures: diff --git a/schemas/dbt/manifest/v11.json b/schemas/dbt/manifest/v11.json index 5bc21a93886..b133a238369 100644 --- a/schemas/dbt/manifest/v11.json +++ b/schemas/dbt/manifest/v11.json @@ -1835,8 +1835,7 @@ "deprecation_date": { "anyOf": [ { - "type": "string", - "format": "date-time" + "type": "string" }, { "type": "null" @@ -5242,6 +5241,17 @@ "enabled": { "type": "boolean", "default": true + }, + "group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true @@ -5369,6 +5379,12 @@ "config": { "$ref": "#/$defs/SemanticModelConfig" }, + "unrendered_config": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "primary_entity": { "anyOf": [ { @@ -5379,6 +5395,17 @@ } ], "default": null + }, + "group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -5399,6 +5426,7 @@ "title": "WritableManifest", "properties": { "metadata": { + "description": "Metadata about the manifest", "$ref": "#/$defs/ManifestMetadata" }, "nodes": { diff --git a/tests/functional/semantic_models/fixtures.py b/tests/functional/semantic_models/fixtures.py index c9a3c445ff7..12cb3d9d68b 100644 --- a/tests/functional/semantic_models/fixtures.py +++ b/tests/functional/semantic_models/fixtures.py @@ -1,4 +1,4 @@ -metricflow_time_spine_sql = """ +simple_metricflow_time_spine_sql = """ SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day """ @@ -10,6 +10,18 @@ select 3 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at """ +groups_yml = """ +version: 2 + +groups: + - name: some_group + owner: + email: me@gmail.com + - name: some_other_group + owner: + email: me@gmail.com +""" + models_people_metrics_yml = """ version: 2 @@ -24,6 +36,23 @@ my_meta: 'testing' """ +disabled_models_people_metrics_yml = """ +version: 2 + +metrics: + - name: number_of_people + config: + enabled: false + group: some_group + label: "Number of people" + description: Total count of people + type: simple + type_params: + measure: people + meta: + my_meta: 'testing' +""" + semantic_model_people_yml = """ version: 2 @@ -50,3 +79,160 @@ defaults: agg_time_dimension: created_at """ + +enabled_semantic_model_people_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + model: ref('people') + config: + enabled: true + group: some_group + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at +""" + +disabled_semantic_model_people_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + model: ref('people') + config: + enabled: false + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at +""" + +schema_yml = """models: + - name: fct_revenue + description: This is the model fct_revenue. It should be able to use doc blocks + +semantic_models: + - name: revenue + description: This is the revenue semantic model. It should be able to use doc blocks + model: ref('fct_revenue') + + defaults: + agg_time_dimension: ds + + measures: + - name: txn_revenue + expr: revenue + agg: sum + agg_time_dimension: ds + create_metric: true + - name: sum_of_things + expr: 2 + agg: sum + agg_time_dimension: ds + - name: has_revenue + expr: true + agg: sum_boolean + agg_time_dimension: ds + - name: discrete_order_value_p99 + expr: order_total + agg: percentile + agg_time_dimension: ds + agg_params: + percentile: 0.99 + use_discrete_percentile: True + use_approximate_percentile: False + - name: test_agg_params_optional_are_empty + expr: order_total + agg: percentile + agg_time_dimension: ds + agg_params: + percentile: 0.99 + - name: test_non_additive + expr: txn_revenue + agg: sum + non_additive_dimension: + name: ds + window_choice: max + + dimensions: + - name: ds + type: time + expr: created_at + type_params: + time_granularity: day + + entities: + - name: user + type: foreign + expr: user_id + - name: id + type: primary + +metrics: + - name: simple_metric + label: Simple Metric + type: simple + type_params: + measure: sum_of_things +""" + +schema_without_semantic_model_yml = """models: + - name: fct_revenue + description: This is the model fct_revenue. It should be able to use doc blocks +""" + +fct_revenue_sql = """select + 1 as id, + 10 as user_id, + 1000 as revenue, + current_timestamp as created_at""" + +metricflow_time_spine_sql = """ +with days as ( + {{dbt_utils.date_spine('day' + , "to_date('01/01/2000','mm/dd/yyyy')" + , "to_date('01/01/2027','mm/dd/yyyy')" + ) + }} +), + +final as ( + select cast(date_day as date) as date_day + from days +) + +select * +from final +""" diff --git a/tests/functional/semantic_models/test_semantic_model_configs.py b/tests/functional/semantic_models/test_semantic_model_configs.py new file mode 100644 index 00000000000..38009cddbe0 --- /dev/null +++ b/tests/functional/semantic_models/test_semantic_model_configs.py @@ -0,0 +1,206 @@ +import pytest +from dbt.exceptions import ParsingError +from dbt.contracts.graph.model_config import SemanticModelConfig + +from dbt.tests.util import run_dbt, update_config_file, get_manifest + +from tests.functional.semantic_models.fixtures import ( + models_people_sql, + metricflow_time_spine_sql, + semantic_model_people_yml, + disabled_models_people_metrics_yml, + models_people_metrics_yml, + disabled_semantic_model_people_yml, + enabled_semantic_model_people_yml, + groups_yml, +) + + +# Test disabled config at semantic_models level in yaml file +class TestConfigYamlLevel: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": disabled_semantic_model_people_yml, + "people_metrics.yml": disabled_models_people_metrics_yml, + "groups.yml": groups_yml, + } + + def test_yaml_level(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "semantic_model.test.semantic_people" not in manifest.semantic_models + assert "semantic_model.test.semantic_people" in manifest.disabled + + assert "group.test.some_group" in manifest.groups + assert "semantic_model.test.semantic_people" not in manifest.groups + + +# Test disabled config at semantic_models level with a still enabled metric +class TestDisabledConfigYamlLevelEnabledMetric: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": disabled_semantic_model_people_yml, + "people_metrics.yml": models_people_metrics_yml, + "groups.yml": groups_yml, + } + + def test_yaml_level(self, project): + with pytest.raises( + ParsingError, + match="The measure `people` is referenced on disabled semantic model `semantic_people`.", + ): + run_dbt(["parse"]) + + +# Test disabling semantic model config but not metric config in dbt_project.yml +class TestMismatchesConfigProjectLevel: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_people_yml, + "people_metrics.yml": models_people_metrics_yml, + "groups.yml": groups_yml, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "semantic-models": { + "test": { + "enabled": True, + } + } + } + + def test_project_level(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "semantic_model.test.semantic_people" in manifest.semantic_models + assert "group.test.some_group" in manifest.groups + assert manifest.semantic_models["semantic_model.test.semantic_people"].group is None + + new_enabled_config = { + "semantic-models": { + "test": { + "enabled": False, + } + } + } + update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") + with pytest.raises( + ParsingError, + match="The measure `people` is referenced on disabled semantic model `semantic_people`.", + ): + run_dbt(["parse"]) + + +# Test disabling semantic model and metric configs in dbt_project.yml +class TestConfigProjectLevel: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_people_yml, + "people_metrics.yml": models_people_metrics_yml, + "groups.yml": groups_yml, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "semantic-models": { + "test": { + "enabled": True, + } + }, + "metrics": { + "test": { + "enabled": True, + } + }, + } + + def test_project_level(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "semantic_model.test.semantic_people" in manifest.semantic_models + assert "group.test.some_group" in manifest.groups + assert "group.test.some_other_group" in manifest.groups + assert manifest.semantic_models["semantic_model.test.semantic_people"].group is None + + new_group_config = { + "semantic-models": { + "test": { + "group": "some_other_group", + } + }, + } + update_config_file(new_group_config, project.project_root, "dbt_project.yml") + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + + assert "semantic_model.test.semantic_people" in manifest.semantic_models + assert "group.test.some_other_group" in manifest.groups + assert "group.test.some_group" in manifest.groups + assert ( + manifest.semantic_models["semantic_model.test.semantic_people"].group + == "some_other_group" + ) + + new_enabled_config = { + "semantic-models": { + "test": { + "enabled": False, + } + }, + "metrics": { + "test": { + "enabled": False, + } + }, + } + update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + + assert "semantic_model.test.semantic_people" not in manifest.semantic_models + assert "semantic_model.test.semantic_people" in manifest.disabled + + assert "group.test.some_group" in manifest.groups + assert "semantic_model.test.semantic_people" not in manifest.groups + + +# Test inheritence - set configs at project and semantic_model level - expect semantic_model level to win +class TestConfigsInheritence: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": enabled_semantic_model_people_yml, + "people_metrics.yml": models_people_metrics_yml, + "groups.yml": groups_yml, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"semantic-models": {"enabled": False}} + + def test_project_plus_yaml_level(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "semantic_model.test.semantic_people" in manifest.semantic_models + config_test_table = manifest.semantic_models.get( + "semantic_model.test.semantic_people" + ).config + + assert isinstance(config_test_table, SemanticModelConfig) diff --git a/tests/functional/semantic_models/test_semantic_model_parsing.py b/tests/functional/semantic_models/test_semantic_model_parsing.py index 5ee6798aeaa..048e9844b83 100644 --- a/tests/functional/semantic_models/test_semantic_model_parsing.py +++ b/tests/functional/semantic_models/test_semantic_model_parsing.py @@ -9,104 +9,13 @@ from dbt.tests.util import write_file from tests.functional.assertions.test_runner import dbtTestRunner -schema_yml = """models: - - name: fct_revenue - description: This is the model fct_revenue. It should be able to use doc blocks - -semantic_models: - - name: revenue - description: This is the revenue semantic model. It should be able to use doc blocks - model: ref('fct_revenue') - - defaults: - agg_time_dimension: ds - - measures: - - name: txn_revenue - expr: revenue - agg: sum - agg_time_dimension: ds - create_metric: true - - name: sum_of_things - expr: 2 - agg: sum - agg_time_dimension: ds - - name: has_revenue - expr: true - agg: sum_boolean - agg_time_dimension: ds - - name: discrete_order_value_p99 - expr: order_total - agg: percentile - agg_time_dimension: ds - agg_params: - percentile: 0.99 - use_discrete_percentile: True - use_approximate_percentile: False - - name: test_agg_params_optional_are_empty - expr: order_total - agg: percentile - agg_time_dimension: ds - agg_params: - percentile: 0.99 - - name: test_non_additive - expr: txn_revenue - agg: sum - non_additive_dimension: - name: ds - window_choice: max - - dimensions: - - name: ds - type: time - expr: created_at - type_params: - time_granularity: day - - entities: - - name: user - type: foreign - expr: user_id - - name: id - type: primary - -metrics: - - name: simple_metric - label: Simple Metric - type: simple - type_params: - measure: sum_of_things -""" - -schema_without_semantic_model_yml = """models: - - name: fct_revenue - description: This is the model fct_revenue. It should be able to use doc blocks -""" - -fct_revenue_sql = """select - 1 as id, - 10 as user_id, - 1000 as revenue, - current_timestamp as created_at""" - -metricflow_time_spine_sql = """ -with days as ( - {{dbt_utils.date_spine('day' - , "to_date('01/01/2000','mm/dd/yyyy')" - , "to_date('01/01/2027','mm/dd/yyyy')" - ) - }} -), - -final as ( - select cast(date_day as date) as date_day - from days +from tests.functional.semantic_models.fixtures import ( + schema_without_semantic_model_yml, + fct_revenue_sql, + metricflow_time_spine_sql, + schema_yml, ) -select * -from final -""" - class TestSemanticModelParsing: @pytest.fixture(scope="class") diff --git a/tests/functional/semantic_models/test_semantic_models.py b/tests/functional/semantic_models/test_semantic_models.py index 627aae9b7a7..3d3be94b341 100644 --- a/tests/functional/semantic_models/test_semantic_models.py +++ b/tests/functional/semantic_models/test_semantic_models.py @@ -7,7 +7,7 @@ from tests.functional.semantic_models.fixtures import ( models_people_sql, - metricflow_time_spine_sql, + simple_metricflow_time_spine_sql, semantic_model_people_yml, models_people_metrics_yml, ) @@ -18,7 +18,7 @@ class TestSemanticModelDependsOn: def models(self): return { "people.sql": models_people_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, + "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, "semantic_models.yml": semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, } @@ -41,7 +41,7 @@ class TestSemanticModelUnknownModel: def models(self): return { "not_people.sql": models_people_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, + "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, "semantic_models.yml": semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, }