From 3f61cc0ea862ba30d8fe6fb305b55785becf9f56 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Tue, 1 Oct 2024 10:52:34 -0700 Subject: [PATCH] /* PR_START p--short-term-perf 17 */ Update patterns to use a filter. --- .../candidate_push_down/push_down_visitor.py | 18 +++--------------- .../specs/patterns/entity_link_pattern.py | 10 ++++++---- .../specs/patterns/no_group_by_metric.py | 7 ++++--- .../specs/patterns/spec_pattern.py | 13 ++++++++----- .../specs/patterns/typed_patterns.py | 15 ++++++++------- 5 files changed, 29 insertions(+), 34 deletions(-) diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py index 84e8c7e8db..69347a6fab 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py @@ -4,7 +4,7 @@ import typing from contextlib import contextmanager from dataclasses import dataclass -from typing import Dict, FrozenSet, Iterator, List, Optional, Sequence, Set, Tuple +from typing import Dict, Iterator, List, Optional, Sequence, Tuple from dbt_semantic_interfaces.enum_extension import assert_values_exhausted from dbt_semantic_interfaces.references import MetricReference @@ -15,7 +15,6 @@ from metricflow_semantics.mf_logging.formatting import indent from metricflow_semantics.mf_logging.lazy_formattable import LazyFormat from metricflow_semantics.mf_logging.pretty_print import mf_pformat, mf_pformat_many -from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.model.semantics.element_filter import LinkableElementFilter from metricflow_semantics.query.group_by_item.candidate_push_down.group_by_item_candidate import GroupByItemCandidateSet @@ -148,8 +147,6 @@ def __init__( suggestion_generator: Optional[QueryItemSuggestionGenerator], group_by_item_resolver: GroupByItemResolver, source_spec_patterns: Sequence[SpecPattern] = (), - with_any_property: Optional[FrozenSet[LinkableElementProperty]] = None, - without_any_property: Optional[FrozenSet[LinkableElementProperty]] = None, filter_location: Optional[WhereFilterLocation] = None, ) -> None: """Initializer. @@ -168,8 +165,6 @@ def __init__( self._semantic_manifest_lookup = manifest_lookup self._source_spec_patterns = tuple(source_spec_patterns) self._path_from_start_node_tracker = DagTraversalPathTracker() - self._with_any_property = with_any_property - self._without_any_property = without_any_property self._suggestion_generator = suggestion_generator self._filter_location = filter_location self._group_by_item_resolver_for_query = group_by_item_resolver @@ -180,17 +175,10 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu with self._path_from_start_node_tracker.track_node_visit(node) as current_traversal_path: logger.debug(LazyFormat(lambda: f"Handling {node.ui_description}")) - without_any_property: Set[LinkableElementProperty] = set() - if self._without_any_property is not None: - without_any_property.update(self._without_any_property) - for spec_pattern in self._source_spec_patterns: - without_any_property.update(spec_pattern.without_linkable_element_properties) - items_available_for_measure = self._semantic_manifest_lookup.metric_lookup.linkable_elements_for_measure( measure_reference=node.measure_reference, - element_filter=LinkableElementFilter( - with_any_of=self._with_any_property or LinkableElementProperty.all_properties(), - without_any_of=frozenset(without_any_property), + element_filter=LinkableElementFilter.merge_iterable( + spec_pattern.element_pre_filter for spec_pattern in self._source_spec_patterns ), ) diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py index a1581f7ac9..8121f0267a 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py @@ -3,7 +3,7 @@ import logging from dataclasses import dataclass from enum import Enum -from typing import Any, FrozenSet, List, Optional, Sequence, Tuple +from typing import Any, List, Optional, Sequence, Tuple from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums.date_part import DatePart @@ -11,6 +11,7 @@ from typing_extensions import override from metricflow_semantics.model.linkable_element_property import LinkableElementProperty +from metricflow_semantics.model.semantics.element_filter import LinkableElementFilter from metricflow_semantics.specs.instance_spec import InstanceSpec, LinkableInstanceSpec from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_set import group_specs_by_type @@ -152,10 +153,11 @@ def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableIns @property @override - def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: + def element_pre_filter(self) -> LinkableElementFilter: if ( self.parameter_set.metric_subquery_entity_links is None or len(self.parameter_set.metric_subquery_entity_links) == 0 ): - return frozenset({LinkableElementProperty.METRIC}) - return frozenset() + return LinkableElementFilter(without_any_of=frozenset({LinkableElementProperty.METRIC})) + + return LinkableElementFilter() diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/no_group_by_metric.py b/metricflow-semantics/metricflow_semantics/specs/patterns/no_group_by_metric.py index 175447cdd7..03b005ee68 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/no_group_by_metric.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/no_group_by_metric.py @@ -1,11 +1,12 @@ from __future__ import annotations from dataclasses import dataclass -from typing import FrozenSet, List, Sequence +from typing import List, Sequence from typing_extensions import override from metricflow_semantics.model.linkable_element_property import LinkableElementProperty +from metricflow_semantics.model.semantics.element_filter import LinkableElementFilter from metricflow_semantics.specs.instance_spec import InstanceSpec, LinkableInstanceSpec from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_set import group_specs_by_type @@ -30,5 +31,5 @@ def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableIns @property @override - def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: - return frozenset({LinkableElementProperty.METRIC}) + def element_pre_filter(self) -> LinkableElementFilter: + return LinkableElementFilter(without_any_of=frozenset({LinkableElementProperty.METRIC})) diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/spec_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/spec_pattern.py index caaad9d5d9..de4bcea7f7 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/spec_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/spec_pattern.py @@ -1,9 +1,9 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, FrozenSet, Sequence +from typing import TYPE_CHECKING, Sequence -from metricflow_semantics.model.linkable_element_property import LinkableElementProperty +from metricflow_semantics.model.semantics.element_filter import LinkableElementFilter if TYPE_CHECKING: from metricflow_semantics.specs.instance_spec import InstanceSpec @@ -25,6 +25,9 @@ def matches_any(self, candidate_specs: Sequence[InstanceSpec]) -> bool: return len(self.match(candidate_specs)) > 0 @property - def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: - """Returns the set of properties of linkable elements that this won't match.""" - return frozenset() + def element_pre_filter(self) -> LinkableElementFilter: + """Returns a filter for a `LinkableElementSet` that can reduce the number of items to match. + + i.e. the filter can produce a superset of the elements that will match. + """ + return LinkableElementFilter() diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py b/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py index f70487b50e..e461cc05e3 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import FrozenSet, List, Sequence +from typing import List, Sequence from dbt_semantic_interfaces.call_parameter_sets import ( DimensionCallParameterSet, @@ -13,6 +13,7 @@ from typing_extensions import override from metricflow_semantics.model.linkable_element_property import LinkableElementProperty +from metricflow_semantics.model.semantics.element_filter import LinkableElementFilter from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName from metricflow_semantics.specs.instance_spec import InstanceSpec, LinkableInstanceSpec from metricflow_semantics.specs.patterns.entity_link_pattern import ( @@ -53,8 +54,8 @@ def from_call_parameter_set( # noqa: D102 @property @override - def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: - return frozenset({LinkableElementProperty.METRIC}) + def element_pre_filter(self) -> LinkableElementFilter: + return LinkableElementFilter(without_any_of=frozenset({LinkableElementProperty.METRIC})) @dataclass(frozen=True) @@ -99,8 +100,8 @@ def from_call_parameter_set( @property @override - def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: - return frozenset({LinkableElementProperty.METRIC}) + def element_pre_filter(self) -> LinkableElementFilter: + return LinkableElementFilter(without_any_of=frozenset({LinkableElementProperty.METRIC})) @dataclass(frozen=True) @@ -130,8 +131,8 @@ def from_call_parameter_set(entity_call_parameter_set: EntityCallParameterSet) - @property @override - def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: - return frozenset({LinkableElementProperty.METRIC}) + def element_pre_filter(self) -> LinkableElementFilter: + return LinkableElementFilter(without_any_of=frozenset({LinkableElementProperty.METRIC})) @dataclass(frozen=True)