From e518fce553370a5fea801c59b62ab97d11e7254a Mon Sep 17 00:00:00 2001 From: Kyle Kent Date: Fri, 11 Aug 2023 17:59:08 -0700 Subject: [PATCH 1/3] added digit spec to childrens parents depth --- core/dbt/graph/graph.py | 4 ++-- core/dbt/graph/selector.py | 3 ++- core/dbt/graph/selector_spec.py | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/dbt/graph/graph.py b/core/dbt/graph/graph.py index 69a2f21258a..b2964c74e3d 100644 --- a/core/dbt/graph/graph.py +++ b/core/dbt/graph/graph.py @@ -52,9 +52,9 @@ def exclude_edge_type(self, edge_type_to_exclude): ), ) - def select_childrens_parents(self, selected: Set[UniqueId]) -> Set[UniqueId]: + def select_childrens_parents(self, selected: Set[UniqueId], max_depth: Optional[int] = None) -> Set[UniqueId]: ancestors_for = self.select_children(selected) | selected - return self.select_parents(ancestors_for) | ancestors_for + return self.select_parents(ancestors_for, max_depth) | ancestors_for def select_children( self, selected: Set[UniqueId], max_depth: Optional[int] = None diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index 1a198fef858..727c1963fe1 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -106,7 +106,8 @@ def collect_specified_neighbors( """ additional: Set[UniqueId] = set() if spec.childrens_parents: - additional.update(self.graph.select_childrens_parents(selected)) + depth = spec.childrens_parents_depth + additional.update(self.graph.select_childrens_parents(selected, depth)) if spec.parents: depth = spec.parents_depth diff --git a/core/dbt/graph/selector_spec.py b/core/dbt/graph/selector_spec.py index cf8481ccf65..df11600d72c 100644 --- a/core/dbt/graph/selector_spec.py +++ b/core/dbt/graph/selector_spec.py @@ -12,7 +12,7 @@ RAW_SELECTOR_PATTERN = re.compile( r"\A" - r"(?P(\@))?" + r"(?P((?P(\d*))\@))?" r"(?P((?P(\d*))\+))?" r"((?P([\w.]+)):)?(?P(.*?))" r"(?P(\+(?P(\d*))))?" @@ -66,6 +66,7 @@ class SelectionCriteria: method_arguments: List[str] value: Any childrens_parents: bool + childrens_parents_depth: Optional[int] parents: bool parents_depth: Optional[int] children: bool @@ -116,6 +117,7 @@ def selection_criteria_from_dict( parents_depth = _match_to_int(dct, "parents_depth") children_depth = _match_to_int(dct, "children_depth") + childrens_parents_depth = _match_to_int(dct, "childrens_parents_depth") # If defined field in selector, override CLI flag indirect_selection = IndirectSelection( @@ -128,6 +130,7 @@ def selection_criteria_from_dict( method_arguments=method_arguments, value=dct["value"], childrens_parents=bool(dct.get("childrens_parents")), + childrens_parents_depth=childrens_parents_depth, parents=bool(dct.get("parents")), parents_depth=parents_depth, children=bool(dct.get("children")), From 9e6d5a5944e25ffe0e74e8c73b03a451912d0a72 Mon Sep 17 00:00:00 2001 From: Kyle Kent Date: Fri, 11 Aug 2023 21:45:57 -0700 Subject: [PATCH 2/3] added unit tests --- test/unit/test_graph_selection.py | 82 ++++++++++++++------------- test/unit/test_graph_selector_spec.py | 22 +++++++ 2 files changed, 66 insertions(+), 38 deletions(-) diff --git a/test/unit/test_graph_selection.py b/test/unit/test_graph_selection.py index 572c8fed10d..9bb563a0e9d 100644 --- a/test/unit/test_graph_selection.py +++ b/test/unit/test_graph_selection.py @@ -97,6 +97,7 @@ def id_macro(arg): (["1+Y.f"], [], {"m.X.c", "m.Y.f"}), # childrens parents (["@X.c"], [], {"m.X.a", "m.X.c", "m.Y.f", "m.X.g"}), + (["1@X.c"], [], {"m.X.a", "m.X.c", "m.Y.f", "m.X.g"}), # multiple selection/exclusion (["tag:abc", "tag:bcef"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.X.e", "m.Y.f"}), (["tag:abc", "tag:bcef"], ["tag:efg"], {"m.X.a", "m.Y.b", "m.X.c"}), @@ -138,48 +139,51 @@ def test_run_specs(include, exclude, expected): param_specs = [ - ("a", False, None, False, None, "fqn", "a", False), - ("+a", True, None, False, None, "fqn", "a", False), - ("256+a", True, 256, False, None, "fqn", "a", False), - ("a+", False, None, True, None, "fqn", "a", False), - ("a+256", False, None, True, 256, "fqn", "a", False), - ("+a+", True, None, True, None, "fqn", "a", False), - ("16+a+32", True, 16, True, 32, "fqn", "a", False), - ("@a", False, None, False, None, "fqn", "a", True), - ("a.b", False, None, False, None, "fqn", "a.b", False), - ("+a.b", True, None, False, None, "fqn", "a.b", False), - ("256+a.b", True, 256, False, None, "fqn", "a.b", False), - ("a.b+", False, None, True, None, "fqn", "a.b", False), - ("a.b+256", False, None, True, 256, "fqn", "a.b", False), - ("+a.b+", True, None, True, None, "fqn", "a.b", False), - ("16+a.b+32", True, 16, True, 32, "fqn", "a.b", False), - ("@a.b", False, None, False, None, "fqn", "a.b", True), - ("a.b.*", False, None, False, None, "fqn", "a.b.*", False), - ("+a.b.*", True, None, False, None, "fqn", "a.b.*", False), - ("256+a.b.*", True, 256, False, None, "fqn", "a.b.*", False), - ("a.b.*+", False, None, True, None, "fqn", "a.b.*", False), - ("a.b.*+256", False, None, True, 256, "fqn", "a.b.*", False), - ("+a.b.*+", True, None, True, None, "fqn", "a.b.*", False), - ("16+a.b.*+32", True, 16, True, 32, "fqn", "a.b.*", False), - ("@a.b.*", False, None, False, None, "fqn", "a.b.*", True), - ("tag:a", False, None, False, None, "tag", "a", False), - ("+tag:a", True, None, False, None, "tag", "a", False), - ("256+tag:a", True, 256, False, None, "tag", "a", False), - ("tag:a+", False, None, True, None, "tag", "a", False), - ("tag:a+256", False, None, True, 256, "tag", "a", False), - ("+tag:a+", True, None, True, None, "tag", "a", False), - ("16+tag:a+32", True, 16, True, 32, "tag", "a", False), - ("@tag:a", False, None, False, None, "tag", "a", True), - ("source:a", False, None, False, None, "source", "a", False), - ("source:a+", False, None, True, None, "source", "a", False), - ("source:a+1", False, None, True, 1, "source", "a", False), - ("source:a+32", False, None, True, 32, "source", "a", False), - ("@source:a", False, None, False, None, "source", "a", True), + ("a", False, None, False, None, "fqn", "a", False, None), + ("+a", True, None, False, None, "fqn", "a", False, None), + ("256+a", True, 256, False, None, "fqn", "a", False, None), + ("a+", False, None, True, None, "fqn", "a", False, None), + ("a+256", False, None, True, 256, "fqn", "a", False, None), + ("+a+", True, None, True, None, "fqn", "a", False, None), + ("16+a+32", True, 16, True, 32, "fqn", "a", False, None), + ("@a", False, None, False, None, "fqn", "a", True, None), + ("20@a", False, None, False, None, "fqn", "a", True, 20), + ("a.b", False, None, False, None, "fqn", "a.b", False, None), + ("+a.b", True, None, False, None, "fqn", "a.b", False, None), + ("256+a.b", True, 256, False, None, "fqn", "a.b", False, None), + ("a.b+", False, None, True, None, "fqn", "a.b", False, None), + ("a.b+256", False, None, True, 256, "fqn", "a.b", False, None), + ("+a.b+", True, None, True, None, "fqn", "a.b", False, None), + ("16+a.b+32", True, 16, True, 32, "fqn", "a.b", False, None), + ("@a.b", False, None, False, None, "fqn", "a.b", True, None), + ("5@a.b", False, None, False, None, "fqn", "a.b", True, 5), + ("a.b.*", False, None, False, None, "fqn", "a.b.*", False, None), + ("+a.b.*", True, None, False, None, "fqn", "a.b.*", False, None), + ("256+a.b.*", True, 256, False, None, "fqn", "a.b.*", False, None), + ("a.b.*+", False, None, True, None, "fqn", "a.b.*", False, None), + ("a.b.*+256", False, None, True, 256, "fqn", "a.b.*", False, None), + ("+a.b.*+", True, None, True, None, "fqn", "a.b.*", False, None), + ("16+a.b.*+32", True, 16, True, 32, "fqn", "a.b.*", False, None), + ("@a.b.*", False, None, False, None, "fqn", "a.b.*", True, None), + ("12@a.b.*", False, None, False, None, "fqn", "a.b.*", True, 12), + ("tag:a", False, None, False, None, "tag", "a", False, None), + ("+tag:a", True, None, False, None, "tag", "a", False, None), + ("256+tag:a", True, 256, False, None, "tag", "a", False, None), + ("tag:a+", False, None, True, None, "tag", "a", False, None), + ("tag:a+256", False, None, True, 256, "tag", "a", False, None), + ("+tag:a+", True, None, True, None, "tag", "a", False, None), + ("16+tag:a+32", True, 16, True, 32, "tag", "a", False, None), + ("3@tag:a", False, None, False, None, "tag", "a", True, 3), + ("source:a", False, None, False, None, "source", "a", False, None), + ("source:a+", False, None, True, None, "source", "a", False, None), + ("source:a+1", False, None, True, 1, "source", "a", False, None), + ("source:a+32", False, None, True, 32, "source", "a", False, None), + ("1@source:a", False, None, False, None, "source", "a", True, 1), ] @pytest.mark.parametrize( - "spec,parents,parents_depth,children,children_depth,filter_type,filter_value,childrens_parents", + "spec,parents,parents_depth,children,children_depth,filter_type,filter_value,childrens_parents,childrens_parents_depth", param_specs, ids=id_macro, ) @@ -192,6 +196,7 @@ def test_parse_specs( filter_type, filter_value, childrens_parents, + childrens_parents_depth ): parsed = graph_selector.SelectionCriteria.from_single_spec(spec) assert parsed.parents == parents @@ -201,6 +206,7 @@ def test_parse_specs( assert parsed.method == filter_type assert parsed.value == filter_value assert parsed.childrens_parents == childrens_parents + assert parsed.childrens_parents_depth == childrens_parents_depth invalid_specs = [ diff --git a/test/unit/test_graph_selector_spec.py b/test/unit/test_graph_selector_spec.py index 8a19a8b5934..69aed2d9b92 100644 --- a/test/unit/test_graph_selector_spec.py +++ b/test/unit/test_graph_selector_spec.py @@ -23,6 +23,7 @@ def test_raw_parse_simple(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_simple_infer_path(): @@ -37,6 +38,7 @@ def test_raw_parse_simple_infer_path(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_simple_infer_path_modified(): @@ -51,6 +53,22 @@ def test_raw_parse_simple_infer_path_modified(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None + + +def test_raw_parse_simple_infer_fqn_childrens_parents(): + raw = "1@asdf" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == "asdf" + assert result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + assert result.childrens_parents_depth == 1 def test_raw_parse_simple_infer_fqn_parents(): @@ -65,6 +83,7 @@ def test_raw_parse_simple_infer_fqn_parents(): assert result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_simple_infer_fqn_children(): @@ -79,6 +98,7 @@ def test_raw_parse_simple_infer_fqn_children(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_complex(): @@ -93,6 +113,7 @@ def test_raw_parse_complex(): assert result.parents assert result.parents_depth == 2 assert result.children_depth == 4 + assert result.childrens_parents_depth is None def test_raw_parse_weird(): @@ -108,6 +129,7 @@ def test_raw_parse_weird(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_invalid(): From aeecf498206f239dfa4e81cc299a2b17fd2b3a7e Mon Sep 17 00:00:00 2001 From: Kyle Kent Date: Fri, 11 Aug 2023 22:31:24 -0700 Subject: [PATCH 3/3] changie log --- .changes/unreleased/Features-20230811-223109.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230811-223109.yaml diff --git a/.changes/unreleased/Features-20230811-223109.yaml b/.changes/unreleased/Features-20230811-223109.yaml new file mode 100644 index 00000000000..7bd87fa57a1 --- /dev/null +++ b/.changes/unreleased/Features-20230811-223109.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allows for specifying a childs parents max depth +time: 2023-08-11T22:31:09.729033-07:00 +custom: + Author: kentkr + Issue: CT-2599