From 8ccc7d0145b97c591e7cfbcd8dd766edb7194983 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 6 Sep 2024 15:42:29 -0400 Subject: [PATCH 1/4] backport #10644 --- .../unreleased/Features-20240829-135320.yaml | 6 ++ core/dbt/config/project.py | 9 ++- core/dbt/config/runtime.py | 1 + core/dbt/contracts/project.py | 1 + tests/unit/config/test_project.py | 55 ++++++++++++++++++- tests/unit/utils/project.py | 1 + 6 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 .changes/unreleased/Features-20240829-135320.yaml diff --git a/.changes/unreleased/Features-20240829-135320.yaml b/.changes/unreleased/Features-20240829-135320.yaml new file mode 100644 index 00000000000..c7f5cf9d8b4 --- /dev/null +++ b/.changes/unreleased/Features-20240829-135320.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add support for behavior flags +time: 2024-08-29T13:53:20.16122-04:00 +custom: + Author: mikealfare + Issue: "10618" diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 739dd2d6737..7bb39acfb13 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -491,6 +491,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": rendered.selectors_dict["selectors"] ) dbt_cloud = cfg.dbt_cloud + flags: Dict[str, Any] = cfg.flags project = Project( project_name=name, @@ -535,6 +536,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": project_env_vars=project_env_vars, restrict_access=cfg.restrict_access, dbt_cloud=dbt_cloud, + flags=flags, ) # sanity check - this means an internal issue project.validate() @@ -579,11 +581,6 @@ def from_project_root( ) = package_and_project_data_from_root(project_root) selectors_dict = selector_data_from_root(project_root) - if "flags" in project_dict: - # We don't want to include "flags" in the Project, - # it goes in ProjectFlags - project_dict.pop("flags") - return cls.from_dicts( project_root=project_root, project_dict=project_dict, @@ -656,6 +653,7 @@ class Project: project_env_vars: Dict[str, Any] restrict_access: bool dbt_cloud: Dict[str, Any] + flags: Dict[str, Any] @property def all_source_paths(self) -> List[str]: @@ -735,6 +733,7 @@ def to_project_config(self, with_packages=False): "require-dbt-version": [v.to_version_string() for v in self.dbt_version], "restrict-access": self.restrict_access, "dbt-cloud": self.dbt_cloud, + "flags": self.flags, } ) if self.query_comment: diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index e32005aa91f..f9993f1cdf0 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -188,6 +188,7 @@ def from_parts( log_cache_events=log_cache_events, dependencies=dependencies, dbt_cloud=project.dbt_cloud, + flags=project.flags, ) # Called by 'load_projects' in this class diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index c2090eb4d10..46a913c9c43 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -248,6 +248,7 @@ class Project(dbtClassMixin): query_comment: Optional[Union[QueryComment, NoValue, str]] = field(default_factory=NoValue) restrict_access: bool = False dbt_cloud: Optional[Dict[str, Any]] = None + flags: Dict[str, Any] = field(default_factory=dict) class Config(dbtMashConfig): # These tell mashumaro to use aliases for jsonschema and for "from_dict" diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index a4aa63f9585..16e42b32265 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -3,6 +3,7 @@ import os import unittest import pytest +from typing import Any, Dict from unittest import mock @@ -11,7 +12,7 @@ import dbt.exceptions from dbt.adapters.factory import load_plugin from dbt.adapters.contracts.connection import QueryComment, DEFAULT_QUERY_COMMENT -from dbt.config.project import Project +from dbt.config.project import Project, _get_required_version from dbt.contracts.project import PackageConfig, LocalPackage, GitPackage from dbt.node_types import NodeType from dbt_common.exceptions import DbtRuntimeError @@ -45,7 +46,7 @@ def test_fixture_paths(self, project: Project): def test__str__(self, project: Project): assert ( str(project) - == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {'database': True, 'schema': True, 'identifier': True}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'vars': {}, 'require-dbt-version': ['=0.0.0'], 'restrict-access': False, 'dbt-cloud': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" + == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'vars': {}, 'require-dbt-version': ['=0.0.0'], 'restrict-access': False, 'dbt-cloud': {}, 'flags': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" ) def test_get_selector(self, project: Project): @@ -537,3 +538,53 @@ def setUp(self): def test_setting_multiple_flags(self): with pytest.raises(dbt.exceptions.DbtProjectError): set_from_args(self.args, None) + + +class TestGetRequiredVersion: + @pytest.fixture + def project_dict(self) -> Dict[str, Any]: + return { + "name": "test_project", + "require-dbt-version": ">0.0.0", + } + + def test_supported_version(self, project_dict: Dict[str, Any]) -> None: + specifiers = _get_required_version(project_dict=project_dict, verify_version=True) + assert set(x.to_version_string() for x in specifiers) == {">0.0.0"} + + def test_unsupported_version(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = ">99999.0.0" + with pytest.raises( + dbt.exceptions.DbtProjectError, match="This version of dbt is not supported" + ): + _get_required_version(project_dict=project_dict, verify_version=True) + + def test_unsupported_version_no_check(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = ">99999.0.0" + specifiers = _get_required_version(project_dict=project_dict, verify_version=False) + assert set(x.to_version_string() for x in specifiers) == {">99999.0.0"} + + def test_supported_version_range(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = [">0.0.0", "<=99999.0.0"] + specifiers = _get_required_version(project_dict=project_dict, verify_version=True) + assert set(x.to_version_string() for x in specifiers) == {">0.0.0", "<=99999.0.0"} + + def test_unsupported_version_range(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = [">0.0.0", "<=0.0.1"] + with pytest.raises( + dbt.exceptions.DbtProjectError, match="This version of dbt is not supported" + ): + _get_required_version(project_dict=project_dict, verify_version=True) + + def test_unsupported_version_range_no_check(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = [">0.0.0", "<=0.0.1"] + specifiers = _get_required_version(project_dict=project_dict, verify_version=False) + assert set(x.to_version_string() for x in specifiers) == {">0.0.0", "<=0.0.1"} + + def test_impossible_version_range(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = [">99999.0.0", "<=0.0.1"] + with pytest.raises( + dbt.exceptions.DbtProjectError, + match="The package version requirement can never be satisfied", + ): + _get_required_version(project_dict=project_dict, verify_version=True) diff --git a/tests/unit/utils/project.py b/tests/unit/utils/project.py index b34e03da494..347b1e130ce 100644 --- a/tests/unit/utils/project.py +++ b/tests/unit/utils/project.py @@ -67,4 +67,5 @@ def project(selector_config: SelectorConfig) -> Project: project_env_vars={}, restrict_access=False, dbt_cloud={}, + flags={}, ) From 4c9fb1cc24c0b8766b2d45b3a40a393630dd1839 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 17 Sep 2024 09:52:46 -0400 Subject: [PATCH 2/4] remove backport artifact --- tests/unit/config/test_project.py | 53 +------------------------------ 1 file changed, 1 insertion(+), 52 deletions(-) diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index 16e42b32265..681df5da619 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -3,7 +3,6 @@ import os import unittest import pytest -from typing import Any, Dict from unittest import mock @@ -12,7 +11,7 @@ import dbt.exceptions from dbt.adapters.factory import load_plugin from dbt.adapters.contracts.connection import QueryComment, DEFAULT_QUERY_COMMENT -from dbt.config.project import Project, _get_required_version +from dbt.config.project import Project from dbt.contracts.project import PackageConfig, LocalPackage, GitPackage from dbt.node_types import NodeType from dbt_common.exceptions import DbtRuntimeError @@ -538,53 +537,3 @@ def setUp(self): def test_setting_multiple_flags(self): with pytest.raises(dbt.exceptions.DbtProjectError): set_from_args(self.args, None) - - -class TestGetRequiredVersion: - @pytest.fixture - def project_dict(self) -> Dict[str, Any]: - return { - "name": "test_project", - "require-dbt-version": ">0.0.0", - } - - def test_supported_version(self, project_dict: Dict[str, Any]) -> None: - specifiers = _get_required_version(project_dict=project_dict, verify_version=True) - assert set(x.to_version_string() for x in specifiers) == {">0.0.0"} - - def test_unsupported_version(self, project_dict: Dict[str, Any]) -> None: - project_dict["require-dbt-version"] = ">99999.0.0" - with pytest.raises( - dbt.exceptions.DbtProjectError, match="This version of dbt is not supported" - ): - _get_required_version(project_dict=project_dict, verify_version=True) - - def test_unsupported_version_no_check(self, project_dict: Dict[str, Any]) -> None: - project_dict["require-dbt-version"] = ">99999.0.0" - specifiers = _get_required_version(project_dict=project_dict, verify_version=False) - assert set(x.to_version_string() for x in specifiers) == {">99999.0.0"} - - def test_supported_version_range(self, project_dict: Dict[str, Any]) -> None: - project_dict["require-dbt-version"] = [">0.0.0", "<=99999.0.0"] - specifiers = _get_required_version(project_dict=project_dict, verify_version=True) - assert set(x.to_version_string() for x in specifiers) == {">0.0.0", "<=99999.0.0"} - - def test_unsupported_version_range(self, project_dict: Dict[str, Any]) -> None: - project_dict["require-dbt-version"] = [">0.0.0", "<=0.0.1"] - with pytest.raises( - dbt.exceptions.DbtProjectError, match="This version of dbt is not supported" - ): - _get_required_version(project_dict=project_dict, verify_version=True) - - def test_unsupported_version_range_no_check(self, project_dict: Dict[str, Any]) -> None: - project_dict["require-dbt-version"] = [">0.0.0", "<=0.0.1"] - specifiers = _get_required_version(project_dict=project_dict, verify_version=False) - assert set(x.to_version_string() for x in specifiers) == {">0.0.0", "<=0.0.1"} - - def test_impossible_version_range(self, project_dict: Dict[str, Any]) -> None: - project_dict["require-dbt-version"] = [">99999.0.0", "<=0.0.1"] - with pytest.raises( - dbt.exceptions.DbtProjectError, - match="The package version requirement can never be satisfied", - ): - _get_required_version(project_dict=project_dict, verify_version=True) From 75fbe4bd78237acf2bba06876e6211a717afd346 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 18 Sep 2024 17:41:18 +0100 Subject: [PATCH 3/4] fix test_project.py --- tests/unit/config/test_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index 681df5da619..19f65d7f9b6 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -45,7 +45,7 @@ def test_fixture_paths(self, project: Project): def test__str__(self, project: Project): assert ( str(project) - == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'vars': {}, 'require-dbt-version': ['=0.0.0'], 'restrict-access': False, 'dbt-cloud': {}, 'flags': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" + == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'vars': {}, 'require-dbt-version': ['=0.0.0'], 'restrict-access': False, 'dbt-cloud': {}, 'flags': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" ) def test_get_selector(self, project: Project): From c4408ef7590e49066af5a6a55b3199f895438972 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 18 Sep 2024 13:01:44 -0400 Subject: [PATCH 4/4] fix expected string that got clobbered in the backport --- tests/unit/config/test_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index 681df5da619..bd5c1e7692a 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -45,7 +45,7 @@ def test_fixture_paths(self, project: Project): def test__str__(self, project: Project): assert ( str(project) - == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'vars': {}, 'require-dbt-version': ['=0.0.0'], 'restrict-access': False, 'dbt-cloud': {}, 'flags': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" + == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {'database': True, 'schema': True, 'identifier': True}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'vars': {}, 'require-dbt-version': ['=0.0.0'], 'restrict-access': False, 'dbt-cloud': {}, 'flags': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" ) def test_get_selector(self, project: Project):