From 3125a14611607984897f39ac17c6f25fd8fd824f Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 23 Apr 2024 21:22:44 -0700 Subject: [PATCH 1/5] Add `Project` fixture for unit tests and test `Project` class methods --- tests/unit/config/test_project.py | 111 ++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index 46dbda6b909..38684404991 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -11,8 +11,11 @@ import dbt.exceptions from dbt.adapters.factory import load_plugin from dbt.adapters.contracts.connection import QueryComment, DEFAULT_QUERY_COMMENT +from dbt.config.project import Project, RenderComponents, VarProvider +from dbt.config.selectors import SelectorConfig from dbt.contracts.project import PackageConfig, LocalPackage, GitPackage from dbt.node_types import NodeType +from dbt_common.exceptions import DbtRuntimeError from dbt_common.semver import VersionSpecifier from dbt.flags import set_from_args @@ -27,6 +30,114 @@ ) +class TestProjectWithPytest: + @pytest.fixture(scope="function") + def selector_config(self) -> SelectorConfig: + return SelectorConfig.selectors_from_dict( + data={ + "selectors": [ + { + "name": "my_selector", + "definition": "give me cats", + "default": True, + } + ] + } + ) + + @pytest.fixture(scope="function") + def project(self, selector_config: SelectorConfig) -> Project: + return Project( + project_name="test_project", + version=1.0, + project_root="doesnt/actually/exist", + profile_name="test_profile", + model_paths=["models"], + macro_paths=["macros"], + seed_paths=["seeds"], + test_paths=["tests"], + analysis_paths=["analyses"], + docs_paths=["docs"], + asset_paths=["assets"], + target_path="target", + snapshot_paths=["snapshots"], + clean_targets=["target"], + log_path="path/to/project/logs", + packages_install_path="dbt_packages", + packages_specified_path="packages.yml", + quoting={"database": True, "schema": True, "identifier": True}, + models={}, + on_run_start=[], + on_run_end=[], + dispatch=[ + {"macro_namespace": "dbt_utils", "search_order": ["test_project", "dbt_utils"]} + ], + seeds={}, + snapshots={}, + sources={}, + data_tests={}, + unit_tests={}, + metrics={}, + semantic_models={}, + saved_queries={}, + exposures={}, + vars=VarProvider({}), + dbt_version=[VersionSpecifier.from_version_string(dbt.version.__version__)], + packages=PackageConfig([]), + manifest_selectors={}, + selectors=selector_config, + query_comment=QueryComment(), + config_version=1, + unrendered=RenderComponents({}, {}, {}), + project_env_vars={}, + restrict_access=False, + dbt_cloud={}, + ) + + def test_all_source_paths(self, project: Project): + assert ( + project.all_source_paths.sort() + == ["models", "seeds", "snapshots", "analyses", "macros"].sort() + ) + + def test_generic_test_paths(self, project: Project): + assert project.generic_test_paths == ["tests/generic"] + + def test_fixture_paths(self, project: Project): + assert project.fixture_paths == ["tests/fixtures"] + + def test__str__(self, project: Project): + assert ( + str(project) + == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {'database': True, 'schema': True, 'identifier': True}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'vars': {}, 'require-dbt-version': ['=1.8.0-b3'], 'restrict-access': False, 'dbt-cloud': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" + ) + + def test_get_selector(self, project: Project): + selector = project.get_selector("my_selector") + assert selector.raw == "give me cats" + + with pytest.raises(DbtRuntimeError): + project.get_selector("doesnt_exist") + + def test_get_default_selector_name(self, project: Project): + default_selector_name = project.get_default_selector_name() + assert default_selector_name == "my_selector" + + project.selectors["my_selector"]["default"] = False + default_selector_name = project.get_default_selector_name() + assert default_selector_name is None + + def test_get_macro_search_order(self, project: Project): + search_order = project.get_macro_search_order("dbt_utils") + assert search_order == ["test_project", "dbt_utils"] + + search_order = project.get_macro_search_order("doesnt_exist") + assert search_order is None + + def test_project_target_path(self, project: Project): + assert project.project_target_path == "doesnt/actually/exist/target" + + class TestProject(BaseConfigTest): def test_defaults(self): project = project_from_config_norender( From 571879633b3cd57a82f928aa784225ba1ed2d852 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 23 Apr 2024 21:27:31 -0700 Subject: [PATCH 2/5] Move `Project.__eq__` unit tests to new pytest class testing --- tests/unit/config/test_project.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index 38684404991..ba562a6aee2 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -137,6 +137,15 @@ def test_get_macro_search_order(self, project: Project): def test_project_target_path(self, project: Project): assert project.project_target_path == "doesnt/actually/exist/target" + def test_eq(self, project: Project): + other = deepcopy(project) + assert project == other + + def test_neq(self, project: Project): + other = deepcopy(project) + other.project_name = "other project" + assert project != other + class TestProject(BaseConfigTest): def test_defaults(self): @@ -171,21 +180,6 @@ def test_defaults(self): # embarrassing str(project) - def test_eq(self): - project = project_from_config_norender( - self.default_project_data, project_root=self.project_dir - ) - other = project_from_config_norender( - self.default_project_data, project_root=self.project_dir - ) - self.assertEqual(project, other) - - def test_neq(self): - project = project_from_config_norender( - self.default_project_data, project_root=self.project_dir - ) - self.assertNotEqual(project, object()) - def test_implicit_overrides(self): self.default_project_data.update( { From ab6acf0be97b92075dd250a62d5bae9f43cb40d0 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 23 Apr 2024 21:29:51 -0700 Subject: [PATCH 3/5] Move `Project.hashed_name` unit test to pytest testing class --- tests/unit/config/test_project.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index ba562a6aee2..afe40fc9be1 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -146,6 +146,9 @@ def test_neq(self, project: Project): other.project_name = "other project" assert project != other + def test_hashed_name(self, project: Project): + assert project.hashed_name() == "6e72a69d5c5cca8f0400338441c022e4" + class TestProject(BaseConfigTest): def test_defaults(self): @@ -194,12 +197,6 @@ def test_implicit_overrides(self): set(["other-models", "seeds", "snapshots", "analyses", "macros"]), ) - def test_hashed_name(self): - project = project_from_config_norender( - self.default_project_data, project_root=self.project_dir - ) - self.assertEqual(project.hashed_name(), "754cd47eac1d6f50a5f7cd399ec43da4") - def test_all_overrides(self): # log-path is not tested because it is set exclusively from flags, not cfg self.default_project_data.update( From c9c05be2f3808a16d907a3aa6a24f11f9b342ef8 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 23 Apr 2024 21:32:43 -0700 Subject: [PATCH 4/5] Rename some testing class in `test_project.py` to align with testing split --- tests/unit/config/test_project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index afe40fc9be1..946b20eca3e 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -30,7 +30,7 @@ ) -class TestProjectWithPytest: +class TestProjectMethods: @pytest.fixture(scope="function") def selector_config(self) -> SelectorConfig: return SelectorConfig.selectors_from_dict( @@ -150,7 +150,7 @@ def test_hashed_name(self, project: Project): assert project.hashed_name() == "6e72a69d5c5cca8f0400338441c022e4" -class TestProject(BaseConfigTest): +class TestProjectInitialization(BaseConfigTest): def test_defaults(self): project = project_from_config_norender( self.default_project_data, project_root=self.project_dir From 39020d8638d6e89e7916c369d88a2cb28ade8586 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Wed, 24 Apr 2024 10:03:54 -0700 Subject: [PATCH 5/5] Refactor `project` fixture to make accessible to other unit tests --- tests/unit/config/test_project.py | 66 +--------------------------- tests/unit/conftest.py | 1 + tests/unit/utils/project.py | 72 +++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 65 deletions(-) create mode 100644 tests/unit/utils/project.py diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index 946b20eca3e..f51719c7e33 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -11,8 +11,7 @@ import dbt.exceptions from dbt.adapters.factory import load_plugin from dbt.adapters.contracts.connection import QueryComment, DEFAULT_QUERY_COMMENT -from dbt.config.project import Project, RenderComponents, VarProvider -from dbt.config.selectors import SelectorConfig +from dbt.config.project import Project from dbt.contracts.project import PackageConfig, LocalPackage, GitPackage from dbt.node_types import NodeType from dbt_common.exceptions import DbtRuntimeError @@ -31,69 +30,6 @@ class TestProjectMethods: - @pytest.fixture(scope="function") - def selector_config(self) -> SelectorConfig: - return SelectorConfig.selectors_from_dict( - data={ - "selectors": [ - { - "name": "my_selector", - "definition": "give me cats", - "default": True, - } - ] - } - ) - - @pytest.fixture(scope="function") - def project(self, selector_config: SelectorConfig) -> Project: - return Project( - project_name="test_project", - version=1.0, - project_root="doesnt/actually/exist", - profile_name="test_profile", - model_paths=["models"], - macro_paths=["macros"], - seed_paths=["seeds"], - test_paths=["tests"], - analysis_paths=["analyses"], - docs_paths=["docs"], - asset_paths=["assets"], - target_path="target", - snapshot_paths=["snapshots"], - clean_targets=["target"], - log_path="path/to/project/logs", - packages_install_path="dbt_packages", - packages_specified_path="packages.yml", - quoting={"database": True, "schema": True, "identifier": True}, - models={}, - on_run_start=[], - on_run_end=[], - dispatch=[ - {"macro_namespace": "dbt_utils", "search_order": ["test_project", "dbt_utils"]} - ], - seeds={}, - snapshots={}, - sources={}, - data_tests={}, - unit_tests={}, - metrics={}, - semantic_models={}, - saved_queries={}, - exposures={}, - vars=VarProvider({}), - dbt_version=[VersionSpecifier.from_version_string(dbt.version.__version__)], - packages=PackageConfig([]), - manifest_selectors={}, - selectors=selector_config, - query_comment=QueryComment(), - config_version=1, - unrendered=RenderComponents({}, {}, {}), - project_env_vars={}, - restrict_access=False, - dbt_cloud={}, - ) - def test_all_source_paths(self, project: Project): assert ( project.all_source_paths.sort() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5e9acb84907..a3c42f9f8e8 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -6,6 +6,7 @@ # All manifest related fixtures. from tests.unit.utils.manifest import * # noqa +from tests.unit.utils.project import * # noqa @pytest.fixture diff --git a/tests/unit/utils/project.py b/tests/unit/utils/project.py new file mode 100644 index 00000000000..53bad5214a8 --- /dev/null +++ b/tests/unit/utils/project.py @@ -0,0 +1,72 @@ +import pytest + +import dbt.config +import dbt.exceptions +from dbt.adapters.contracts.connection import QueryComment +from dbt.config.project import Project, RenderComponents, VarProvider +from dbt.config.selectors import SelectorConfig +from dbt.contracts.project import PackageConfig +from dbt_common.semver import VersionSpecifier + + +@pytest.fixture(scope="function") +def selector_config() -> SelectorConfig: + return SelectorConfig.selectors_from_dict( + data={ + "selectors": [ + { + "name": "my_selector", + "definition": "give me cats", + "default": True, + } + ] + } + ) + + +@pytest.fixture(scope="function") +def project(selector_config: SelectorConfig) -> Project: + return Project( + project_name="test_project", + version=1.0, + project_root="doesnt/actually/exist", + profile_name="test_profile", + model_paths=["models"], + macro_paths=["macros"], + seed_paths=["seeds"], + test_paths=["tests"], + analysis_paths=["analyses"], + docs_paths=["docs"], + asset_paths=["assets"], + target_path="target", + snapshot_paths=["snapshots"], + clean_targets=["target"], + log_path="path/to/project/logs", + packages_install_path="dbt_packages", + packages_specified_path="packages.yml", + quoting={"database": True, "schema": True, "identifier": True}, + models={}, + on_run_start=[], + on_run_end=[], + dispatch=[{"macro_namespace": "dbt_utils", "search_order": ["test_project", "dbt_utils"]}], + seeds={}, + snapshots={}, + sources={}, + data_tests={}, + unit_tests={}, + metrics={}, + semantic_models={}, + saved_queries={}, + exposures={}, + vars=VarProvider({}), + dbt_version=[VersionSpecifier.from_version_string(dbt.version.__version__)], + packages=PackageConfig([]), + manifest_selectors={}, + selectors=selector_config, + query_comment=QueryComment(), + config_version=1, + unrendered=RenderComponents({}, {}, {}), + project_env_vars={}, + restrict_access=False, + dbt_cloud={}, + )