diff --git a/.changes/unreleased/Features-20220408-132725.yaml b/.changes/unreleased/Features-20220408-132725.yaml new file mode 100644 index 00000000000..778c54257d2 --- /dev/null +++ b/.changes/unreleased/Features-20220408-132725.yaml @@ -0,0 +1,7 @@ +kind: Features +body: add enabled as a source config +time: 2022-04-08T13:27:25.292454-04:00 +custom: + Author: nathaniel-may + Issue: "3662" + PR: "5008" diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index 50074c71359..3352303a47f 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -335,6 +335,40 @@ def replace(self, **kwargs): @dataclass class SourceConfig(BaseConfig): enabled: bool = True + # to be implmented to complete CT-201 + # quoting: Dict[str, Any] = field( + # default_factory=dict, + # metadata=MergeBehavior.Update.meta(), + # ) + # freshness: Optional[Dict[str, Any]] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # loader: Optional[str] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # # TODO what type is this? docs say: "" + # loaded_at_field: Optional[str] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # database: Optional[str] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # schema: Optional[str] = field( + # default=None, + # metadata=CompareBehavior.Exclude.meta(), + # ) + # meta: Dict[str, Any] = field( + # default_factory=dict, + # metadata=MergeBehavior.Update.meta(), + # ) + # tags: Union[List[str], str] = field( + # default_factory=list_str, + # metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), + # ) @dataclass diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index d1c0b4a7a5a..80e956c06e0 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -242,6 +242,7 @@ class Quoting(dbtClassMixin, Mergeable): @dataclass class UnparsedSourceTableDefinition(HasColumnTests, HasTests): + config: Dict[str, Any] = field(default_factory=dict) loaded_at_field: Optional[str] = None identifier: Optional[str] = None quoting: Quoting = field(default_factory=Quoting) @@ -322,6 +323,7 @@ class SourcePatch(dbtClassMixin, Replaceable): path: Path = field( metadata=dict(description="The path to the patch-defining yml file"), ) + config: Dict[str, Any] = field(default_factory=dict) description: Optional[str] = None meta: Optional[Dict[str, Any]] = None database: Optional[str] = None diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 0f4c7e97537..70f727c3634 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -465,6 +465,8 @@ def parse_project( else: dct = block.file.dict_from_yaml parser.parse_file(block, dct=dct) + # Came out of here with UnpatchedSourceDefinition containing configs at the source level + # and not configs at the table level (as expected) else: parser.parse_file(block) project_parsed_path_count += 1 diff --git a/core/dbt/parser/sources.py b/core/dbt/parser/sources.py index 44dcdb29b67..28c0428cbbe 100644 --- a/core/dbt/parser/sources.py +++ b/core/dbt/parser/sources.py @@ -1,6 +1,6 @@ import itertools from pathlib import Path -from typing import Iterable, Dict, Optional, Set, List, Any +from typing import Iterable, Dict, Optional, Set, Any from dbt.adapters.factory import get_adapter from dbt.config import RuntimeConfig from dbt.context.context_config import ( @@ -137,15 +137,13 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> ParsedSourceDefinit tags = sorted(set(itertools.chain(source.tags, table.tags))) config = self._generate_source_config( - fqn=target.fqn, + target=target, rendered=True, - project_name=target.package_name, ) unrendered_config = self._generate_source_config( - fqn=target.fqn, + target=target, rendered=False, - project_name=target.package_name, ) if not isinstance(config, SourceConfig): @@ -261,19 +259,29 @@ def parse_source_test( ) return node - def _generate_source_config(self, fqn: List[str], rendered: bool, project_name: str): + def _generate_source_config(self, target: UnpatchedSourceDefinition, rendered: bool): generator: BaseContextConfigGenerator if rendered: generator = ContextConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) + # configs with precendence set + precedence_configs = dict() + # first apply source configs + precedence_configs.update(target.source.config) + # then overrite anything that is defined on source tables + # this is not quite complex enough for configs that can be set as top-level node keys, but + # it works while source configs can only include `enabled`. + precedence_configs.update(target.table.config) + return generator.calculate_node_config( config_call_dict={}, - fqn=fqn, + fqn=target.fqn, resource_type=NodeType.Source, - project_name=project_name, + project_name=target.package_name, base=False, + patch_config_dict=precedence_configs, ) def _get_relation_name(self, node: ParsedSourceDefinition): diff --git a/test/unit/test_contracts_graph_unparsed.py b/test/unit/test_contracts_graph_unparsed.py index f82e617f55c..c4df44f7100 100644 --- a/test/unit/test_contracts_graph_unparsed.py +++ b/test/unit/test_contracts_graph_unparsed.py @@ -308,6 +308,7 @@ def test_table_defaults(self): to_dict = { 'name': 'foo', 'description': '', + 'config': {}, 'loader': '', 'freshness': {'error_after': {}, 'warn_after': {}}, 'quoting': {}, @@ -316,6 +317,7 @@ def test_table_defaults(self): { 'name': 'table1', 'description': '', + 'config': {}, 'docs': {'show': True}, 'tests': [], 'columns': [], @@ -327,6 +329,7 @@ def test_table_defaults(self): { 'name': 'table2', 'description': 'table 2', + 'config': {}, 'docs': {'show': True}, 'tests': [], 'columns': [], diff --git a/tests/functional/permission/test_permissions.py b/tests/functional/permission/test_permissions.py deleted file mode 100644 index e2999f47548..00000000000 --- a/tests/functional/permission/test_permissions.py +++ /dev/null @@ -1,50 +0,0 @@ -import pytest -import os - -from dbt.tests.util import run_dbt -from tests.functional.permission.fixtures import models, project_files # noqa: F401 - - -class TestPermissions: - @pytest.fixture(autouse=True) - def setUp(self, project): - path = os.path.join(project.test_data_dir, "seed.sql") - project.run_sql_file(path) - - @pytest.fixture(scope="class") - def profiles_config_update(self, unique_schema): - return { - "test": { - "outputs": { - "default": { - "type": "postgres", - "threads": 4, - "host": "localhost", - "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), - "user": os.getenv("POSTGRES_TEST_USER", "root"), - "pass": os.getenv("POSTGRES_TEST_PASS", "password"), - "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), - "schema": unique_schema, - }, - "noaccess": { - "type": "postgres", - "threads": 4, - "host": "localhost", - "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), - "user": "noaccess", - "pass": "password", - "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), - "schema": unique_schema, - }, - }, - } - } - - def test_no_create_schema_permissions( - self, - project, - ): - # the noaccess user does not have permissions to create a schema -- this should fail - project.run_sql('drop schema if exists "{}" cascade'.format(project.test_schema)) - with pytest.raises(RuntimeError): - run_dbt(["run", "--target", "noaccess"], expect_pass=False) diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py new file mode 100644 index 00000000000..23d0d6fd5de --- /dev/null +++ b/tests/functional/sources/test_source_configs.py @@ -0,0 +1,230 @@ +import pytest +from dbt.contracts.graph.model_config import SourceConfig + +from dbt.tests.util import run_dbt, update_config_file, get_manifest + + +class SourceConfigTests: + @pytest.fixture(scope="class", autouse=True) + def setUp(self): + pytest.expected_config = SourceConfig( + enabled=True, + ) + + +models__schema_yml = """version: 2 + +sources: + - name: test_source + tables: + - name: test_table + - name: other_source + tables: + - name: test_table +""" + + +# Test enabled config in dbt_project.yml +# expect pass, already implemented +class TestSourceEnabledConfigProjectLevel(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models__schema_yml, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "sources": { + "test": { + "test_source": { + "enabled": True, + }, + } + } + } + + def test_enabled_source_config_dbt_project(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + + new_enabled_config = { + "sources": { + "test": { + "test_source": { + "enabled": False, + }, + } + } + } + update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + + assert ( + "source.test.test_source.test_table" not in manifest.sources + ) # or should it be there with enabled: false?? + assert "source.test.other_source.test_table" in manifest.sources + + +disabled_source_level__schema_yml = """version: 2 + +sources: + - name: test_source + config: + enabled: False + tables: + - name: test_table + - name: disabled_test_table +""" + + +# Test enabled config at sources level in yml file +class TestConfigYamlSourceLevel(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": disabled_source_level__schema_yml, + } + + def test_source_config_yaml_source_level(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" not in manifest.sources + assert "source.test.test_source.disabled_test_table" not in manifest.sources + + +disabled_source_table__schema_yml = """version: 2 + +sources: + - name: test_source + tables: + - name: test_table + - name: disabled_test_table + config: + enabled: False +""" + + +# Test enabled config at source table level in yaml file +class TestConfigYamlSourceTable(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": disabled_source_table__schema_yml, + } + + def test_source_config_yaml_source_table(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.disabled_test_table" not in manifest.sources + + +all_configs_everywhere__schema_yml = """version: 2 + +sources: + - name: test_source + config: + enabled: False + tables: + - name: test_table + config: + enabled: True + - name: other_test_table +""" + + +# Test inheritence - set configs at project, source, and source-table level - expect source-table level to win +class TestSourceConfigsInheritence1(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": all_configs_everywhere__schema_yml} + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"sources": {"enabled": True}} + + def test_source_all_configs_source_table(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.other_test_table" not in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + + assert isinstance(config_test_table, SourceConfig) + assert config_test_table == pytest.expected_config + + +all_configs_not_table_schema_yml = """version: 2 + +sources: + - name: test_source + config: + enabled: True + tables: + - name: test_table + - name: other_test_table +""" + + +# Test inheritence - set configs at project and source level - expect source level to win +class TestSourceConfigsInheritence2(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": all_configs_not_table_schema_yml} + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"sources": {"enabled": False}} + + def test_source_two_configs_source_level(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.other_test_table" in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + config_other_test_table = manifest.sources.get( + "source.test.test_source.other_test_table" + ).config + + assert isinstance(config_test_table, SourceConfig) + assert isinstance(config_other_test_table, SourceConfig) + + assert config_test_table == config_other_test_table + assert config_test_table == pytest.expected_config + + +all_configs_project_source__schema_yml = """version: 2 + +sources: + - name: test_source + tables: + - name: test_table + config: + enabled: True + - name: other_test_table +""" + + +# Test inheritence - set configs at project and source-table level - expect source-table level to win +class TestSourceConfigsInheritence3(SourceConfigTests): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": all_configs_project_source__schema_yml} + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"sources": {"enabled": False}} + + def test_source_two_configs_source_table(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + assert "source.test.test_source.test_table" in manifest.sources + assert "source.test.test_source.other_test_table" not in manifest.sources + config_test_table = manifest.sources.get("source.test.test_source.test_table").config + + assert isinstance(config_test_table, SourceConfig) + assert config_test_table == pytest.expected_config