diff --git a/.changes/unreleased/Features-20231004-103938.yaml b/.changes/unreleased/Features-20231004-103938.yaml new file mode 100644 index 000000000..37982fd3b --- /dev/null +++ b/.changes/unreleased/Features-20231004-103938.yaml @@ -0,0 +1,6 @@ +kind: Features +body: 'Add support for specifying groups and roles in grant statements' +time: 2023-10-04T10:39:38.680813-07:00 +custom: + Author: soksamnanglim + Issue: "415" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ff08b6190..164e1d9e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ To confirm you have correct `dbt-core` and adapter versions installed please run `dbt-redshift` contains [unit](https://github.com/dbt-labs/dbt-redshift/tree/main/tests/unit) and [functional](https://github.com/dbt-labs/dbt-redshift/tree/main/tests/functional) tests. Functional tests require testing against an actual Redshift warehouse. We have CI set up to test against a Redshift warehouse during PR checks. -In order to run functional tests locally, you will need a `test.env` file in the root of the repository that contains credentials for your Redshift warehouse. +In order to run functional tests locally, you will need a `test.env` file in the root of the repository that contains credentials for your Redshift warehouse. You'll need all the objects provided in `test.env.example` in `test.env` for all the tests to pass. Note: This `test.env` file is git-ignored, but please be extra careful to never check in credentials or other sensitive information when developing. To create your `test.env` file, copy the provided example file, then supply your relevant credentials. diff --git a/Makefile b/Makefile index f32c3ba8f..081940838 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,19 @@ .DEFAULT_GOAL:=help +CI_FLAGS =\ + DBT_TEST_USER_1=$(if $(DBT_TEST_USER_1),$(DBT_TEST_USER_1),dbt_test_user_1)\ + DBT_TEST_USER_2=$(if $(DBT_TEST_USER_2),$(DBT_TEST_USER_2),dbt_test_user_2)\ + DBT_TEST_USER_3=$(if $(DBT_TEST_USER_3),$(DBT_TEST_USER_3),dbt_test_user_3)\ + DBT_TEST_GROUP_1=$(if $(DBT_TEST_GROUP_1),$(DBT_TEST_GROUP_1),dbt_test_group_1)\ + DBT_TEST_GROUP_2=$(if $(DBT_TEST_GROUP_2),$(DBT_TEST_GROUP_2),dbt_test_group_2)\ + DBT_TEST_GROUP_3=$(if $(DBT_TEST_GROUP_3),$(DBT_TEST_GROUP_3),dbt_test_group_3)\ + DBT_TEST_ROLE_1=$(if $(DBT_TEST_ROLE_1),$(DBT_TEST_ROLE_1),dbt_test_role_1)\ + DBT_TEST_ROLE_2=$(if $(DBT_TEST_ROLE_2),$(DBT_TEST_ROLE_2),dbt_test_role_2)\ + DBT_TEST_ROLE_3=$(if $(DBT_TEST_ROLE_3),$(DBT_TEST_ROLE_3),dbt_test_role_3)\ + RUSTFLAGS=$(if $(RUSTFLAGS),$(RUSTFLAGS),"-D warnings")\ + LOG_DIR=$(if $(LOG_DIR),$(LOG_DIR),./logs)\ + DBT_LOG_FORMAT=$(if $(DBT_LOG_FORMAT),$(DBT_LOG_FORMAT),json) + .PHONY: dev dev: ## Installs adapter in develop mode along with development dependencies @\ diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index d498685ed..4fe8b0651 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -187,3 +187,93 @@ def generate_python_submission_response(self, submission_result: Any) -> Adapter def debug_query(self): """Override for DebugTask method""" self.execute("select 1 as id") + + # grant-related methods + @available + def standardize_grants_dict(self, grants_table): + """ + Override for standardize_grants_dict + """ + grants_dict = {} # Dict[str, Dict[str, List[str]]] + for row in grants_table: + grantee_type = row["grantee_type"] + grantee = row["grantee"] + privilege = row["privilege_type"] + if privilege not in grants_dict: + grants_dict[privilege] = {} + + if grantee_type not in grants_dict[privilege]: + grants_dict[privilege][grantee_type] = [] + + grants_dict[privilege][grantee_type].append(grantee) + + return grants_dict + + @available + def diff_of_two_nested_dicts(self, dict_a, dict_b): + """ + Given two dictionaries of type Dict[str, Dict[str, List[str]]]: + dict_a = {'key_x': {'key_a': 'VALUE_1'}, 'KEY_Y': {'key_b': value_2'}} + dict_b = {'key_x': {'key_a': 'VALUE_1'}, 'KEY_Y': {'key_b': value_2'}} + Return the same dictionary representation of dict_a MINUS dict_b, + performing a case-insensitive comparison between the strings in each. + All keys returned will be in the original case of dict_a. + returns {'key_x': ['VALUE_2'], 'KEY_Y': ['value_3']} + """ + + dict_diff = {} + + for k, v_a in dict_a.items(): + if k.casefold() in dict_b: + v_b = dict_b[k.casefold()] + + for sub_key, values_a in v_a.items(): + if sub_key in v_b: + values_b = v_b[sub_key] + diff_values = [v for v in values_a if v.casefold() not in values_b] + if diff_values: + if k in dict_diff: + dict_diff[k][sub_key] = diff_values + else: + dict_diff[k] = {sub_key: diff_values} + else: + if k in dict_diff: + if values_a: + dict_diff[k][sub_key] = values_a + else: + if values_a: + dict_diff[k] = {sub_key: values_a} + else: + dict_diff[k] = v_a + + return dict_diff + + @available + def process_grant_dicts(self, unknown_dict): + """ + Given a dictionary where the type can either be of type: + - Dict[str, List[str]] + - Dict[str, List[Dict[str, List[str]] + Return a processed dictionary of the type Dict[str, Dict[str, List[str]] + """ + first_value = next(iter(unknown_dict.values())) + if first_value: + is_dict = isinstance(first_value[0], dict) + else: + is_dict = False + + temp = {} + if not is_dict: + for privilege, grantees in unknown_dict.items(): + if grantees: + temp[privilege] = {"user": grantees} + else: + for privilege, grantee_map in unknown_dict.items(): + grantees_map_temp = {} + for grantee_type, grantees in grantee_map[0].items(): + if grantees: + grantees_map_temp[grantee_type] = grantees + if grantees_map_temp: + temp[privilege] = grantees_map_temp + + return temp diff --git a/dbt/include/redshift/macros/adapters/apply_grants.sql b/dbt/include/redshift/macros/adapters/apply_grants.sql index fa6523a26..2351a66a1 100644 --- a/dbt/include/redshift/macros/adapters/apply_grants.sql +++ b/dbt/include/redshift/macros/adapters/apply_grants.sql @@ -1,5 +1,42 @@ -{% macro redshift__get_show_grant_sql(relation) %} +{# ------- DCL STATEMENT TEMPLATES ------- #} + +{%- macro redshift__get_grant_sql(relation, privilege, grantee_dict) -%} + {#-- generates a multiple-grantees grant privilege statement --#} + grant {{privilege}} on {{relation}} to + {%- for grantee_type, grantees in grantee_dict.items() -%} + {%- if grantee_type == 'user' and grantees -%} + {{ " " + (grantees | join(', ')) }} + {%- elif grantee_type == 'group' and grantees -%} + {{ " " + ("group " + grantees | join(', group ')) }} + {%- elif grantee_type == 'role' and grantees -%} + {{ " " + ("role " + grantees | join(', role ')) }} + {%- endif -%} + {%- if not loop.last -%} + , + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} + +{%- macro redshift__get_revoke_sql(relation, privilege, revokee_dict) -%} + {#-- generates a multiple-grantees revoke privilege statement --#} + revoke {{privilege}} on {{relation}} from + {%- for revokee_type, revokees in revokee_dict.items() -%} + {%- if revokee_type == 'user' and revokees -%} + {{ " " + (revokees | join(', ')) }} + {%- elif revokee_type == 'group' and revokees -%} + {{ " " + ("group " + revokees | join(', group ')) }} + {%- elif revokee_type == 'role' and revokees -%} + {{ " " + ("role " + revokees | join(', role ')) }} + {%- endif -%} + {%- if not loop.last -%} + , + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} + +{% macro redshift__get_show_grant_sql(relation) %} +{#-- shows the privilege grants on a table for users, groups, and roles --#} with privileges as ( -- valid options per https://docs.aws.amazon.com/redshift/latest/dg/r_HAS_TABLE_PRIVILEGE.html @@ -16,6 +53,7 @@ with privileges as ( ) select + 'user' as grantee_type, u.usename as grantee, p.privilege_type from pg_user u @@ -24,4 +62,72 @@ where has_table_privilege(u.usename, '{{ relation }}', privilege_type) and u.usename != current_user and not u.usesuper +union all +-- check that group has table privilege +select + 'group' as grantee_type, + g.groname as grantee, + p.privilege_type +from pg_group g +cross join privileges p +where exists( + select * + from information_schema.table_privileges tp + where tp.grantee=g.groname + and tp.table_schema=replace(split_part('{{ relation }}', '.', 2), '"', '') + and tp.table_name=replace(split_part('{{ relation }}', '.', 3), '"', '') + and LOWER(tp.privilege_type)=p.privilege_type +) + +union all +-- check that role has table privilege +select + 'role' as grantee_type, + r.role_name as grantee, + p.privilege_type +from svv_roles r +cross join privileges p +where exists( + select * + from svv_relation_privileges rp + where rp.identity_name=r.role_name + and rp.namespace_name=replace(split_part('{{ relation }}', '.', 2), '"', '') + and rp.relation_name=replace(split_part('{{ relation }}', '.', 3), '"', '') + and LOWER(rp.privilege_type)=p.privilege_type +) + +{% endmacro %} + +{% macro redshift__apply_grants(relation, grant_config, should_revoke=True) %} + {#-- Override for apply grants --#} + {#-- If grant_config is {} or None, this is a no-op --#} + {% if grant_config %} + {#-- change grant_config to Dict[str, Dict[str, List[str]] format --#} + {% set grant_config = adapter.process_grant_dicts(grant_config) %} + + {% if should_revoke %} + {#-- We think that there is a chance that grants are carried over. --#} + {#-- Show the current grants for users, roles, and groups and calculate the diffs. --#} + {% set current_grants_table = run_query(get_show_grant_sql(relation)) %} + {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %} + {% set needs_granting = adapter.diff_of_two_nested_dicts(grant_config, current_grants_dict) %} + {% set needs_revoking = adapter.diff_of_two_nested_dicts(current_grants_dict, grant_config) %} + {% if not (needs_granting or needs_revoking) %} + {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}} + {% endif %} + {% else %} + {#-- We don't think there's any chance of previous grants having carried over. --#} + {#-- Jump straight to granting what the user has configured. --#} + {% set needs_revoking = {} %} + {% set needs_granting = grant_config %} + {% endif %} + {% if needs_granting or needs_revoking %} + {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %} + {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %} + {% set dcl_statement_list = revoke_statement_list + grant_statement_list %} + {% if dcl_statement_list %} + {{ call_dcl_statements(dcl_statement_list) }} + {% endif %} + {% endif %} + {% endif %} {% endmacro %} diff --git a/scripts/env-setup.sh b/scripts/env-setup.sh index 866d8f749..844958b08 100644 --- a/scripts/env-setup.sh +++ b/scripts/env-setup.sh @@ -8,3 +8,9 @@ echo "INTEGRATION_TESTS_SECRETS_PREFIX=REDSHIFT_TEST" >> $GITHUB_ENV echo "DBT_TEST_USER_1=dbt_test_user_1" >> $GITHUB_ENV echo "DBT_TEST_USER_2=dbt_test_user_2" >> $GITHUB_ENV echo "DBT_TEST_USER_3=dbt_test_user_3" >> $GITHUB_ENV +echo "DBT_TEST_GROUP_1=dbt_test_group_1" >> $GITHUB_ENV +echo "DBT_TEST_GROUP_2=dbt_test_group_2" >> $GITHUB_ENV +echo "DBT_TEST_GROUP_3=dbt_test_group_3" >> $GITHUB_ENV +echo "DBT_TEST_ROLE_1=dbt_test_role_1" >> $GITHUB_ENV +echo "DBT_TEST_ROLE_2=dbt_test_role_2" >> $GITHUB_ENV +echo "DBT_TEST_ROLE_3=dbt_test_role_3" >> $GITHUB_ENV diff --git a/test.env.example b/test.env.example index 6816b4ec2..17360a6d8 100644 --- a/test.env.example +++ b/test.env.example @@ -24,3 +24,13 @@ REDSHIFT_TEST_IAM_ROLE_PROFILE= DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 + +# Database groups for testing +DBT_TEST_GROUP_1=dbt_test_group_1 +DBT_TEST_GROUP_2=dbt_test_group_2 +DBT_TEST_GROUP_3=dbt_test_group_3 + +# Database roles for testing +DBT_TEST_ROLE_1=dbt_test_role_1 +DBT_TEST_ROLE_2=dbt_test_role_2 +DBT_TEST_ROLE_3=dbt_test_role_3 diff --git a/tests/functional/adapter/conftest.py b/tests/functional/adapter/conftest.py index c5c980154..53a9405b4 100644 --- a/tests/functional/adapter/conftest.py +++ b/tests/functional/adapter/conftest.py @@ -1,4 +1,47 @@ import pytest +import os + +from dbt_common.exceptions import DbtDatabaseError + +# This is a hack to prevent the fixture from running more than once +GRANTS_AND_ROLES_SETUP = False + +GROUPS = { + "DBT_TEST_GROUP_1": "dbt_test_group_1", + "DBT_TEST_GROUP_2": "dbt_test_group_2", + "DBT_TEST_GROUP_3": "dbt_test_group_3", +} +ROLES = { + "DBT_TEST_ROLE_1": "dbt_test_role_1", + "DBT_TEST_ROLE_2": "dbt_test_role_2", + "DBT_TEST_ROLE_3": "dbt_test_role_3", +} + + +@pytest.fixture(scope="class", autouse=True) +def setup_grants_and_roles(project): + global GRANTS_AND_ROLES_SETUP + for env_name, env_var in GROUPS.items(): + os.environ[env_name] = env_var + for env_name, env_var in ROLES.items(): + os.environ[env_name] = env_var + if not GRANTS_AND_ROLES_SETUP: + with project.adapter.connection_named("__test"): + for group in GROUPS.values(): + try: + project.adapter.execute(f"CREATE GROUP {group}") + except DbtDatabaseError: + # This is expected if the group already exists + pass + + for role in ROLES.values(): + try: + project.adapter.execute(f"CREATE ROLE {role}") + except DbtDatabaseError: + # This is expected if the group already exists + pass + + GRANTS_AND_ROLES_SETUP = True @pytest.fixture diff --git a/tests/functional/adapter/grants/base_grants.py b/tests/functional/adapter/grants/base_grants.py new file mode 100644 index 000000000..a65e3e26e --- /dev/null +++ b/tests/functional/adapter/grants/base_grants.py @@ -0,0 +1,79 @@ +import pytest +import os + +from dbt.tests.util import ( + relation_from_name, + get_connection, +) + +TEST_USER_ENV_VARS = ["DBT_TEST_USER_1", "DBT_TEST_USER_2", "DBT_TEST_USER_3"] +TEST_GROUP_ENV_VARS = ["DBT_TEST_GROUP_1", "DBT_TEST_GROUP_2", "DBT_TEST_GROUP_3"] +TEST_ROLE_ENV_VARS = ["DBT_TEST_ROLE_1", "DBT_TEST_ROLE_2", "DBT_TEST_ROLE_3"] + + +def replace_all(text, dic): + for i, j in dic.items(): + text = text.replace(i, j) + return text + + +class BaseGrantsRedshift: + def privilege_grantee_name_overrides(self): + # these privilege and grantee names are valid on most databases, but not all! + # looking at you, BigQuery + # optionally use this to map from "select" --> "other_select_name", "insert" --> ... + return { + "select": "select", + "insert": "insert", + "fake_privilege": "fake_privilege", + "invalid_user": "invalid_user", + } + + def interpolate_name_overrides(self, yaml_text): + return replace_all(yaml_text, self.privilege_grantee_name_overrides()) + + @pytest.fixture(scope="class", autouse=True) + def get_test_groups(self, project): + test_groups = [] + for env_var in TEST_GROUP_ENV_VARS: + group_name = os.getenv(env_var) + if group_name: + test_groups.append(group_name) + return test_groups + + @pytest.fixture(scope="class", autouse=True) + def get_test_roles(self, project): + test_roles = [] + for env_var in TEST_ROLE_ENV_VARS: + role_name = os.getenv(env_var) + if role_name: + test_roles.append(role_name) + return test_roles + + @pytest.fixture(scope="class", autouse=True) + def get_test_users(self, project): + test_users = [] + for env_var in TEST_USER_ENV_VARS: + user_name = os.getenv(env_var) + if user_name: + test_users.append(user_name) + return test_users + + def get_grants_on_relation(self, project, relation_name): + relation = relation_from_name(project.adapter, relation_name) + adapter = project.adapter + with get_connection(adapter): + kwargs = {"relation": relation} + show_grant_sql = adapter.execute_macro("get_show_grant_sql", kwargs=kwargs) + _, grant_table = adapter.execute(show_grant_sql, fetch=True) + actual_grants = adapter.standardize_grants_dict(grant_table) + return actual_grants + + # This is an override of the BaseGrants class + def assert_expected_grants_match_actual(self, project, actual_grants, expected_grants): + adapter = project.adapter + # need a case-insensitive comparison + # so just a simple "assert expected == actual_grants" won't work + diff_a = adapter.diff_of_two_nested_dicts(actual_grants, expected_grants) + diff_b = adapter.diff_of_two_nested_dicts(expected_grants, actual_grants) + assert diff_a == diff_b == {} diff --git a/tests/functional/adapter/grants/test_incremental_grants.py b/tests/functional/adapter/grants/test_incremental_grants.py new file mode 100644 index 000000000..1e47d4360 --- /dev/null +++ b/tests/functional/adapter/grants/test_incremental_grants.py @@ -0,0 +1,178 @@ +import pytest +from dbt.tests.util import ( + run_dbt, + run_dbt_and_capture, + get_manifest, + write_file, + relation_from_name, + get_connection, +) + +from tests.functional.adapter.grants.base_grants import BaseGrantsRedshift + +my_incremental_model_sql = """ + select 1 as fun +""" + +incremental_model_schema_yml = """ +version: 2 +models: + - name: my_incremental_model + config: + materialized: incremental + grants: + select: ["{{ env_var('DBT_TEST_USER_1') }}"] +""" + +user2_incremental_model_schema_yml = """ +version: 2 +models: + - name: my_incremental_model + config: + materialized: incremental + grants: + select: ["{{ env_var('DBT_TEST_USER_2') }}"] +""" + +extended_incremental_model_schema_yml = """ +version: 2 +models: + - name: my_incremental_model + config: + materialized: incremental + grants: + select: + user: ["{{ env_var('DBT_TEST_USER_1') }}"] + group: ["{{ env_var('DBT_TEST_GROUP_1') }}"] + role: ["{{ env_var('DBT_TEST_ROLE_1') }}"] +""" + +extended2_incremental_model_schema_yml = """ +version: 2 +models: + - name: my_incremental_model + config: + materialized: incremental + grants: + select: + user: ["{{ env_var('DBT_TEST_USER_2') }}"] + group: ["{{ env_var('DBT_TEST_GROUP_2') }}"] + role: ["{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + + +class TestIncrementalGrantsRedshift(BaseGrantsRedshift): + @pytest.fixture(scope="class") + def models(self): + updated_schema = self.interpolate_name_overrides(incremental_model_schema_yml) + return { + "my_incremental_model.sql": my_incremental_model_sql, + "schema.yml": updated_schema, + } + + def test_incremental_grants(self, project, get_test_users, get_test_groups, get_test_roles): + # we want the test to fail, not silently skip + test_users = get_test_users + test_groups = get_test_groups + test_roles = get_test_roles + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + assert len(test_users) == 3 + assert len(test_groups) == 3 + assert len(test_roles) == 3 + + # Incremental materialization, single select grant + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model_id = "model.test.my_incremental_model" + model = manifest.nodes[model_id] + assert model.config.materialized == "incremental" + expected = {select_privilege_name: {"user": [test_users[0]]}} + actual_grants = self.get_grants_on_relation(project, "my_incremental_model") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Incremental materialization, run again without changes + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + assert "revoke " not in log_output + assert "grant " not in log_output # with space to disambiguate from 'show grants' + actual_grants = self.get_grants_on_relation(project, "my_incremental_model") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Incremental materialization, change select grant user + updated_yaml = self.interpolate_name_overrides(user2_incremental_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + assert "revoke " in log_output + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "incremental" + expected = {select_privilege_name: {"user": [test_users[1]]}} + actual_grants = self.get_grants_on_relation(project, "my_incremental_model") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Incremental materialization, same config, now with --full-refresh + run_dbt(["--debug", "run", "--full-refresh"]) + assert len(results) == 1 + # whether grants or revokes happened will vary by adapter + actual_grants = self.get_grants_on_relation(project, "my_incremental_model") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Now drop the schema (with the table in it) + adapter = project.adapter + relation = relation_from_name(adapter, "my_incremental_model") + with get_connection(adapter): + adapter.drop_schema(relation) + + # Incremental materialization, same config, rebuild now that table is missing + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + assert "grant " in log_output + assert "revoke " not in log_output + actual_grants = self.get_grants_on_relation(project, "my_incremental_model") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Additional tests for privilege grants to extended permission types + # Incremental materialization, single select grant + updated_yaml = self.interpolate_name_overrides(extended_incremental_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + # select grant to *_1 users, groups, and roles, select revoke from DBT_TEST_USER_2 + assert "grant " in log_output + assert "revoke " in log_output + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "incremental" + expected = { + select_privilege_name: { + "user": [test_users[0]], + "group": [test_groups[0]], + "role": [test_roles[0]], + } + } + + actual_grants = self.get_grants_on_relation(project, "my_incremental_model") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Incremental materialization, change select grant + updated_yaml = self.interpolate_name_overrides(extended2_incremental_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + # select grant to *_2 users,groups, and roles, select revoke from *_1 users, groups, and roles + assert "grant " in log_output + assert "revoke " in log_output + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "incremental" + expected = { + select_privilege_name: { + "user": [test_users[1]], + "group": [test_groups[1]], + "role": [test_roles[1]], + } + } + actual_grants = self.get_grants_on_relation(project, "my_incremental_model") + self.assert_expected_grants_match_actual(project, actual_grants, expected) diff --git a/tests/functional/adapter/grants/test_model_table_grants.py b/tests/functional/adapter/grants/test_model_table_grants.py new file mode 100644 index 000000000..2e9108722 --- /dev/null +++ b/tests/functional/adapter/grants/test_model_table_grants.py @@ -0,0 +1,271 @@ +import pytest +from dbt.tests.util import ( + run_dbt, + get_manifest, + write_file, +) +from tests.functional.adapter.grants.base_grants import BaseGrantsRedshift + +my_model_sql = """ + select 1 as fun +""" + +table_model_schema_yml = """ +version: 2 +models: + - name: my_model_table + config: + materialized: table + grants: + select: ["{{ env_var('DBT_TEST_USER_1') }}"] +""" + +user2_table_model_schema_yml = """ +version: 2 +models: + - name: my_model_table + config: + materialized: table + grants: + select: ["{{ env_var('DBT_TEST_USER_2') }}"] +""" + +multiple_users_table_model_schema_yml = """ +version: 2 +models: + - name: my_model_table + config: + materialized: table + grants: + select: ["{{ env_var('DBT_TEST_USER_1') }}", "{{ env_var('DBT_TEST_USER_2') }}"] +""" + +multiple_privileges_table_model_schema_yml = """ +version: 2 +models: + - name: my_model_table + config: + materialized: table + grants: + select: ["{{ env_var('DBT_TEST_USER_1') }}"] + insert: ["{{ env_var('DBT_TEST_USER_2') }}"] +""" + +# table materialization single select +extended_table_model_schema_yml = """ +version: 2 +models: + - name: my_model_table + config: + materialized: table + grants: + select: + user: ["{{ env_var('DBT_TEST_USER_1') }}"] + group: ["{{ env_var('DBT_TEST_GROUP_1') }}"] + role: ["{{ env_var('DBT_TEST_ROLE_1') }}"] +""" + +# table materialization change select +extended2_table_model_schema_yml = """ +version: 2 +models: + - name: my_model_table + config: + materialized: table + grants: + select: + user: ["{{ env_var('DBT_TEST_USER_2') }}"] + group: ["{{ env_var('DBT_TEST_GROUP_2') }}"] + role: ["{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +# table materialization multiple grantees +extended_multiple_grantees_table_model_schema_yml = """ +version: 2 +models: + - name: my_model_table + config: + materialized: table + grants: + select: + user: ["{{ env_var('DBT_TEST_USER_1') }}", "{{ env_var('DBT_TEST_USER_2') }}"] + group: ["{{ env_var('DBT_TEST_GROUP_1') }}", "{{ env_var('DBT_TEST_GROUP_2') }}"] + role: ["{{ env_var('DBT_TEST_ROLE_1') }}", "{{ env_var('DBT_TEST_ROLE_2') }}"] +""" +# table materialization multiple privileges +extended_multiple_privileges_table_model_schema_yml = """ +version: 2 +models: + - name: my_model_table + config: + materialized: table + grants: + select: + user: ["{{ env_var('DBT_TEST_USER_1') }}"] + group: ["{{ env_var('DBT_TEST_GROUP_1') }}"] + role: ["{{ env_var('DBT_TEST_ROLE_1') }}"] + insert: + user: ["{{ env_var('DBT_TEST_USER_2') }}"] + group: ["{{ env_var('DBT_TEST_GROUP_2') }}"] + role: ["{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + + +class TestModelGrantsTableRedshift(BaseGrantsRedshift): + @pytest.fixture(scope="class") + def models(self): + updated_schema = self.interpolate_name_overrides(table_model_schema_yml) + return { + "my_model_table.sql": my_model_sql, + "schema.yml": updated_schema, + } + + def test_table_grants(self, project, get_test_users, get_test_groups, get_test_roles): + # Override/refactor the tests from dbt-core # + # we want the test to fail, not silently skip + test_users = get_test_users + test_groups = get_test_groups + test_roles = get_test_roles + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + insert_privilege_name = self.privilege_grantee_name_overrides()["insert"] + assert len(test_users) == 3 + assert len(test_groups) == 3 + assert len(test_roles) == 3 + + # Table materialization, single select grant + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_table" + model = manifest.nodes[model_id] + assert model.config.materialized == "table" + expected = {select_privilege_name: {"user": [test_users[0]]}} + actual_grants = self.get_grants_on_relation(project, "my_model_table") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Table materialization, change select grant user + updated_yaml = self.interpolate_name_overrides(user2_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "table" + expected = {select_privilege_name: {"user": [test_users[1]]}} + actual_grants = self.get_grants_on_relation(project, "my_model_table") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Table materialization, multiple grantees + updated_yaml = self.interpolate_name_overrides(multiple_users_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "table" + expected = {select_privilege_name: {"user": [test_users[0], test_users[1]]}} + actual_grants = self.get_grants_on_relation(project, "my_model_table") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Table materialization, multiple privileges + updated_yaml = self.interpolate_name_overrides(multiple_privileges_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + user_expected = { + select_privilege_name: [test_users[0]], + insert_privilege_name: [test_users[1]], + } + assert model.config.grants == user_expected + assert model.config.materialized == "table" + expected = { + select_privilege_name: {"user": [test_users[0]]}, + insert_privilege_name: {"user": [test_users[1]]}, + } + actual_grants = self.get_grants_on_relation(project, "my_model_table") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Additional tests for privilege grants to extended permission types + # Table materialization, single select grant + updated_yaml = self.interpolate_name_overrides(extended_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "table" + expected = { + select_privilege_name: { + "user": [test_users[0]], + "group": [test_groups[0]], + "role": [test_roles[0]], + } + } + + actual_grants = self.get_grants_on_relation(project, "my_model_table") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Table materialization, change select grant + updated_yaml = self.interpolate_name_overrides(extended2_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "table" + expected = { + select_privilege_name: { + "user": [test_users[1]], + "group": [test_groups[1]], + "role": [test_roles[1]], + } + } + actual_grants = self.get_grants_on_relation(project, "my_model_table") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Table materialization, multiple grantees + updated_yaml = self.interpolate_name_overrides( + extended_multiple_grantees_table_model_schema_yml + ) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "table" + expected = { + select_privilege_name: { + "user": [test_users[0], test_users[1]], + "group": [test_groups[0], test_groups[1]], + "role": [test_roles[0], test_roles[1]], + } + } + actual_grants = self.get_grants_on_relation(project, "my_model_table") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # Table materialization, multiple privileges + updated_yaml = self.interpolate_name_overrides( + extended_multiple_privileges_table_model_schema_yml + ) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "table" + expected = { + select_privilege_name: { + "user": [test_users[0]], + "group": [test_groups[0]], + "role": [test_roles[0]], + }, + insert_privilege_name: { + "user": [test_users[1]], + "group": [test_groups[1]], + "role": [test_roles[1]], + }, + } + actual_grants = self.get_grants_on_relation(project, "my_model_table") + self.assert_expected_grants_match_actual(project, actual_grants, expected) diff --git a/tests/functional/adapter/grants/test_model_view_grants.py b/tests/functional/adapter/grants/test_model_view_grants.py new file mode 100644 index 000000000..ebf1055a6 --- /dev/null +++ b/tests/functional/adapter/grants/test_model_view_grants.py @@ -0,0 +1,81 @@ +from base_grants import BaseGrantsRedshift +import pytest +from dbt.tests.util import ( + run_dbt, + get_manifest, + write_file, +) + +my_model_sql = """ + select 1 as fun +""" + +model_schema_yml = """ +version: 2 +models: + - name: my_model_view + config: + materialized: view + grants: + select: ["{{ env_var('DBT_TEST_USER_1') }}"] +""" + +user2_model_schema_yml = """ +version: 2 +models: + - name: my_model_view + config: + materialized: view + grants: + select: ["{{ env_var('DBT_TEST_USER_2') }}"] +""" + + +class TestModelGrantsViewRedshift(BaseGrantsRedshift): + @pytest.fixture(scope="class") + def models(self): + updated_schema = self.interpolate_name_overrides(model_schema_yml) + return { + "my_model_view.sql": my_model_sql, + "schema.yml": updated_schema, + } + + def test_view_table_grants(self, project, get_test_users, get_test_groups, get_test_roles): + # Override/refactor the tests from dbt-core # + # we want the test to fail, not silently skip + test_users = get_test_users + test_groups = get_test_groups + test_roles = get_test_roles + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + assert len(test_users) == 3 + assert len(test_groups) == 3 + assert len(test_roles) == 3 + + # View materialization, single select grant + updated_yaml = self.interpolate_name_overrides(model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_view" + model = manifest.nodes[model_id] + # user configuration for grants + user_expected = {select_privilege_name: [test_users[0]]} + assert model.config.grants == user_expected + assert model.config.materialized == "view" + # new configuration for grants + expected = {select_privilege_name: {"user": [test_users[0]]}} + + actual_grants = self.get_grants_on_relation(project, "my_model_view") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # View materialization, change select grant user + updated_yaml = self.interpolate_name_overrides(user2_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + + expected = {select_privilege_name: {"user": [test_users[1]]}} + actual_grants = self.get_grants_on_relation(project, "my_model_view") + self.assert_expected_grants_match_actual(project, actual_grants, expected) diff --git a/tests/functional/adapter/grants/test_seed_grants.py b/tests/functional/adapter/grants/test_seed_grants.py new file mode 100644 index 000000000..feb5cda2f --- /dev/null +++ b/tests/functional/adapter/grants/test_seed_grants.py @@ -0,0 +1,177 @@ +import pytest +from dbt.tests.util import ( + run_dbt, + run_dbt_and_capture, + get_manifest, + write_file, +) +from tests.functional.adapter.grants.base_grants import BaseGrantsRedshift + + +seeds__my_seed_csv = """ +id,name,some_date +1,Easton,1981-05-20T06:46:51 +2,Lillian,1978-09-03T18:10:33 +""".lstrip() + +# rewrite this +schema_base_yml = """ +version: 2 +seeds: + - name: my_seed + config: + grants: + select: ["{{ env_var('DBT_TEST_USER_1') }}"] +""" + +# rewrite this +user2_schema_base_yml = """ +version: 2 +seeds: + - name: my_seed + config: + grants: + select: ["{{ env_var('DBT_TEST_USER_2') }}"] +""" + +ignore_grants_yml = """ +version: 2 +seeds: + - name: my_seed + config: + grants: {} +""" + +zero_grants_yml = """ +version: 2 +seeds: + - name: my_seed + config: + grants: + select: [] +""" + +extended_zero_grants_yml = """ +version: 2 +seeds: + - name: my_seed + config: + grants: + select: + user: [] + group: [] + role: [] +""" + + +class TestSeedGrantsRedshift(BaseGrantsRedshift): + def seeds_support_partial_refresh(self): + return True + + @pytest.fixture(scope="class") + def seeds(self): + updated_schema = self.interpolate_name_overrides(schema_base_yml) + return { + "my_seed.csv": seeds__my_seed_csv, + "schema.yml": updated_schema, + } + + def test_seed_grants(self, project, get_test_users): + # debugging for seeds + print("seed testing") + + test_users = get_test_users + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + + # seed command + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + seed_id = "seed.test.my_seed" + seed = manifest.nodes[seed_id] + user_expected = {select_privilege_name: [test_users[0]]} + assert seed.config.grants == user_expected + assert "grant " in log_output + expected = {select_privilege_name: {"user": [test_users[0]]}} + actual_grants = self.get_grants_on_relation(project, "my_seed") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # run it again, with no config changes + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + if self.seeds_support_partial_refresh(): + # grants carried over -- nothing should have changed + assert "revoke " not in log_output + assert "grant " not in log_output + else: + # seeds are always full-refreshed on this adapter, so we need to re-grant + assert "grant " in log_output + actual_grants = self.get_grants_on_relation(project, "my_seed") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # change the grantee, assert it updates + updated_yaml = self.interpolate_name_overrides(user2_schema_base_yml) + write_file(updated_yaml, project.project_root, "seeds", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + expected = {select_privilege_name: {"user": [test_users[1]]}} + actual_grants = self.get_grants_on_relation(project, "my_seed") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # run it again, with --full-refresh, grants should be the same + run_dbt(["seed", "--full-refresh"]) + actual_grants = self.get_grants_on_relation(project, "my_seed") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # change config to 'grants: {}' -- should be completely ignored + updated_yaml = self.interpolate_name_overrides(ignore_grants_yml) + write_file(updated_yaml, project.project_root, "seeds", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + assert "revoke " not in log_output + assert "grant " not in log_output + manifest = get_manifest(project.project_root) + seed_id = "seed.test.my_seed" + seed = manifest.nodes[seed_id] + expected_config = {} + expected_actual = {select_privilege_name: {"user": [test_users[1]]}} + assert seed.config.grants == expected_config + actual_grants = self.get_grants_on_relation(project, "my_seed") + if self.seeds_support_partial_refresh(): + # ACTUAL grants will NOT match expected grants + self.assert_expected_grants_match_actual(project, actual_grants, expected_actual) + else: + # there should be ZERO grants on the seed + self.assert_expected_grants_match_actual(project, actual_grants, expected_config) + + # now run with ZERO grants -- all grants should be removed + # whether explicitly (revoke) or implicitly (recreated without any grants added on) + updated_yaml = self.interpolate_name_overrides(zero_grants_yml) + write_file(updated_yaml, project.project_root, "seeds", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + if self.seeds_support_partial_refresh(): + assert "revoke " in log_output + expected = {} + actual_grants = self.get_grants_on_relation(project, "my_seed") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # run it again -- dbt shouldn't try to grant or revoke anything + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + assert "revoke " not in log_output + assert "grant " not in log_output + actual_grants = self.get_grants_on_relation(project, "my_seed") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # run with ZERO grants -- since all grants were removed previously + # nothing should have changed + updated_yaml = self.interpolate_name_overrides(extended_zero_grants_yml) + write_file(updated_yaml, project.project_root, "seeds", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + assert "grant " not in log_output + assert "revoke " not in log_output + expected = {} + actual_grants = self.get_grants_on_relation(project, "my_seed") + self.assert_expected_grants_match_actual(project, actual_grants, expected) diff --git a/tests/functional/adapter/grants/test_snapshot_grants.py b/tests/functional/adapter/grants/test_snapshot_grants.py new file mode 100644 index 000000000..bc0d3691a --- /dev/null +++ b/tests/functional/adapter/grants/test_snapshot_grants.py @@ -0,0 +1,137 @@ +import pytest +from dbt.tests.util import ( + run_dbt, + run_dbt_and_capture, + get_manifest, + write_file, +) + +from tests.functional.adapter.grants.base_grants import BaseGrantsRedshift + + +my_snapshot_sql = """ +{% snapshot my_snapshot %} + {{ config( + check_cols='all', unique_key='id', strategy='check', + target_database=database, target_schema=schema + ) }} + select 1 as id, cast('blue' as {{ type_string() }}) as color +{% endsnapshot %} +""".strip() + +snapshot_schema_yml = """ +version: 2 +snapshots: + - name: my_snapshot + config: + grants: + select: ["{{ env_var('DBT_TEST_USER_1') }}"] +""" + +user2_snapshot_schema_yml = """ +version: 2 +snapshots: + - name: my_snapshot + config: + grants: + select: ["{{ env_var('DBT_TEST_USER_2') }}"] +""" + +extended_snapshot_schema_yml = """ +version: 2 +snapshots: + - name: my_snapshot + config: + grants: + select: + user: ["{{ env_var('DBT_TEST_USER_1') }}"] + group: ["{{ env_var('DBT_TEST_GROUP_1') }}"] + role: ["{{ env_var('DBT_TEST_ROLE_1') }}"] +""" + +extended2_snapshot_schema_yml = """ +version: 2 +snapshots: + - name: my_snapshot + config: + grants: + select: + user: ["{{ env_var('DBT_TEST_USER_2') }}"] + group: ["{{ env_var('DBT_TEST_GROUP_2') }}"] + role: ["{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + + +class TestSnapshotGrantsRedshift(BaseGrantsRedshift): + @pytest.fixture(scope="class") + def snapshots(self): + return { + "my_snapshot.sql": my_snapshot_sql, + "schema.yml": self.interpolate_name_overrides(snapshot_schema_yml), + } + + def test_snapshot_grants(self, project, get_test_users, get_test_groups, get_test_roles): + print("snapshot testing") + test_users = get_test_users + test_groups = get_test_groups + test_roles = get_test_roles + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + + # run the snapshot + results = run_dbt(["snapshot"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + snapshot_id = "snapshot.test.my_snapshot" + snapshot = manifest.nodes[snapshot_id] + user_expected = {select_privilege_name: [test_users[0]]} + assert snapshot.config.grants == user_expected + expected = {select_privilege_name: {"user": [test_users[0]]}} + actual_grants = self.get_grants_on_relation(project, "my_snapshot") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # run it again, nothing should have changed + (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) + assert len(results) == 1 + assert "revoke " not in log_output + assert "grant " not in log_output + actual_grants = self.get_grants_on_relation(project, "my_snapshot") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # change the grantee, assert it updates + updated_yaml = self.interpolate_name_overrides(user2_snapshot_schema_yml) + write_file(updated_yaml, project.project_root, "snapshots", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) + assert len(results) == 1 + expected = {select_privilege_name: {"user": [test_users[1]]}} + actual_grants = self.get_grants_on_relation(project, "my_snapshot") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # change the grants, assert that it updates + updated_yaml = self.interpolate_name_overrides(extended_snapshot_schema_yml) + write_file(updated_yaml, project.project_root, "snapshots", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) + assert len(results) == 1 + expected = { + select_privilege_name: { + "user": [test_users[0]], + "group": [test_groups[0]], + "role": [test_roles[0]], + } + } + actual_grants = self.get_grants_on_relation(project, "my_snapshot") + self.assert_expected_grants_match_actual(project, actual_grants, expected) + + # change the grants again, assert that it updates + updated_yaml = self.interpolate_name_overrides(extended2_snapshot_schema_yml) + write_file(updated_yaml, project.project_root, "snapshots", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) + assert len(results) == 1 + expected = { + select_privilege_name: { + "user": [test_users[1]], + "group": [test_groups[1]], + "role": [test_roles[1]], + } + } + actual_grants = self.get_grants_on_relation(project, "my_snapshot") + self.assert_expected_grants_match_actual(project, actual_grants, expected) diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py deleted file mode 100644 index b627e450a..000000000 --- a/tests/functional/adapter/test_grants.py +++ /dev/null @@ -1,24 +0,0 @@ -from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants -from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants -from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants -from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants - - -class TestModelGrantsRedshift(BaseModelGrants): - pass - - -class TestIncrementalGrantsRedshift(BaseIncrementalGrants): - pass - - -class TestSeedGrantsRedshift(BaseSeedGrants): - pass - - -class TestSnapshotGrantsRedshift(BaseSnapshotGrants): - pass - - -class TestInvalidGrantsRedshift(BaseModelGrants): - pass