From 5acc314196e58255ef6e6f83045b2e1c1beb7a73 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Tue, 23 Nov 2021 18:18:46 +0100 Subject: [PATCH] Refactor query tag tests (#57) * Refactor query tag tests * Try this * Try storing failures, too * Keep trying * Try it this way --- .../query_tag_tests/macros/check_tag.sql | 10 ++++ .../models/incremental_model_query_tag.sql | 3 ++ .../models/table_model_query_tag.sql | 3 ++ .../models/view_model_query_tag.sql | 3 ++ .../query_tag_tests/seeds/seed_query_tag.csv | 2 + .../snapshots/snapshot_query_tag.sql} | 3 +- .../query_tag_tests/test_query_tags.py | 46 +++++++++++++++++++ .../check_query_tag.sql | 16 ------- .../test_seed_with_query_tag.py | 41 ----------------- .../test_snapshot_query_tag.py | 34 -------------- 10 files changed, 68 insertions(+), 93 deletions(-) create mode 100644 tests/integration/query_tag_tests/macros/check_tag.sql create mode 100644 tests/integration/query_tag_tests/models/incremental_model_query_tag.sql create mode 100644 tests/integration/query_tag_tests/models/table_model_query_tag.sql create mode 100644 tests/integration/query_tag_tests/models/view_model_query_tag.sql create mode 100644 tests/integration/query_tag_tests/seeds/seed_query_tag.csv rename tests/integration/{simple_snapshot_test/check-snapshots-query-tag/snapshot.sql => query_tag_tests/snapshots/snapshot_query_tag.sql} (73%) create mode 100644 tests/integration/query_tag_tests/test_query_tags.py delete mode 100644 tests/integration/simple_seed_test/check-query-tag-expected/check_query_tag.sql delete mode 100644 tests/integration/simple_seed_test/test_seed_with_query_tag.py delete mode 100644 tests/integration/simple_snapshot_test/test_snapshot_query_tag.py diff --git a/tests/integration/query_tag_tests/macros/check_tag.sql b/tests/integration/query_tag_tests/macros/check_tag.sql new file mode 100644 index 000000000..3cf3232f1 --- /dev/null +++ b/tests/integration/query_tag_tests/macros/check_tag.sql @@ -0,0 +1,10 @@ +{% macro check_query_tag() %} + + {% if execute %} + {% set query_tag = get_current_query_tag() %} + {% if query_tag != var("query_tag") %} + {{ exceptions.raise_compiler_error("Query tag not used!") }} + {% endif %} + {% endif %} + +{% endmacro %} diff --git a/tests/integration/query_tag_tests/models/incremental_model_query_tag.sql b/tests/integration/query_tag_tests/models/incremental_model_query_tag.sql new file mode 100644 index 000000000..9f8ef50a9 --- /dev/null +++ b/tests/integration/query_tag_tests/models/incremental_model_query_tag.sql @@ -0,0 +1,3 @@ +{{ config(materialized = 'incremental', unique_key = 'id') }} + +select 1 as id diff --git a/tests/integration/query_tag_tests/models/table_model_query_tag.sql b/tests/integration/query_tag_tests/models/table_model_query_tag.sql new file mode 100644 index 000000000..7f2d6b752 --- /dev/null +++ b/tests/integration/query_tag_tests/models/table_model_query_tag.sql @@ -0,0 +1,3 @@ +{{ config(materialized = 'table') }} + +select 1 as id diff --git a/tests/integration/query_tag_tests/models/view_model_query_tag.sql b/tests/integration/query_tag_tests/models/view_model_query_tag.sql new file mode 100644 index 000000000..21ccea8dd --- /dev/null +++ b/tests/integration/query_tag_tests/models/view_model_query_tag.sql @@ -0,0 +1,3 @@ +{{ config(materialized = 'view') }} + +select 1 as id diff --git a/tests/integration/query_tag_tests/seeds/seed_query_tag.csv b/tests/integration/query_tag_tests/seeds/seed_query_tag.csv new file mode 100644 index 000000000..3ff3deb87 --- /dev/null +++ b/tests/integration/query_tag_tests/seeds/seed_query_tag.csv @@ -0,0 +1,2 @@ +id +1 diff --git a/tests/integration/simple_snapshot_test/check-snapshots-query-tag/snapshot.sql b/tests/integration/query_tag_tests/snapshots/snapshot_query_tag.sql similarity index 73% rename from tests/integration/simple_snapshot_test/check-snapshots-query-tag/snapshot.sql rename to tests/integration/query_tag_tests/snapshots/snapshot_query_tag.sql index 6f467cfc2..5af9ca777 100644 --- a/tests/integration/simple_snapshot_test/check-snapshots-query-tag/snapshot.sql +++ b/tests/integration/query_tag_tests/snapshots/snapshot_query_tag.sql @@ -6,8 +6,7 @@ unique_key='id', strategy='check', check_cols=['color'], - query_tag=var('query_tag') ) }} - select * from {{target.database}}.{{schema}}.seed + select 1 as id, 'blue' as color {% endsnapshot %} diff --git a/tests/integration/query_tag_tests/test_query_tags.py b/tests/integration/query_tag_tests/test_query_tags.py new file mode 100644 index 000000000..24cc7c52a --- /dev/null +++ b/tests/integration/query_tag_tests/test_query_tags.py @@ -0,0 +1,46 @@ +import os +import csv +from tests.integration.base import DBTIntegrationTest, use_profile + +class TestSeedWithQueryTag(DBTIntegrationTest): + @property + def schema(self): + return "query_tag" + + @property + def models(self): + return "models" + + @property + def project_config(self): + return { + 'config-version': 2, + 'models': { + 'test': { + 'query_tag': self.prefix, + 'post-hook': '{{ check_query_tag() }}' + }, + }, + 'seeds': { + 'test': { + 'query_tag': self.prefix, + 'post-hook': '{{ check_query_tag() }}' + }, + }, + 'snapshots': { + 'test': { + 'query_tag': self.prefix, + 'post-hook': '{{ check_query_tag() }}' + }, + }, + } + + def build_all_with_query_tags(self): + self.run_dbt(['build', '--vars', '{{"query_tag": "{}"}}'.format(self.prefix)]) + + @use_profile('snowflake') + def test__snowflake__build_tagged_twice(self): + self.build_all_with_query_tags() + self.build_all_with_query_tags() + + \ No newline at end of file diff --git a/tests/integration/simple_seed_test/check-query-tag-expected/check_query_tag.sql b/tests/integration/simple_seed_test/check-query-tag-expected/check_query_tag.sql deleted file mode 100644 index b193b40c4..000000000 --- a/tests/integration/simple_seed_test/check-query-tag-expected/check_query_tag.sql +++ /dev/null @@ -1,16 +0,0 @@ - - -with query as ( - - -- check that the current value for id=1 is red - select case when ( - select count(*) - from table(information_schema.query_history_by_user()) - where QUERY_TAG = '{{ var('query_tag') }}' - ) > 1 then 0 else 1 end as failures - -) - -select * -from query -where failures = 1 diff --git a/tests/integration/simple_seed_test/test_seed_with_query_tag.py b/tests/integration/simple_seed_test/test_seed_with_query_tag.py deleted file mode 100644 index 5d58b5a87..000000000 --- a/tests/integration/simple_seed_test/test_seed_with_query_tag.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import csv -from tests.integration.base import DBTIntegrationTest, use_profile - -class TestSeedWithQueryTag(DBTIntegrationTest): - @property - def schema(self): - return "simple_seed" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - "seed-paths": ['seeds-config'], - "test-paths": ['check-query-tag-expected'], - 'seeds': { - 'test': { - 'enabled': False, - 'quote_columns': True, - 'query_tag': self.prefix, - 'seed_enabled': { - 'enabled': True, - }, - }, - - } - } - - def assert_query_tag_expected(self): - self.run_dbt(['test', '--select', 'test_type:singular', '--vars', '{{"query_tag": {}}}'.format(self.prefix)]) - - @use_profile('snowflake') - def test_snowflake_big_batched_seed(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - self.assert_query_tag_expected() - \ No newline at end of file diff --git a/tests/integration/simple_snapshot_test/test_snapshot_query_tag.py b/tests/integration/simple_snapshot_test/test_snapshot_query_tag.py deleted file mode 100644 index f96b33bff..000000000 --- a/tests/integration/simple_snapshot_test/test_snapshot_query_tag.py +++ /dev/null @@ -1,34 +0,0 @@ -from tests.integration.base import DBTIntegrationTest, use_profile - -class TestSnapshotWithQueryTag(DBTIntegrationTest): - @property - def schema(self): - return "simple_snapshot_004" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - "snapshot-paths": ['check-snapshots-query-tag'], - "test-paths": ['check-snapshots-query-tag-expected'], - "model-paths": [], - } - - def dbt_run_seed(self): - self.run_sql_file('seed.sql') - - def test_snapshot_with_query_tag(self): - self.run_dbt(["snapshot", "--vars", '{{"query_tag": {}}}'.format(self.prefix)]) - - def assert_query_tag_expected(self): - self.run_dbt(['test', '--select', 'test_type:singular', '--vars', '{{"query_tag": {}}}'.format(self.prefix)]) - - @use_profile('snowflake') - def test__snowflake__snapshot_with_query_tag(self): - self.dbt_run_seed() - self.test_snapshot_with_query_tag() - self.assert_query_tag_expected()