From 68bfa3492c837839303311eb6b77b5f82494cca8 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Mon, 1 Jul 2024 17:11:45 -0500 Subject: [PATCH 01/19] initial add of test and update to columns.sql --- .../dbt/tests/adapter/incremental/fixtures.py | 31 +++++++++++++++++++ .../test_incremental_on_schema_change.py | 17 +++++++++- .../macros/adapters/columns.sql | 4 +-- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py index 6e130266..63ac0aac 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py @@ -303,3 +303,34 @@ from source_data """ + +_MODELS__SRC_ARTISTS = """ +{{ + config( + materialized='table', + ) +}} + +{% if var("version", 0) == 0 %} + + select {{ dbt.current_timestamp() }} as inserted_at, 'taylor' as name + +{% else %} + + -- add a non-zero version to the end of the command to get a different version: + -- --vars "{'version': 1}" + select {{ dbt.current_timestamp() }} as inserted_at, 'taylor' as Name, 'eras' as "Tour" + +{% endif %} +""" + +_MODELS__DIM_ARTISTS = """ +{{ + config( + materialized='incremental', + on_schema_change='append_new_columns', + ) +}} + +select * from {{ ref("src_artists") }} +""" diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index 43b13ae9..3167eef7 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -1,7 +1,7 @@ import pytest from dbt.tests.adapter.incremental import fixtures -from dbt.tests.util import check_relations_equal, run_dbt +from dbt.tests.util import check_relations_equal, run_dbt, run_dbt_and_capture class BaseIncrementalOnSchemaChangeSetup: @@ -20,6 +20,8 @@ def models(self): "incremental_append_new_columns.sql": fixtures._MODELS__INCREMENTAL_APPEND_NEW_COLUMNS, "incremental_sync_all_columns_target.sql": fixtures._MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, "incremental_append_new_columns_remove_one_target.sql": fixtures._MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, + "src_artists.sql": fixtures._MODELS__SRC_ARTISTS, + "dim_artists.sql": fixtures._MODELS__DIM_ARTISTS, } def run_twice_and_assert(self, include, compare_source, compare_target, project): @@ -71,6 +73,19 @@ def test_run_incremental_append_new_columns(self, project): self.run_incremental_append_new_columns(project) self.run_incremental_append_new_columns_remove_one(project) + def test_run_incremental_check_quoting_on_new_columns(self, project): + select = "src_artists dim_artists" + + run_dbt(["run", "--models", select, "--full-refresh"]) + run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) + res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) + assert "NAME" in logs + run_dbt(["run", "--vars", "{'version': 1}"]) + run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) + res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) + assert "Tour" in logs + + def test_run_incremental_sync_all_columns(self, project): self.run_incremental_sync_all_columns(project) self.run_incremental_sync_remove_only(project) diff --git a/dbt/include/global_project/macros/adapters/columns.sql b/dbt/include/global_project/macros/adapters/columns.sql index 663a827b..cef409dd 100644 --- a/dbt/include/global_project/macros/adapters/columns.sql +++ b/dbt/include/global_project/macros/adapters/columns.sql @@ -123,11 +123,11 @@ alter {{ relation.type }} {{ relation }} {% for column in add_columns %} - add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} + add column {{ adapter.quote(column.name) }} {{ column.data_type }}{{ ',' if not loop.last }} {% endfor %}{{ ',' if add_columns and remove_columns }} {% for column in remove_columns %} - drop column {{ column.name }}{{ ',' if not loop.last }} + drop column {{ adapter.quote(column.name) }} {{ ',' if not loop.last }} {% endfor %} {%- endset -%} From d6ad15c7852aa9daf0d7ffb1783aa5aa0d1f6164 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 3 Jul 2024 16:19:48 -0500 Subject: [PATCH 02/19] update test to try and check against postgres --- dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py | 4 ++-- .../adapter/incremental/test_incremental_on_schema_change.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py index 63ac0aac..9db54ac4 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py @@ -313,13 +313,13 @@ {% if var("version", 0) == 0 %} - select {{ dbt.current_timestamp() }} as inserted_at, 'taylor' as name + select {{ dbt.current_timestamp() }} as inserted_at, 'taylor' as f_name {% else %} -- add a non-zero version to the end of the command to get a different version: -- --vars "{'version': 1}" - select {{ dbt.current_timestamp() }} as inserted_at, 'taylor' as Name, 'eras' as "Tour" + select {{ dbt.current_timestamp() }} as inserted_at, 'taylor' as F_name, 'eras' as "Tour" {% endif %} """ diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index 3167eef7..d4aeff7f 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -79,7 +79,7 @@ def test_run_incremental_check_quoting_on_new_columns(self, project): run_dbt(["run", "--models", select, "--full-refresh"]) run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) - assert "NAME" in logs + assert "f_name" in logs run_dbt(["run", "--vars", "{'version': 1}"]) run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) From 3c922d4e1c8c881893c2a9238c766b9d4028ed4b Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Mon, 8 Jul 2024 15:21:28 -0500 Subject: [PATCH 03/19] minor update to tests and push up --- dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py | 4 ++-- .../adapter/incremental/test_incremental_on_schema_change.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py index 9db54ac4..008a1c2c 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py @@ -313,13 +313,13 @@ {% if var("version", 0) == 0 %} - select {{ dbt.current_timestamp() }} as inserted_at, 'taylor' as f_name + select {{ dbt.current_timestamp() }} as inserted_at, 'john' as name {% else %} -- add a non-zero version to the end of the command to get a different version: -- --vars "{'version': 1}" - select {{ dbt.current_timestamp() }} as inserted_at, 'taylor' as F_name, 'eras' as "Tour" + select {{ dbt.current_timestamp() }} as inserted_at, 'john' as Name, 'job' as "Engineer" {% endif %} """ diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index d4aeff7f..b37e95c8 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -79,11 +79,12 @@ def test_run_incremental_check_quoting_on_new_columns(self, project): run_dbt(["run", "--models", select, "--full-refresh"]) run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) - assert "f_name" in logs + breakpoint() + assert "name" in logs run_dbt(["run", "--vars", "{'version': 1}"]) run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) - assert "Tour" in logs + assert "Name" in logs def test_run_incremental_sync_all_columns(self, project): From b221b092d2360f4501dcc4d82ec9f24e8023830d Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 9 Jul 2024 11:09:12 -0500 Subject: [PATCH 04/19] revert macro update for testing --- dbt/include/global_project/macros/adapters/columns.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/global_project/macros/adapters/columns.sql b/dbt/include/global_project/macros/adapters/columns.sql index 782b9e94..cc9fe0a0 100644 --- a/dbt/include/global_project/macros/adapters/columns.sql +++ b/dbt/include/global_project/macros/adapters/columns.sql @@ -123,11 +123,11 @@ alter {{ relation.type }} {{ relation.render() }} {% for column in add_columns %} - add column {{ adapter.quote(column.name) }} {{ column.data_type }}{{ ',' if not loop.last }} + add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} {% endfor %}{{ ',' if add_columns and remove_columns }} {% for column in remove_columns %} - drop column {{ adapter.quote(column.name) }} {{ ',' if not loop.last }} + drop column {{ column.name }} {{ ',' if not loop.last }} {% endfor %} {%- endset -%} From 2765bf014a756bae25781572b1670a879090a453 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 9 Jul 2024 14:01:55 -0500 Subject: [PATCH 05/19] readd adapter.quote --- dbt/include/global_project/macros/adapters/columns.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/global_project/macros/adapters/columns.sql b/dbt/include/global_project/macros/adapters/columns.sql index cc9fe0a0..782b9e94 100644 --- a/dbt/include/global_project/macros/adapters/columns.sql +++ b/dbt/include/global_project/macros/adapters/columns.sql @@ -123,11 +123,11 @@ alter {{ relation.type }} {{ relation.render() }} {% for column in add_columns %} - add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} + add column {{ adapter.quote(column.name) }} {{ column.data_type }}{{ ',' if not loop.last }} {% endfor %}{{ ',' if add_columns and remove_columns }} {% for column in remove_columns %} - drop column {{ column.name }} {{ ',' if not loop.last }} + drop column {{ adapter.quote(column.name) }} {{ ',' if not loop.last }} {% endfor %} {%- endset -%} From de1893aa75970e0d3f3fbce0bdedc6b6c8aac2d6 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 9 Jul 2024 14:32:37 -0500 Subject: [PATCH 06/19] update fixtures and tests (remove breakpoints) test against snowflake (course of original issue) --- dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py | 2 +- .../adapter/incremental/test_incremental_on_schema_change.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py index 008a1c2c..6a8d9f1a 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py @@ -319,7 +319,7 @@ -- add a non-zero version to the end of the command to get a different version: -- --vars "{'version': 1}" - select {{ dbt.current_timestamp() }} as inserted_at, 'john' as Name, 'job' as "Engineer" + select {{ dbt.current_timestamp() }} as inserted_at, 'john' as Name, 'engineer' as "Job" {% endif %} """ diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index b37e95c8..d25b2e9e 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -79,12 +79,11 @@ def test_run_incremental_check_quoting_on_new_columns(self, project): run_dbt(["run", "--models", select, "--full-refresh"]) run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) - breakpoint() - assert "name" in logs + assert "Job" not in logs run_dbt(["run", "--vars", "{'version': 1}"]) run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) - assert "Name" in logs + assert "Job" in logs def test_run_incremental_sync_all_columns(self, project): From 7fdf63980fe07a2455132fbaef86f598a1f2cd5b Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 9 Jul 2024 14:54:54 -0500 Subject: [PATCH 07/19] pre-commit fixes and changelog --- .changes/unreleased/Fixes-20240709-145439.yaml | 6 ++++++ .../incremental/test_incremental_on_schema_change.py | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240709-145439.yaml diff --git a/.changes/unreleased/Fixes-20240709-145439.yaml b/.changes/unreleased/Fixes-20240709-145439.yaml new file mode 100644 index 00000000..275c6bb8 --- /dev/null +++ b/.changes/unreleased/Fixes-20240709-145439.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: add new test to test case sensitivity for column quoting in incremental models that use append_new_columns +time: 2024-07-09T14:54:39.526077-05:00 +custom: + Author: McKnight-42 + Issue: "250" diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index d25b2e9e..45fa9b57 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -75,7 +75,6 @@ def test_run_incremental_append_new_columns(self, project): def test_run_incremental_check_quoting_on_new_columns(self, project): select = "src_artists dim_artists" - run_dbt(["run", "--models", select, "--full-refresh"]) run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) @@ -85,7 +84,6 @@ def test_run_incremental_check_quoting_on_new_columns(self, project): res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) assert "Job" in logs - def test_run_incremental_sync_all_columns(self, project): self.run_incremental_sync_all_columns(project) self.run_incremental_sync_remove_only(project) From 0cb05837c8c9e9a884f98821b0577bb5736a32a3 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 11 Jul 2024 14:40:28 -0500 Subject: [PATCH 08/19] pre-commit fix --- .../incremental/test_incremental_on_schema_change.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index 45fa9b57..3907a431 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -77,11 +77,15 @@ def test_run_incremental_check_quoting_on_new_columns(self, project): select = "src_artists dim_artists" run_dbt(["run", "--models", select, "--full-refresh"]) run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) - res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) + res, logs = run_dbt_and_capture( + ["show", "--inline", "select * from {{ ref('dim_artists') }}"] + ) assert "Job" not in logs run_dbt(["run", "--vars", "{'version': 1}"]) run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) - res, logs = run_dbt_and_capture(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) + res, logs = run_dbt_and_capture( + ["show", "--inline", "select * from {{ ref('dim_artists') }}"] + ) assert "Job" in logs def test_run_incremental_sync_all_columns(self, project): From 52cb4720787b22a04c2cd6dc7e990d66967424c5 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 16 Jul 2024 11:34:04 -0500 Subject: [PATCH 09/19] pull new test out of old OnSchemaChange, to create own bass and test class --- .../test_incremental_on_schema_change.py | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index 3907a431..382a820d 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -20,8 +20,6 @@ def models(self): "incremental_append_new_columns.sql": fixtures._MODELS__INCREMENTAL_APPEND_NEW_COLUMNS, "incremental_sync_all_columns_target.sql": fixtures._MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, "incremental_append_new_columns_remove_one_target.sql": fixtures._MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, - "src_artists.sql": fixtures._MODELS__SRC_ARTISTS, - "dim_artists.sql": fixtures._MODELS__DIM_ARTISTS, } def run_twice_and_assert(self, include, compare_source, compare_target, project): @@ -73,6 +71,29 @@ def test_run_incremental_append_new_columns(self, project): self.run_incremental_append_new_columns(project) self.run_incremental_append_new_columns_remove_one(project) + def test_run_incremental_sync_all_columns(self, project): + self.run_incremental_sync_all_columns(project) + self.run_incremental_sync_remove_only(project) + + def test_run_incremental_fail_on_schema_change(self, project): + select = "model_a incremental_fail" + run_dbt(["run", "--models", select, "--full-refresh"]) + results_two = run_dbt(["run", "--models", select], expect_pass=False) + assert "Compilation Error" in results_two[1].message + + +class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange): + pass + + +class BaseIncrementalCaseSenstivityOnSchemaChange: + @pytest.fixture(scope="class") + def models(self): + return { + "src_artists.sql": fixtures._MODELS__SRC_ARTISTS, + "dim_artists.sql": fixtures._MODELS__DIM_ARTISTS, + } + def test_run_incremental_check_quoting_on_new_columns(self, project): select = "src_artists dim_artists" run_dbt(["run", "--models", select, "--full-refresh"]) @@ -88,16 +109,6 @@ def test_run_incremental_check_quoting_on_new_columns(self, project): ) assert "Job" in logs - def test_run_incremental_sync_all_columns(self, project): - self.run_incremental_sync_all_columns(project) - self.run_incremental_sync_remove_only(project) - def test_run_incremental_fail_on_schema_change(self, project): - select = "model_a incremental_fail" - run_dbt(["run", "--models", select, "--full-refresh"]) - results_two = run_dbt(["run", "--models", select], expect_pass=False) - assert "Compilation Error" in results_two[1].message - - -class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange): +class TestIncrementalCaseSenstivityOnSchemaChange(BaseIncrementalCaseSenstivityOnSchemaChange): pass From f70650bbc47897c7d3d0ff48ee6253595fee106f Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 17 Jul 2024 09:56:43 -0500 Subject: [PATCH 10/19] remove uneeded test class --- .../adapter/incremental/test_incremental_on_schema_change.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index 382a820d..7d32f1ef 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -108,7 +108,3 @@ def test_run_incremental_check_quoting_on_new_columns(self, project): ["show", "--inline", "select * from {{ ref('dim_artists') }}"] ) assert "Job" in logs - - -class TestIncrementalCaseSenstivityOnSchemaChange(BaseIncrementalCaseSenstivityOnSchemaChange): - pass From 29bd5ae88de6d25793984a63a13161c61e888306 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 31 Jul 2024 15:59:42 -0500 Subject: [PATCH 11/19] updating test and model names based on feedback, pushing to test in other adapters --- .../dbt/tests/adapter/incremental/fixtures.py | 6 +++--- .../test_incremental_on_schema_change.py | 15 ++++++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py index 6a8d9f1a..ce3fc961 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py @@ -304,7 +304,7 @@ from source_data """ -_MODELS__SRC_ARTISTS = """ +_MODELS__SRC_JOBS = """ {{ config( materialized='table', @@ -324,7 +324,7 @@ {% endif %} """ -_MODELS__DIM_ARTISTS = """ +_MODELS__DIM_JOBS = """ {{ config( materialized='incremental', @@ -332,5 +332,5 @@ ) }} -select * from {{ ref("src_artists") }} +select * from {{ ref("src_jobs") }} """ diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index 7d32f1ef..a9c76124 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -90,21 +90,18 @@ class BaseIncrementalCaseSenstivityOnSchemaChange: @pytest.fixture(scope="class") def models(self): return { - "src_artists.sql": fixtures._MODELS__SRC_ARTISTS, - "dim_artists.sql": fixtures._MODELS__DIM_ARTISTS, + "src_jobs.sql": fixtures._MODELS__SRC_JOBS, + "dim_jobs.sql": fixtures._MODELS__DIM_JOBS, } def test_run_incremental_check_quoting_on_new_columns(self, project): - select = "src_artists dim_artists" + select = "src_jobs dim_jobs" run_dbt(["run", "--models", select, "--full-refresh"]) - run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) res, logs = run_dbt_and_capture( - ["show", "--inline", "select * from {{ ref('dim_artists') }}"] + ["show", "--inline", "select * from {{ ref('dim_jobs') }}"] ) - assert "Job" not in logs + breakpoint() run_dbt(["run", "--vars", "{'version': 1}"]) - run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) res, logs = run_dbt_and_capture( - ["show", "--inline", "select * from {{ ref('dim_artists') }}"] + ["show", "--inline", "select * from {{ ref('dim_jobs') }}"], ) - assert "Job" in logs From 1d83c867a36bbebf6f7c25a6e493a55e7156028c Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 31 Jul 2024 16:00:47 -0500 Subject: [PATCH 12/19] updating test and model names based on feedback, pushing to test in other adapters --- .../adapter/incremental/test_incremental_on_schema_change.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index a9c76124..9327499f 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -100,7 +100,6 @@ def test_run_incremental_check_quoting_on_new_columns(self, project): res, logs = run_dbt_and_capture( ["show", "--inline", "select * from {{ ref('dim_jobs') }}"] ) - breakpoint() run_dbt(["run", "--vars", "{'version': 1}"]) res, logs = run_dbt_and_capture( ["show", "--inline", "select * from {{ ref('dim_jobs') }}"], From 13f3f35e215e6d62d0fa6bc6e27503335449478c Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 1 Aug 2024 15:10:13 -0500 Subject: [PATCH 13/19] update query to use adapter.quote --- dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py index ce3fc961..2016a360 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py @@ -319,7 +319,7 @@ -- add a non-zero version to the end of the command to get a different version: -- --vars "{'version': 1}" - select {{ dbt.current_timestamp() }} as inserted_at, 'john' as Name, 'engineer' as "Job" + select {{ dbt.current_timestamp() }} as inserted_at, 'john' as Name, 'engineer' as {{ adapter.quote("Job") }} {% endif %} """ From ed01f64330ef5a1db71b3d049f9ef9db01b1e0c1 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 1 Aug 2024 15:40:35 -0500 Subject: [PATCH 14/19] modify fixture to take into account databases that don't append if column already exists --- dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py index 2016a360..2a7f1a88 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py @@ -319,7 +319,7 @@ -- add a non-zero version to the end of the command to get a different version: -- --vars "{'version': 1}" - select {{ dbt.current_timestamp() }} as inserted_at, 'john' as Name, 'engineer' as {{ adapter.quote("Job") }} + select {{ dbt.current_timestamp() }} as inserted_at, 'engineer' as {{ adapter.quote("Job") }} {% endif %} """ From 35710fe40e579941180b815f483eb8f68f977f87 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Fri, 2 Aug 2024 14:24:32 -0500 Subject: [PATCH 15/19] swap double-quotes for single-quotes --- dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py index 2a7f1a88..f0e84c98 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py @@ -319,7 +319,7 @@ -- add a non-zero version to the end of the command to get a different version: -- --vars "{'version': 1}" - select {{ dbt.current_timestamp() }} as inserted_at, 'engineer' as {{ adapter.quote("Job") }} + select {{ dbt.current_timestamp() }} as inserted_at, 'engineer' as {{ adapter.quote('Job') }} {% endif %} """ From ecacae791fdc93969929eda6076874b8189e55cd Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Mon, 5 Aug 2024 16:15:42 -0500 Subject: [PATCH 16/19] update run_dbt_and_capture command --- .../incremental/test_incremental_on_schema_change.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index 9327499f..8d511059 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -97,10 +97,8 @@ def models(self): def test_run_incremental_check_quoting_on_new_columns(self, project): select = "src_jobs dim_jobs" run_dbt(["run", "--models", select, "--full-refresh"]) - res, logs = run_dbt_and_capture( - ["show", "--inline", "select * from {{ ref('dim_jobs') }}"] - ) + res, logs = run_dbt_and_capture(["run", "--inline", "select * from {{ ref('dim_jobs') }}"]) run_dbt(["run", "--vars", "{'version': 1}"]) res, logs = run_dbt_and_capture( - ["show", "--inline", "select * from {{ ref('dim_jobs') }}"], + ["run", "--inline", "select * from {{ ref('dim_jobs') }}"], ) From 27d3cab278b6e71bd44367227c91ce23ca525b00 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Mon, 5 Aug 2024 16:22:27 -0500 Subject: [PATCH 17/19] update run_dbt_and_capture command --- .../adapter/incremental/test_incremental_on_schema_change.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index 8d511059..e732df0f 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -97,8 +97,8 @@ def models(self): def test_run_incremental_check_quoting_on_new_columns(self, project): select = "src_jobs dim_jobs" run_dbt(["run", "--models", select, "--full-refresh"]) - res, logs = run_dbt_and_capture(["run", "--inline", "select * from {{ ref('dim_jobs') }}"]) + res, logs = run_dbt_and_capture(["run", "--inline", "dim_jobs"]) run_dbt(["run", "--vars", "{'version': 1}"]) res, logs = run_dbt_and_capture( - ["run", "--inline", "select * from {{ ref('dim_jobs') }}"], + ["run", "--inline", "dim_jobs"], ) From 002bba50536c108dbe5832cad9c602ab7cdda784 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Mon, 5 Aug 2024 16:24:50 -0500 Subject: [PATCH 18/19] swap command flag --- .../adapter/incremental/test_incremental_on_schema_change.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index e732df0f..17f99a67 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -97,8 +97,8 @@ def models(self): def test_run_incremental_check_quoting_on_new_columns(self, project): select = "src_jobs dim_jobs" run_dbt(["run", "--models", select, "--full-refresh"]) - res, logs = run_dbt_and_capture(["run", "--inline", "dim_jobs"]) + res, logs = run_dbt_and_capture(["run", "--select", "dim_jobs"]) run_dbt(["run", "--vars", "{'version': 1}"]) res, logs = run_dbt_and_capture( - ["run", "--inline", "dim_jobs"], + ["run", "--select", "dim_jobs"], ) From 8670ca250dfb10e3426f37786f8badaf375702e6 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 8 Aug 2024 10:24:16 -0500 Subject: [PATCH 19/19] minor reformat --- .../dbt/tests/adapter/incremental/fixtures.py | 2 -- .../incremental/test_incremental_on_schema_change.py | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py index f0e84c98..5116504d 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/fixtures.py @@ -317,8 +317,6 @@ {% else %} - -- add a non-zero version to the end of the command to get a different version: - -- --vars "{'version': 1}" select {{ dbt.current_timestamp() }} as inserted_at, 'engineer' as {{ adapter.quote('Job') }} {% endif %} diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py index 17f99a67..8e10896b 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_on_schema_change.py @@ -95,10 +95,10 @@ def models(self): } def test_run_incremental_check_quoting_on_new_columns(self, project): - select = "src_jobs dim_jobs" - run_dbt(["run", "--models", select, "--full-refresh"]) + run_dbt(["run", "--models", "src_jobs dim_jobs", "--full-refresh"]) + res, logs = run_dbt_and_capture(["run", "--select", "dim_jobs"]) + run_dbt(["run", "--vars", "{'version': 1}"]) - res, logs = run_dbt_and_capture( - ["run", "--select", "dim_jobs"], - ) + + res, logs = run_dbt_and_capture(["run", "--select", "dim_jobs"])