Skip to content

Commit

Permalink
Add functional tests for custom incremental strategies names 'microba…
Browse files Browse the repository at this point in the history
…tch'
  • Loading branch information
MichelleArk committed Sep 16, 2024
1 parent 6c111f2 commit 25ca7cd
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 1 deletion.
2 changes: 1 addition & 1 deletion dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
git+https://github.com/dbt-labs/dbt-adapters.git@main
git+https://github.com/dbt-labs/dbt-adapters.git@gate-builtin-microbatch
git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter
git+https://github.com/dbt-labs/dbt-common.git@main
git+https://github.com/dbt-labs/dbt-postgres.git@main
Expand Down
76 changes: 76 additions & 0 deletions tests/functional/microbatch/test_microbatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
patch_microbatch_end_time,
relation_from_name,
run_dbt,
run_dbt_and_capture,
write_file,
)

Expand Down Expand Up @@ -69,6 +70,81 @@
select * from {{ source('seed_sources', 'raw_source') }}
"""

custom_microbatch_strategy = """
{% macro get_incremental_microbatch_sql(arg_dict) %}
{% do log('custom microbatch strategy', info=True) %}
{%- set dest_cols_csv = get_quoted_csv(arg_dict["dest_columns"] | map(attribute="name")) -%}
insert into {{ arg_dict["target_relation"] }} ({{ dest_cols_csv }})
(
select {{ dest_cols_csv }}
from {{ arg_dict["temp_relation"] }}
)
{% endmacro %}
"""


class BaseMicrobatchCustomUserStrategy:
@pytest.fixture(scope="class")
def models(self):
return {
"input_model.sql": input_model_sql,
"microbatch_model.sql": microbatch_model_sql,
}

@pytest.fixture(scope="class")
def macros(self):
return {"microbatch.sql": custom_microbatch_strategy}


class TestMicrobatchCustomUserStrategyDefault(BaseMicrobatchCustomUserStrategy):
def test_use_custom_microbatch_strategy_by_default(self, project):
with mock.patch.object(
type(project.adapter), "valid_incremental_strategies", lambda _: []
):
# Initial run
run_dbt(["run"])

# Incremental run uses custom strategy
_, logs = run_dbt_and_capture(["run"])
assert "custom microbatch strategy" in logs


class TestMicrobatchCustomUserStrategyEnvVarTrueValid(BaseMicrobatchCustomUserStrategy):
@mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"})
def test_use_custom_microbatch_strategy_env_var_true_invalid_incremental_strategy(
self, project
):
with mock.patch.object(
type(project.adapter), "valid_incremental_strategies", lambda _: ["microbatch"]
):
# Initial run
run_dbt(["run"])

# Incremental run uses custom strategy
_, logs = run_dbt_and_capture(["run"])
assert "custom microbatch strategy" in logs


# TODO: Consider a behaviour flag here if DBT_EXPERIMENTAL_MICROBATCH is removed
# Since this causes an exception prior to using an override
class TestMicrobatchCustomUserStrategyEnvVarTrueInvalid(BaseMicrobatchCustomUserStrategy):
@mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"})
def test_use_custom_microbatch_strategy_env_var_true_invalid_incremental_strategy(
self, project
):
with mock.patch.object(
type(project.adapter), "valid_incremental_strategies", lambda _: []
):
# Initial run
run_dbt(["run"])

# Incremental run fails
_, logs = run_dbt_and_capture(["run"], expect_pass=False)
assert "'microbatch' is not valid" in logs


class TestMicrobatchCLI:
@pytest.fixture(scope="class")
Expand Down

0 comments on commit 25ca7cd

Please sign in to comment.