Merge branch 'main' into feature/AdapterOverrideUnspecifiedValue

.changes/1.10.0.md

@@ -0,0 +1 @@
+## dbt-adapters 1.10.0 - September 12, 2024

.changes/1.10.1.md

@@ -0,0 +1 @@
+## dbt-adapters 1.10.1 - September 16, 2024

.changes/1.10.2.md

@@ -0,0 +1,5 @@
+## dbt-adapters 1.10.2 - October 01, 2024
+
+### Under the Hood
+
+- dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315))

.changes/1.5.0.md

@@ -0,0 +1,11 @@
+## dbt-adapters 1.5.0 - September 10, 2024
+
+### Features
+
+- Compare 'snapshot_get_time' and snapshot 'updated_at' data types ([#242](https://github.com/dbt-labs/dbt-adapters/issues/242))
+- Add Behavior Flag framework ([#281](https://github.com/dbt-labs/dbt-adapters/issues/281))
+- Add EventTimeFilter to BaseRelation, which renders a filtered relation when start or end is set ([#294](https://github.com/dbt-labs/dbt-adapters/issues/294))
+
+### Dependencies
+
+- Update dbt-common pin to >=1.8 ([#299](https://github.com/dbt-labs/dbt-adapters/pull/299))

.changes/1.6.0.md

@@ -0,0 +1,5 @@
+## dbt-adapters 1.6.0 - September 12, 2024
+
+### Features
+
+- Default microbatch strategy implementation and base tests ([#302](https://github.com/dbt-labs/dbt-adapters/issues/302))

.changes/1.6.1.md

@@ -0,0 +1 @@
+## dbt-adapters 1.6.1 - September 16, 2024

.changes/1.7.0.md

@@ -0,0 +1,5 @@
+## dbt-adapters 1.7.0 - September 19, 2024
+
+### Features
+
+- Allow configuring of snapshot column names ([#289](https://github.com/dbt-labs/dbt-adapters/issues/289))

.changes/unreleased/Features-20240927-134248.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Enable setting current value of dbt_valid_to
+time: 2024-09-27T13:42:48.654556-04:00
+custom:
+  Author: gshank
+  Issue: "320"

.github/workflows/precommit-autoupdate.yml

@@ -0,0 +1,22 @@
+name: "Run pre-commit autoupdate"
+
+on:
+  schedule:
+    - cron: "30 1 * * SAT"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  precommit-autoupdate:
+    name: "Run pre-commit autoupdate"
+    uses: dbt-labs/actions/.github/workflows/pre-commit-autoupdate.yml
+    secrets:
+      TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }}
+      SLACK_WEBHOOK_PR_URL: ${{ secrets.SLACK_DEV_ADAPTER_PULL_REQUESTS }}
+      SLACK_WEBHOOK_ALERTS_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }}

.pre-commit-config.yaml

@@ -38,7 +38,7 @@ repos:
         -   --max-line-length=99
         -   --select=E,F,W
         -   --ignore=E203,E501,E704,E741,W503,W504
-        -   --per-file-ignores=*/__init__.py:F401
+        -   --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.9.0

CHANGELOG.md

@@ -5,6 +5,44 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
+## dbt-adapters 1.10.2 - October 01, 2024
+
+### Under the Hood
+
+- dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315))
+
+
+
+## dbt-adapters 1.10.1 - September 16, 2024
+
+## dbt-adapters 1.10.0 - September 12, 2024
+
+## dbt-adapters 1.7.0 - September 19, 2024
+
+### Features
+
+- Allow configuring of snapshot column names ([#289](https://github.com/dbt-labs/dbt-adapters/issues/289))
+
+## dbt-adapters 1.6.1 - September 16, 2024
+
+## dbt-adapters 1.6.0 - September 12, 2024
+
+### Features
+
+- Default microbatch strategy implementation and base tests ([#302](https://github.com/dbt-labs/dbt-adapters/issues/302))
+
+## dbt-adapters 1.5.0 - September 10, 2024
+
+### Features
+
+- Compare 'snapshot_get_time' and snapshot 'updated_at' data types ([#242](https://github.com/dbt-labs/dbt-adapters/issues/242))
+- Add Behavior Flag framework ([#281](https://github.com/dbt-labs/dbt-adapters/issues/281))
+- Add EventTimeFilter to BaseRelation, which renders a filtered relation when start or end is set ([#294](https://github.com/dbt-labs/dbt-adapters/issues/294))
+
+### Dependencies
+
+- Update dbt-common pin to >=1.8 ([#299](https://github.com/dbt-labs/dbt-adapters/pull/299))
+
 ## dbt-adapters 1.4.1 - August 09, 2024
 
 ### Fixes
@@ -19,7 +57,6 @@ and is generated by [Changie](https://github.com/miniscruff/changie).
 - [@jeancochrane](https://github.com/jeancochrane) ([#5273](https://github.com/dbt-labs/dbt-adapters/issues/5273))
 - [@leahwicz](https://github.com/leahwicz) ([#219](https://github.com/dbt-labs/dbt-adapters/issues/219))
 
-
 ## dbt-adapters 1.4.0 - July 30, 2024
 
 ### Features

dbt-tests-adapter/dbt/tests/__about__.py

@@ -1 +1 @@
-version = "1.9.2"
+version = "1.10.2"

dbt-tests-adapter/dbt/tests/adapter/hooks/test_run_hooks.py

@@ -3,6 +3,7 @@
 
 import pytest
 
+from dbt_common.exceptions import DbtDatabaseError
 from dbt.tests.adapter.hooks import fixtures
 from dbt.tests.util import check_table_does_not_exist, run_dbt
 
@@ -11,8 +12,8 @@ class BasePrePostRunHooks:
     @pytest.fixture(scope="function")
     def setUp(self, project):
         project.run_sql_file(project.test_data_dir / Path("seed_run.sql"))
-        project.run_sql(f"drop table if exists { project.test_schema }.schemas")
-        project.run_sql(f"drop table if exists { project.test_schema }.db_schemas")
+        project.run_sql(f"drop table if exists {project.test_schema}.schemas")
+        project.run_sql(f"drop table if exists {project.test_schema}.db_schemas")
         os.environ["TERM_TEST"] = "TESTING"
 
     @pytest.fixture(scope="class")
@@ -158,7 +159,8 @@ def project_config_update(self):
         }
 
     def test_missing_column_pre_hook(self, project):
-        run_dbt(["run"], expect_pass=False)
+        with pytest.raises(DbtDatabaseError):
+            run_dbt(["run"], expect_pass=False)
 
 
 class TestAfterRunHooks(BaseAfterRunHooks):

dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_microbatch.py

@@ -0,0 +1,96 @@
+import os
+from pprint import pformat
+from unittest import mock
+
+import pytest
+
+from dbt.tests.util import relation_from_name, run_dbt
+
+try:
+    # patch_microbatch_end_time introduced in dbt 1.9.0
+    from dbt.tests.util import patch_microbatch_end_time
+except ImportError:
+    from freezegun import freeze_time as patch_microbatch_end_time
+
+_input_model_sql = """
+{{ config(materialized='table', event_time='event_time') }}
+select 1 as id, TIMESTAMP '2020-01-01 00:00:00-0' as event_time
+union all
+select 2 as id, TIMESTAMP '2020-01-02 00:00:00-0' as event_time
+union all
+select 3 as id, TIMESTAMP '2020-01-03 00:00:00-0' as event_time
+"""
+
+_microbatch_model_sql = """
+{{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }}
+select * from {{ ref('input_model') }}
+"""
+
+
+class BaseMicrobatch:
+    @pytest.fixture(scope="class")
+    def microbatch_model_sql(self) -> str:
+        """
+        This is the SQL that defines the microbatch model, including any {{ config(..) }}
+        """
+        return _microbatch_model_sql
+
+    @pytest.fixture(scope="class")
+    def input_model_sql(self) -> str:
+        """
+        This is the SQL that defines the input model to the microbatch model, including any {{ config(..) }}.
+        event_time is a required configuration of this input
+        """
+        return _input_model_sql
+
+    @pytest.fixture(scope="class")
+    def insert_two_rows_sql(self, project) -> str:
+        test_schema_relation = project.adapter.Relation.create(
+            database=project.database, schema=project.test_schema
+        )
+        return f"insert into {test_schema_relation}.input_model (id, event_time) values (4, TIMESTAMP '2020-01-04 00:00:00-0'), (5, TIMESTAMP '2020-01-05 00:00:00-0')"
+
+    @pytest.fixture(scope="class")
+    def models(self, microbatch_model_sql, input_model_sql):
+        return {
+            "input_model.sql": input_model_sql,
+            "microbatch_model.sql": microbatch_model_sql,
+        }
+
+    def assert_row_count(self, project, relation_name: str, expected_row_count: int):
+        relation = relation_from_name(project.adapter, relation_name)
+        result = project.run_sql(f"select * from {relation}", fetch="all")
+
+        assert len(result) == expected_row_count, f"{relation_name}:{pformat(result)}"
+
+    @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"})
+    def test_run_with_event_time(self, project, insert_two_rows_sql):
+        # initial run -- backfills all data
+        with patch_microbatch_end_time("2020-01-03 13:57:00"):
+            run_dbt(["run"])
+        self.assert_row_count(project, "microbatch_model", 3)
+
+        # our partition grain is "day" so running the same day without new data should produce the same results
+        with patch_microbatch_end_time("2020-01-03 14:57:00"):
+            run_dbt(["run"])
+        self.assert_row_count(project, "microbatch_model", 3)
+
+        # add next two days of data
+        project.run_sql(insert_two_rows_sql)
+
+        self.assert_row_count(project, "input_model", 5)
+
+        # re-run without changing current time => no insert
+        with patch_microbatch_end_time("2020-01-03 14:57:00"):
+            run_dbt(["run", "--select", "microbatch_model"])
+        self.assert_row_count(project, "microbatch_model", 3)
+
+        # re-run by advancing time by one day changing current time => insert 1 row
+        with patch_microbatch_end_time("2020-01-04 14:57:00"):
+            run_dbt(["run", "--select", "microbatch_model"])
+        self.assert_row_count(project, "microbatch_model", 4)
+
+        # re-run by advancing time by one more day changing current time => insert 1 more row
+        with patch_microbatch_end_time("2020-01-05 14:57:00"):
+            run_dbt(["run", "--select", "microbatch_model"])
+        self.assert_row_count(project, "microbatch_model", 5)

dbt-tests-adapter/pyproject.toml

@@ -31,6 +31,7 @@ dependencies = [
     # `dbt-core` takes the packages below as dependencies, so they are unpinned to avoid conflicts
     "dbt-adapters",
     "pyyaml",
+    "freezegun",
 ]
 [project.urls]
 Homepage = "https://github.com/dbt-labs/dbt-adapters"

dbt/adapters/__about__.py

@@ -1 +1 @@
-version = "1.4.1"
+version = "1.7.0"

dbt/adapters/base/impl.py

@@ -22,8 +22,9 @@
     Union,
     TYPE_CHECKING,
 )
-
+import os
 import pytz
+from dbt_common.behavior_flags import Behavior, BehaviorFlag
 from dbt_common.clients.jinja import CallableMacroGenerator
 from dbt_common.contracts.constraints import (
     ColumnLevelConstraint,
@@ -54,7 +55,7 @@
     BaseConnectionManager,
     Connection,
 )
-from dbt.adapters.base.meta import AdapterMeta, available
+from dbt.adapters.base.meta import AdapterMeta, available, available_property
 from dbt.adapters.base.relation import (
     BaseRelation,
     ComponentName,
@@ -82,7 +83,6 @@
     QuoteConfigTypeError,
     RelationReturnedMultipleResultsError,
     RenameToNoneAttemptedError,
-    SnapshotTargetIncompleteError,
     SnapshotTargetNotSnapshotTableError,
     UnexpectedNonTimestampError,
 )
@@ -261,7 +261,7 @@ class BaseAdapter(metaclass=AdapterMeta):
 
     MAX_SCHEMA_METADATA_RELATIONS = 100
 
-    # This static member variable can be overriden in concrete adapter
+    # This static member variable can be overridden in concrete adapter
     # implementations to indicate adapter support for optional capabilities.
     _capabilities = CapabilityDict({})
 
@@ -271,6 +271,8 @@ def __init__(self, config, mp_context: SpawnContext) -> None:
         self.connections = self.ConnectionManager(config, mp_context)
         self._macro_resolver: Optional[MacroResolverProtocol] = None
         self._macro_context_generator: Optional[MacroContextGeneratorCallable] = None
+        # this will be updated to include global behavior flags once they exist
+        self.behavior = []  # type: ignore
 
     ###
     # Methods to set / access a macro resolver
@@ -291,6 +293,27 @@ def set_macro_context_generator(
     ) -> None:
         self._macro_context_generator = macro_context_generator
 
+    @available_property
+    def behavior(self) -> Behavior:
+        return self._behavior
+
+    @behavior.setter  # type: ignore
+    def behavior(self, flags: List[BehaviorFlag]) -> None:
+        flags.extend(self._behavior_flags)
+        try:
+            # we don't always get project flags, for example during `dbt debug`
+            self._behavior = Behavior(flags, self.config.flags)
+        except AttributeError:
+            # in that case, don't load any behavior to avoid unexpected defaults
+            self._behavior = Behavior([], {})
+
+    @property
+    def _behavior_flags(self) -> List[BehaviorFlag]:
+        """
+        This method should be overwritten by adapter maintainers to provide platform-specific flags
+        """
+        return []
+
     ###
     # Methods that pass through to the connection manager
     ###
@@ -740,7 +763,9 @@ def get_missing_columns(
         return [col for (col_name, col) in from_columns.items() if col_name in missing_columns]
 
     @available.parse_none
-    def valid_snapshot_target(self, relation: BaseRelation) -> None:
+    def valid_snapshot_target(
+        self, relation: BaseRelation, column_names: Optional[Dict[str, str]] = None
+    ) -> None:
         """Ensure that the target relation is valid, by making sure it has the
         expected columns.
 
@@ -758,21 +783,16 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None:
 
         columns = self.get_columns_in_relation(relation)
         names = set(c.name.lower() for c in columns)
-        expanded_keys = ("scd_id", "valid_from", "valid_to")
-        extra = []
         missing = []
-        for legacy in expanded_keys:
-            desired = "dbt_" + legacy
+        # Note: we're not checking dbt_updated_at here because it's not
+        # always present.
+        for column in ("dbt_scd_id", "dbt_valid_from", "dbt_valid_to"):
+            desired = column_names[column] if column_names else column
             if desired not in names:
                 missing.append(desired)
-                if legacy in names:
-                    extra.append(legacy)
 
         if missing:
-            if extra:
-                raise SnapshotTargetIncompleteError(extra, missing)
-            else:
-                raise SnapshotTargetNotSnapshotTableError(missing)
+            raise SnapshotTargetNotSnapshotTableError(missing)
 
     @available.parse_none
     def expand_target_column_types(
@@ -1549,7 +1569,11 @@ def valid_incremental_strategies(self):
         return ["append"]
 
     def builtin_incremental_strategies(self):
-        return ["append", "delete+insert", "merge", "insert_overwrite"]
+        builtin_strategies = ["append", "delete+insert", "merge", "insert_overwrite"]
+        if os.environ.get("DBT_EXPERIMENTAL_MICROBATCH"):
+            builtin_strategies.append("microbatch")
+
+        return builtin_strategies
 
     @available.parse_none
     def get_incremental_strategy_macro(self, model_context, strategy: str):