Merge branch 'main' into prep-release/1.9.0b1_11132192713

dbt-labs · Oct 16, 2024 · 85eda9a · 85eda9a
2 parents 6ad2301 + 805a08e
commit 85eda9a
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 3 deletions.
diff --git a/.changes/unreleased/Features-20240927-133927.yaml b/.changes/unreleased/Features-20240927-133927.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Enable setting current value of dbt_valid_to
+time: 2024-09-27T13:39:27.268886-04:00
+custom:
+  Author: gshank
+  Issue: "1112"
diff --git a/dbt/include/spark/macros/materializations/snapshot.sql b/dbt/include/spark/macros/materializations/snapshot.sql
@@ -24,7 +24,12 @@
     {% endif %}
     on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}
     when matched
-     and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+     {% if config.get("dbt_valid_to_current") %}
+       and ( DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or
+             DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null )
+     {% else %}
+       and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+     {% endif %}
      and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')
         then update
         set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}

diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py
@@ -67,7 +67,7 @@ def model(dbt, spark):
                 "ResourceClass": "SingleNode"
             }
         },
-        packages=['spacy', 'torch', 'pydantic>=1.10.8']
+        packages=['spacy', 'torch', 'pydantic>=1.10.8', 'numpy<2']
     )
     data = [[1,2]] * 10
     return spark.createDataFrame(data, schema=['test', 'test2'])
@@ -85,7 +85,11 @@ def model(dbt, spark):
 
 
 @pytest.mark.skip_profile(
-    "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc"
+    "apache_spark",
+    "spark_session",
+    "databricks_sql_endpoint",
+    "spark_http_odbc",
+    "databricks_http_cluster",
 )
 class TestChangingSchemaSpark:
     """