From 23c6147844bdc7278c4b4ff4166db942e325014b Mon Sep 17 00:00:00 2001 From: etheleon Date: Thu, 3 Aug 2023 09:28:35 +0800 Subject: [PATCH 1/7] Update seed.sql include tblproperties --- dbt/include/spark/macros/materializations/seed.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index 196479cb0..23ff339bc 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -69,6 +69,7 @@ {{ clustered_cols(label="clustered by") }} {{ location_clause() }} {{ comment_clause() }} + {{ tblproperties_clause() }} {% endset %} {% call statement('_') -%} From c4a3b8b4c6b9f6864e1ee4b23b9fc4bd52d214b3 Mon Sep 17 00:00:00 2001 From: etheleon Date: Thu, 3 Aug 2023 20:54:19 +0800 Subject: [PATCH 2/7] Update seed.sql typo in name of macro --- dbt/include/spark/macros/materializations/seed.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index 23ff339bc..b174058ef 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -69,7 +69,7 @@ {{ clustered_cols(label="clustered by") }} {{ location_clause() }} {{ comment_clause() }} - {{ tblproperties_clause() }} + {{ dbt_spark_tblproperties_clause() }} {% endset %} {% call statement('_') -%} From 3bd8916075785c36c2fb12e816d12c89565794e6 Mon Sep 17 00:00:00 2001 From: etheleon Date: Fri, 4 Aug 2023 01:18:33 +0800 Subject: [PATCH 3/7] update macro name and follow pattern from other use cases * changed macro `dbt_spark_tblproperties_clause()` to `spark__tblproperties_clause()` * create new macro `tblproperties_clause()` which references the above --- dbt/include/spark/macros/adapters.sql | 6 +++++- dbt/include/spark/macros/materializations/seed.sql | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index 202564e4e..ac1faae9e 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -1,4 +1,8 @@ -{% macro dbt_spark_tblproperties_clause() -%} +{% macro tblproperties_clause() %} + {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }} +{%- endmacro -%} + +{% macro spark__tblproperties_clause() -%} {%- set tblproperties = config.get('tblproperties') -%} {%- if tblproperties is not none %} tblproperties ( diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index b174058ef..23ff339bc 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -69,7 +69,7 @@ {{ clustered_cols(label="clustered by") }} {{ location_clause() }} {{ comment_clause() }} - {{ dbt_spark_tblproperties_clause() }} + {{ tblproperties_clause() }} {% endset %} {% call statement('_') -%} From a8ac2b374e0ddcacebb947c7e338fbbf730f4008 Mon Sep 17 00:00:00 2001 From: etheleon Date: Wed, 9 Aug 2023 12:56:07 +0800 Subject: [PATCH 4/7] move mention of macro to adapters from seed --- dbt/include/spark/macros/adapters.sql | 2 ++ dbt/include/spark/macros/materializations/seed.sql | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index ac1faae9e..471d1deef 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -160,10 +160,12 @@ {% endif %} {{ file_format_clause() }} {{ options_clause() }} + {{ tblproperties_clause() }} {{ partition_cols(label="partitioned by") }} {{ clustered_cols(label="clustered by") }} {{ location_clause() }} {{ comment_clause() }} + as {{ compiled_code }} {%- endif -%} diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index 23ff339bc..196479cb0 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -69,7 +69,6 @@ {{ clustered_cols(label="clustered by") }} {{ location_clause() }} {{ comment_clause() }} - {{ tblproperties_clause() }} {% endset %} {% call statement('_') -%} From 32c9a8271eb45a21397de6f521379a16bfd328c1 Mon Sep 17 00:00:00 2001 From: etheleon Date: Thu, 10 Aug 2023 01:41:42 +0800 Subject: [PATCH 5/7] include changelog --- .changes/unreleased/Fixes-20230810-014122.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20230810-014122.yaml diff --git a/.changes/unreleased/Fixes-20230810-014122.yaml b/.changes/unreleased/Fixes-20230810-014122.yaml new file mode 100644 index 000000000..3844a827e --- /dev/null +++ b/.changes/unreleased/Fixes-20230810-014122.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: include tblproperties macro in adaptors +time: 2023-08-10T01:41:22.782982+08:00 +custom: + Author: etheleon + Issue: "865" From 2710ab81828104df8ca7fc1acbf8402191487a9b Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 9 Aug 2023 11:44:39 -0700 Subject: [PATCH 6/7] Update Fixes-20230810-014122.yaml --- .changes/unreleased/Fixes-20230810-014122.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Fixes-20230810-014122.yaml b/.changes/unreleased/Fixes-20230810-014122.yaml index 3844a827e..190941404 100644 --- a/.changes/unreleased/Fixes-20230810-014122.yaml +++ b/.changes/unreleased/Fixes-20230810-014122.yaml @@ -1,5 +1,5 @@ kind: Fixes -body: include tblproperties macro in adaptors +body: include tblproperties macro in adaptor.sql time: 2023-08-10T01:41:22.782982+08:00 custom: Author: etheleon From 82d290e19ef489607872cd11bb584bb8b1c9da9f Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 9 Aug 2023 11:45:15 -0700 Subject: [PATCH 7/7] Update Fixes-20230810-014122.yaml --- .changes/unreleased/Fixes-20230810-014122.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Fixes-20230810-014122.yaml b/.changes/unreleased/Fixes-20230810-014122.yaml index 190941404..fcb34237e 100644 --- a/.changes/unreleased/Fixes-20230810-014122.yaml +++ b/.changes/unreleased/Fixes-20230810-014122.yaml @@ -1,5 +1,5 @@ kind: Fixes -body: include tblproperties macro in adaptor.sql +body: include tblproperties macro in adapters.sql create table time: 2023-08-10T01:41:22.782982+08:00 custom: Author: etheleon