From 4a882963f1504133f9d64760cc44a7bcf7630ac8 Mon Sep 17 00:00:00 2001 From: Luis Date: Sat, 3 Jun 2023 17:28:11 -0700 Subject: [PATCH 1/5] Added incremental predicates --- macros/tables/snowflake/sat.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/macros/tables/snowflake/sat.sql b/macros/tables/snowflake/sat.sql index 75c5b42e5..1a7add715 100644 --- a/macros/tables/snowflake/sat.sql +++ b/macros/tables/snowflake/sat.sql @@ -29,6 +29,7 @@ {%- set window_cols = automate_dv.expand_column_list(columns=[src_pk, src_hashdiff, src_ldts]) -%} {%- set pk_cols = automate_dv.expand_column_list(columns=[src_pk]) -%} {%- set enable_ghost_record = var('enable_ghost_records', false) -%} +{%- set predicates = model.config.get('predicates', none) -%} {%- if model.config.materialized == 'vault_insert_by_rank' %} {%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%} @@ -65,6 +66,12 @@ latest_records AS ( FROM source_data ) AS source_records ON {{ automate_dv.multikey(src_pk, prefix=['current_records','source_records'], condition='=') }} + WHERE 1=1 + {%- if predicates is not none %} + {% for predicate in predicates %} + and {{ predicate }} + {% endfor %} + {% endif %} ) AS a WHERE a.rank = 1 ), From 5dc8a92d75aca0b5d4af8eae7558006b8fad12e6 Mon Sep 17 00:00:00 2001 From: Luis Date: Thu, 8 Jun 2023 14:42:19 -0700 Subject: [PATCH 2/5] Update hub.sql --- macros/tables/snowflake/hub.sql | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/macros/tables/snowflake/hub.sql b/macros/tables/snowflake/hub.sql index d10bde57a..237cd686d 100644 --- a/macros/tables/snowflake/hub.sql +++ b/macros/tables/snowflake/hub.sql @@ -28,6 +28,8 @@ {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%} +{%- set predicates = model.config.get('predicates', none) -%} + {%- if model.config.materialized == 'vault_insert_by_rank' %} {%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%} {%- endif %} @@ -106,8 +108,13 @@ records_to_insert AS ( {%- if automate_dv.is_any_incremental() %} LEFT JOIN {{ this }} AS d ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} - WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} + {%- if predicates is not none %} + {% for predicate in predicates %} + and d.{{ predicate }} + {% endfor %} + {% endif %} {%- endif %} + WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} ) SELECT * FROM records_to_insert From 48940e99064d75cb9b2a96d45a8b717a2f68c213 Mon Sep 17 00:00:00 2001 From: Luis Date: Sun, 11 Jun 2023 09:56:35 -0700 Subject: [PATCH 3/5] Update hub.sql --- macros/tables/snowflake/hub.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/tables/snowflake/hub.sql b/macros/tables/snowflake/hub.sql index 237cd686d..7dff510ee 100644 --- a/macros/tables/snowflake/hub.sql +++ b/macros/tables/snowflake/hub.sql @@ -110,7 +110,7 @@ records_to_insert AS ( ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} {%- if predicates is not none %} {% for predicate in predicates %} - and d.{{ predicate }} + AND d.{{ predicate }} {% endfor %} {% endif %} {%- endif %} From b88fcdd20c9230876b5f3c284fce08fc1e729f18 Mon Sep 17 00:00:00 2001 From: Luis Date: Sun, 11 Jun 2023 09:56:56 -0700 Subject: [PATCH 4/5] Update sat.sql --- macros/tables/snowflake/sat.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/tables/snowflake/sat.sql b/macros/tables/snowflake/sat.sql index 1a7add715..06ddec17f 100644 --- a/macros/tables/snowflake/sat.sql +++ b/macros/tables/snowflake/sat.sql @@ -69,7 +69,7 @@ latest_records AS ( WHERE 1=1 {%- if predicates is not none %} {% for predicate in predicates %} - and {{ predicate }} + AND {{ predicate }} {% endfor %} {% endif %} ) AS a From c1040babeccbb15a1b07920b494c72edbdd93b76 Mon Sep 17 00:00:00 2001 From: Luis Date: Wed, 21 Jun 2023 23:08:24 -0700 Subject: [PATCH 5/5] Update hub.sql --- macros/tables/snowflake/hub.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/tables/snowflake/hub.sql b/macros/tables/snowflake/hub.sql index 7dff510ee..69a9b48fc 100644 --- a/macros/tables/snowflake/hub.sql +++ b/macros/tables/snowflake/hub.sql @@ -113,8 +113,8 @@ records_to_insert AS ( AND d.{{ predicate }} {% endfor %} {% endif %} - {%- endif %} WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} + {%- endif %} ) SELECT * FROM records_to_insert