add folders & lists snapshots

luutuankiet · Nov 24, 2024 · 4062e13 · 4062e13
1 parent fb7f12b
commit 4062e13
Show file tree

Hide file tree

Showing 4 changed files with 94 additions and 14 deletions.
diff --git a/dbt_project/models/raw_source/src__folders_raw.sql b/dbt_project/models/raw_source/src__folders_raw.sql
@@ -1,20 +1,38 @@
 {{ config(
     materialized = 'table'
 ) }}
-WITH source AS (
+WITH source_active AS (
+    {# direct pull from tick api. contains current data. #}
+
     SELECT
-        {{ setup_nulls(
-            source(
-                'raw_data',
-                'folders_raw'
-            )
-        ) }}
+        *
     FROM
         {{ source(
             'raw_data',
             'folders_raw'
         ) }}
 ),
+source_snp AS (
+    {# pulls the deleted portion of the data that is gone from tick api. #}
+    SELECT
+        *
+    FROM
+        {{ ref(
+            'snp_folders_raw',
+        ) }}
+    WHERE dbt_valid_to is not null
+),
+
+source as (
+    select 
+        {{ setup_nulls(source('raw_data', 'folders_raw')) }}
+        from source_active
+    UNION ALL
+    select 
+        {{ setup_nulls(source('raw_data', 'folders_raw')) }}
+        from source_snp
+),
+
 renamed AS (
     SELECT
         {{ adapter.quote("id") }} :: text AS "folder_id",

diff --git a/dbt_project/models/raw_source/src__lists_raw.sql b/dbt_project/models/raw_source/src__lists_raw.sql
@@ -2,21 +2,48 @@
     materialized = 'table'
 ) }}
 
-WITH source AS (
+WITH source_active AS (
+    {# direct pull from tick api. contains current data. #}
 
     SELECT
-        {{ setup_nulls(
-            source(
-                'raw_data',
-                'lists_raw'
-            )
-        ) }}
+        {{dbt_utils.star(
+            from=source('raw_data','lists_raw'),
+            except=['modifiedtime']
+        )}},
+        -- gotta explicitly handle this cauaes snap's data casted as timestamp
+        modifiedtime :: text as modifiedtime
     FROM
         {{ source(
             'raw_data',
             'lists_raw'
         ) }}
 ),
+source_snp AS (
+    {# pulls the deleted portion of the data that is gone from tick api. #}
+    SELECT
+        {{dbt_utils.star(
+            from=source('raw_data','lists_raw'),
+            except=['modifiedtime']
+        )}},
+        -- gotta explicitly handle this cauaes snap's data casted as timestamp
+        modifiedtime :: text as modifiedtime
+    FROM
+        {{ ref(
+            'snp_lists_raw',
+        ) }}
+    WHERE dbt_valid_to is not null
+),
+
+source as (
+    select 
+        {{ setup_nulls(source('raw_data', 'lists_raw')) }}
+        from source_active
+    UNION ALL
+    select 
+        {{ setup_nulls(source('raw_data', 'lists_raw')) }}
+        from source_snp
+),
+
 renamed AS (
     SELECT
         {{ adapter.quote("id") }} :: text AS "list_id",

diff --git a/dbt_project/snapshots/snp_folders_raw.sql b/dbt_project/snapshots/snp_folders_raw.sql
@@ -0,0 +1,17 @@
+{% snapshot snp_folders_raw %}
+  {{ config(
+    target_schema = 'snapshots',
+    strategy = 'check',
+    unique_key = 'id',
+    check_cols = 'all',
+    invalidate_hard_deletes = true
+  ) }}
+
+  SELECT
+    *
+  FROM
+    {{ source(
+      'raw_data',
+      'folders_raw'
+    ) }}
+{% endsnapshot %}
diff --git a/dbt_project/snapshots/snp_lists_raw.sql b/dbt_project/snapshots/snp_lists_raw.sql
@@ -0,0 +1,18 @@
+{% snapshot snp_lists_raw %}
+  {{ config(
+    target_schema = 'snapshots',
+    strategy = 'timestamp',
+    unique_key = 'id',
+    updated_at = 'modifiedtime',
+    invalidate_hard_deletes = true
+  ) }}
+
+  SELECT
+    {{ dbt_utils.star(source('raw_data', 'lists_raw'), except = ['modifiedtime']) }},
+    modifiedtime :: TIMESTAMP AS "modifiedtime"
+  FROM
+    {{ source(
+      'raw_data',
+      'lists_raw'
+    ) }}
+{% endsnapshot %}