Pipedrive dlt-dbt v1.5

.gitignore

@@ -0,0 +1,10 @@
+# ignore secrets, virtual environments and typical python compilation artifacts
+secrets.toml
+# ignore basic python artifacts
+.env
+**/__pycache__/
+**/*.py[cod]
+**/*$py.class
+# ignore duckdb
+*.duckdb
+*.wal

README.md

@@ -130,12 +130,12 @@ for data model which can be further modified as required.
 
 1. The schema of data modelled above using dlt-dbt-generator:
 
-   ![picture1](https://storage.googleapis.com/dlt-blog-images/pipedrive-dlt-dbt-package%20(1).png)
+   ![picture1](https://storage.googleapis.com/dlt-blog-images/pipedrive_dlt_dbt_dim_model.png)
 
 > Please note that this is a starting template for your data model and is not the final product. It is advised to customize the
 > data model as per your needs.
 
-Here's the link to the DB diagram: [link](https://dbdiagram.io/d/pipedrive-dlt-dbt-package-66ff4a7cfb079c7ebd49efa9).
+Here's the link to the DB diagram: [link](https://dbdiagram.io/d/pipedrive_dlt_dbt_v1-5-6749a24ae9daa85aca219163).
 
 #### Optional: Advanced Usage (Generator and Licensing)
 

dbt_project.yml

@@ -1,29 +1,29 @@
-name: 'pipedrive'
-
-config-version: 2
-version: '0.1'
-
-profile: 'pipedrive'
-
-model-paths: ["models"]
-test-paths: ["tests"]
-analysis-paths: ["analysis"]
-macro-paths: ["macros"]
-
-target-path: "target"
-clean-targets:
-    - "target"
-    - "dbt_modules"
-    - "logs"
-
-require-dbt-version: [">=1.0.0", "<2.0.0"]
-
-models:
-  pipedrive:
-    materialized: table
-    staging:
-      materialized: view
-      +docs:
-        node_color: 'silver'
-    +docs:
+name: 'pipedrive'
+
+config-version: 2
+version: '0.1'
+
+profile: 'pipedrive'
+
+model-paths: ["models"]
+test-paths: ["tests"]
+analysis-paths: ["analysis"]
+macro-paths: ["macros"]
+
+target-path: "target"
+clean-targets:
+    - "target"
+    - "dbt_modules"
+    - "logs"
+
+require-dbt-version: [">=1.0.0", "<2.0.0"]
+
+models:
+  pipedrive:
+    materialized: table
+    staging:
+      materialized: view
+      +docs:
+        node_color: 'silver'
+    +docs:
       node_color: 'gold'

models/dlt_active_load_ids.sql

@@ -1,13 +1,13 @@
-/* check if the destination already has a list of processed ids
-if not we will process all loads with status 0 (==success) */
-{% set active_ids_exist = adapter.get_relation(
-        database=this.database ,
-        schema=this.schema,
-        identifier="dlt_processed_load_ids" ) %}
-
-SELECT load_id FROM {{  source('raw_data', '_dlt_loads') }}
-WHERE status = 0
-/* exclude already processed load_ids */
-{% if active_ids_exist and not should_full_refresh() %}
-AND load_id NOT IN (SELECT load_id FROM {{ source('transformed_data', 'dlt_processed_load_ids') }})
+/* check if the destination already has a list of processed ids
+if not we will process all loads with status 0 (==success) */
+{% set active_ids_exist = adapter.get_relation(
+        database=this.database ,
+        schema=this.schema,
+        identifier="dlt_processed_load_ids" ) %}
+
+SELECT load_id FROM {{  source('raw_data', '_dlt_loads') }}
+WHERE status = 0
+/* exclude already processed load_ids */
+{% if active_ids_exist and not should_full_refresh() %}
+AND load_id NOT IN (SELECT load_id FROM {{ source('transformed_data', 'dlt_processed_load_ids') }})
 {% endif %}

models/dlt_processed_load_ids.sql

@@ -1,33 +1,29 @@
-{{
-    config(
-        materialized='incremental'
-    )
-}}
--- depends_on: {{ ref('dim__dlt_loads') }}
--- depends_on: {{ ref('fact_activities') }}
--- depends_on: {{ ref('dim_persons') }}
--- depends_on: {{ ref('dim_organizations') }}
--- depends_on: {{ ref('dim_products') }}
--- depends_on: {{ ref('dim_users') }}
--- depends_on: {{ ref('dim_leads') }}
--- depends_on: {{ ref('dim_custom_fields_mapping') }}
--- depends_on: {{ ref('dim_deals_flow_activity') }}
--- depends_on: {{ ref('fact_deals') }}
--- depends_on: {{ ref('dim_deals_flow_deal_change') }}
--- depends_on: {{ ref('dim_deals_participants') }}
--- depends_on: {{ ref('dim_activities__participants') }}
--- depends_on: {{ ref('dim_deals__label') }}
--- depends_on: {{ ref('dim_deals_flow_activity__participants') }}
--- depends_on: {{ ref('dim_deals_participants__person_id__email') }}
--- depends_on: {{ ref('dim_deals_participants__person_id__phone') }}
--- depends_on: {{ ref('dim_deals_participants__person__phone') }}
--- depends_on: {{ ref('dim_deals_participants__person__email') }}
--- depends_on: {{ ref('dim_deals_participants__person__im') }}
--- depends_on: {{ ref('dim_persons__phone') }}
--- depends_on: {{ ref('dim_persons__email') }}
--- depends_on: {{ ref('dim_persons__im') }}
--- depends_on: {{ ref('dim_persons__labels') }}
--- depends_on: {{ ref('dim_persons__multiple_options') }}
--- depends_on: {{ ref('dim_products__prices') }}
-/* we save all currently active load ids into the processed ids table */
+{{
+    config(
+        materialized='incremental'
+    )
+}}
+-- depends_on: {{ ref('dim__dlt_loads') }}
+-- depends_on: {{ ref('dim_activities') }}
+-- depends_on: {{ ref('dim_notes') }}
+-- depends_on: {{ ref('dim_persons') }}
+-- depends_on: {{ ref('dim_organizations') }}
+-- depends_on: {{ ref('dim_users') }}
+-- depends_on: {{ ref('dim_custom_fields_mapping') }}
+-- depends_on: {{ ref('dim_deals_flow_activity') }}
+-- depends_on: {{ ref('dim_deals') }}
+-- depends_on: {{ ref('dim_deals_flow_deal_change') }}
+-- depends_on: {{ ref('dim_deals_participants') }}
+-- depends_on: {{ ref('dim_deals_flow_note') }}
+-- depends_on: {{ ref('dim_activities__participants') }}
+-- depends_on: {{ ref('dim_deals_flow_activity__participants') }}
+-- depends_on: {{ ref('dim_deals_participants__person_id__email') }}
+-- depends_on: {{ ref('dim_deals_participants__person_id__phone') }}
+-- depends_on: {{ ref('dim_deals_participants__person__phone') }}
+-- depends_on: {{ ref('dim_deals_participants__person__email') }}
+-- depends_on: {{ ref('dim_deals_participants__person__im') }}
+-- depends_on: {{ ref('dim_persons__phone') }}
+-- depends_on: {{ ref('dim_persons__email') }}
+-- depends_on: {{ ref('dim_persons__im') }}
+/* we save all currently active load ids into the processed ids table */
 select load_id, {{ current_timestamp() }} as inserted_at FROM {{ ref('dlt_active_load_ids') }}

models/marts/dim__dlt_loads.sql

@@ -1,14 +1,14 @@
-/* Table: _dlt_loads */
-{{
-    config(
-        materialized='incremental'
-    )
-}}
-/* Description: Created by DLT. Tracks completed loads */
-SELECT
-    t.load_id,
-    t.schema_name,
-    t.status,
-    t.inserted_at,
-    t.schema_version_hash,
+/* Table: _dlt_loads */
+{{
+    config(
+        materialized='incremental'
+    )
+}}
+/* Description: Created by DLT. Tracks completed loads */
+SELECT
+    t.load_id,
+    t.schema_name,
+    t.status,
+    t.inserted_at,
+    t.schema_version_hash,
 FROM  {{ ref('stg__dlt_loads') }} as t

target/compiled/pipedrive/models/marts/dim_deals_flow_activity.sql → models/marts/dim_activities.sql

@@ -1,35 +1,39 @@
-/* Table: deals_flow_activity */
-
-SELECT
-    t.id,
-    t.company_id,
-    t.user_id,
-    t.done,
-    t.type,
-    t.due_date,
-    t.due_time,
-    t.duration,
-    t.busy_flag,
-    t.add_time,
-    t.marked_as_done_time,
-    t.subject,
-    t.public_description,
-    t.org_id,
-    t.person_id,
-    t.deal_id,
-    t.active_flag,
-    t.update_time,
-    t.private,
-    t.created_by_user_id,
-    t.org_name,
-    t.person_name,
-    t.deal_title,
-    t.owner_name,
-    t.person_dropbox_bcc,
-    t.deal_dropbox_bcc,
-    t.assigned_to_user_id,
-    t.type_name,
-    t.timestamp,
-    t._dlt_load_id,
-    t._dlt_id,
-FROM  `dlthub-sandbox`.`pipedrive_data_transformed`.`stg_deals_flow_activity` as t
+/* Table: activities */
+{{
+    config(
+        materialized='incremental'
+    )
+}}
+SELECT
+    t.id,
+    t.company_id,
+    t.user_id,
+    t.done,
+    t.type,
+    t.due_date,
+    t.due_time,
+    t.duration,
+    t.busy_flag,
+    t.add_time,
+    t.marked_as_done_time,
+    t.subject,
+    t.public_description,
+    t.org_id,
+    t.person_id,
+    t.deal_id,
+    t.active_flag,
+    t.update_time,
+    t.private,
+    t.created_by_user_id,
+    t.org_name,
+    t.person_name,
+    t.deal_title,
+    t.owner_name,
+    t.person_dropbox_bcc,
+    t.deal_dropbox_bcc,
+    t.assigned_to_user_id,
+    t.type_name,
+    t._dlt_load_id,
+    t._dlt_id,
+    t.marked_as_done_time__v_text,
+FROM  {{ ref('stg_activities') }} as t

models/marts/dim_activities__participants.sql

@@ -1,18 +1,18 @@
-/* Table: activities__participants */
-/* Parent: activities */
-{{
-    config(
-        materialized='incremental'
-    )
-}}
-SELECT
-    t.person_id,
-    t.primary_flag,
-    t._dlt_root_id,
-    t._dlt_parent_id,
-    t._dlt_list_idx,
-    t._dlt_id,
-FROM  {{ ref('stg_activities__participants') }} as t
-/* this join to the parent table is not necessarily needed for this template to work */
-JOIN {{ ref('stg_activities') }} as pt
+/* Table: activities__participants */
+/* Parent: activities */
+{{
+    config(
+        materialized='incremental'
+    )
+}}
+SELECT
+    t.person_id,
+    t.primary_flag,
+    t._dlt_root_id,
+    t._dlt_parent_id,
+    t._dlt_list_idx,
+    t._dlt_id,
+FROM  {{ ref('stg_activities__participants') }} as t
+/* this join to the parent table is not necessarily needed for this template to work */
+JOIN {{ ref('stg_activities') }} as pt
 ON (t._dlt_parent_id = pt._dlt_id)

models/marts/dim_custom_fields_mapping.sql

@@ -1,16 +1,16 @@
-/* Table: custom_fields_mapping */
-{{
-    config(
-        materialized='incremental'
-    )
-}}
-SELECT
-    t.options,
-    t.endpoint,
-    t.hash_string,
-    t.name,
-    t.normalized_name,
-    t.field_type,
-    t._dlt_load_id,
-    t._dlt_id,
+/* Table: custom_fields_mapping */
+{{
+    config(
+        materialized='incremental'
+    )
+}}
+SELECT
+    t.options,
+    t.endpoint,
+    t.hash_string,
+    t.name,
+    t.normalized_name,
+    t.field_type,
+    t._dlt_load_id,
+    t._dlt_id,
 FROM  {{ ref('stg_custom_fields_mapping') }} as t