update/get-ticket-property-history-columns (#126)

* update/get-ticket-property-history-columns * update docs and changelog, regen docs * update changelog * add under the hood items * make breaking * regen docs * Apply suggestions from code review Co-authored-by: Joe Markiewicz <[email protected]> * release review updates and databricks sql test run addition * catalog env var added to databricks test * update changelog --------- Co-authored-by: Joe Markiewicz <[email protected]>
fivetran · Jul 31, 2024 · d1d6e9c · d1d6e9c
1 parent 81e916b
commit d1d6e9c
Show file tree

Hide file tree

Showing 18 changed files with 117 additions and 57 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -21,4 +21,7 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_SQL_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_SQL_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_TOKEN" --project="dbt-package-testing-363917")
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -69,5 +69,21 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
-      bash .buildkite/scripts/run_models.sh databricks
+      bash .buildkite/scripts/run_models.sh databricks
+
+  - label: ":databricks: :database: Run Tests - Databricks SQL Warehouse"
+    key: "run_dbt_databricks_sql"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_SQL_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_SQL_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks-sql
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -16,6 +16,15 @@ db=$1
 echo `pwd`
 cd integration_tests
 dbt deps
+if [ "$db" = "databricks-sql" ]; then
+dbt seed --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db" --full-refresh
+dbt compile --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
+dbt run --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db" --full-refresh
+dbt test --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
+dbt run --vars '{hubspot_schema: hubspot_sqlw_tests, hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db"
+dbt run --vars '{hubspot_schema: hubspot_sqlw_tests, hubspot_marketing_enabled: false, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true, hubspot__pass_through_all_columns: true, hubspot_using_all_email_events: false, hubspot_owner_enabled: false}' --target "$db"
+dbt test --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
+else
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"            
@@ -26,5 +35,5 @@ dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}
 dbt run --vars '{hubspot_deal_enabled: true, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true}' --target "$db"
 dbt run --vars '{hubspot__pass_through_all_columns: true, hubspot_service_enabled: true, hubspot_using_all_email_events: false}' --target "$db"
 dbt test --target "$db"
-
+fi
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md
@@ -4,48 +4,27 @@
 **This PR will result in the following new package version:**
 <!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
 
-**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:**
+**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:**
+<!--- Copy/paste the CHANGELOG for this version below. -->
 
 ## PR Checklist
 ### Basic Validation
 Please acknowledge that you have successfully performed the following commands locally:
-- [ ] dbt compile
-- [ ] dbt run –full-refresh
-- [ ] dbt run
-- [ ] dbt test
-- [ ] dbt run –vars (if applicable)
+- [ ] dbt run –full-refresh && dbt test
+- [ ] dbt run (if incremental models are present) && dbt test
 
 Before marking this PR as "ready for review" the following have been applied:
-- [ ] The appropriate issue has been linked and tagged
-- [ ] You are assigned to the corresponding issue and this PR
+- [ ] The appropriate issue has been linked, tagged, and properly assigned
+- [ ] All necessary documentation and version upgrades have been applied
+    <!--- Be sure to update the package version in the dbt_project.yml, integration_tests/dbt_project.yml, and README if necessary. -->
+- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
 - [ ] BuildKite integration tests are passing
+- [ ] Detailed validation steps have been provided below
 
 ### Detailed Validation
-Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review":
-- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature.
-- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages.
-- [ ] You have provided details below around the validation steps performed to gain confidence in these changes.
+Please share any and all of your validation steps:
 <!--- Provide the steps you took to validate your changes below. -->
 
-### Standard Updates
-Please acknowledge that your PR contains the following standard updates:
-- Package versioning has been appropriately indexed in the following locations:
-    - [ ] indexed within dbt_project.yml
-    - [ ] indexed within integration_tests/dbt_project.yml
-- [ ] CHANGELOG has individual entries for each respective change in this PR
-    <!--- If there is a parallel upstream change, remember to reference the corresponding CHANGELOG as an individual entry.  -->
-- [ ] README updates have been applied (if applicable)
-    <!--- Remember to check the following README locations for common updates. →
-        <!--- Suggested install range (needed for breaking changes) →
-        <!--- Dependency matrix is appropriately updated (if applicable) →
-        <!--- New variable documentation (if applicable) -->
-- [ ] DECISIONLOG updates have been updated (if applicable)
-- [ ] Appropriate yml documentation has been added (if applicable)
-
-### dbt Docs
-Please acknowledge that after the above were all completed the below were applied to your branch:
-- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
-
 ### If you had to summarize this PR in an emoji, which would it be?
 <!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
-:dancer:
+:dancer:
diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml
@@ -0,0 +1,13 @@
+name: 'auto release'
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  call-workflow-passing-data:
+    if: github.event.pull_request.merged
+    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
+    secrets: inherit
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,18 @@
+# dbt_hubspot_source v0.15.0
+[PR #126](https://github.com/fivetran/dbt_hubspot_source/pull/126) includes the following updates:
+
+## 🚨 Breaking Changes 🚨
+- Added field `_fivetran_end` to macro `get_ticket_property_history_columns()` to ensure the column is available for use downstream.
+  - While this should not affect most users, this will add a new column `_fivetran_end` to `stg_hubspot__ticket_property_history` if you do not have `_fivetran_end` in your source `TICKET_PROPERTY_HISTORY` table.
+
+## Documentation
+- Update documentation to include `_fivetran_end` under model `stg_hubspot__ticket_property_history`.
+
+## Under the hood
+- Updated the maintainer PR template to the current format.
+- Included auto-releaser GitHub Actions workflow to automate future releases.
+- Added integration testing for Databricks SQL Warehouse.
+
 # dbt_hubspot_source v0.14.1
 [PR #125](https://github.com/fivetran/dbt_hubspot_source/pull/125) includes the following updates:
 

diff --git a/README.md b/README.md
@@ -44,7 +44,7 @@ Include the following hubspot_source package version in your `packages.yml` file
 ```yaml
 packages:
   - package: fivetran/hubspot_source
-    version: [">=0.14.0", "<0.15.0"]
+    version: [">=0.15.0", "<0.16.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `hubspot` schema. If this is not where your HubSpot data is (for example, if your HubSpot schema is named `hubspot_fivetran`), add the following configuration to your root `dbt_project.yml` file:

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.14.1'
+version: '0.15.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -45,10 +45,18 @@ integration_tests:
       schema: hubspot_source_integration_tests_21
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: hubspot_source_integration_tests_21
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
+    databricks-sql:
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
+      host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
+      http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}"
+      schema: hubspot_sqlw_tests
+      threads: 8
+      token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}"
+      type: databricks
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,10 +1,12 @@
 name: 'hubspot_source_integration_tests'
-version: '0.14.1'
+version: '0.15.0'
 profile: 'integration_tests'
 config-version: 2
+
 models:
-  hubspot_source:
-    +schema:
+  +schema: "{{ 'hubspot_sqlw_tests' if target.name == 'databricks-sql' else 'hubspot' }}"
+  # +schema: "hubspot_{{ var('directed_schema','dev') }}" ## To be used for validation testing
+
 vars:
   hubspot_schema: hubspot_source_integration_tests_21
   hubspot_source:

diff --git a/integration_tests/seeds/ticket_property_history_data.csv b/integration_tests/seeds/ticket_property_history_data.csv
@@ -1,12 +1,12 @@
-_fivetran_synced,ticket_id,name,source,source_id,timestamp_instant,value
-2020-07-09 11:06:21.056,1,seed name, source name, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,seed data is cool, source name1, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,i can make these names whatever i want, source name2, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,seed name1, source name3, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,seed name2, source name4, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,seed name3, source name5, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,stop being lazy with names, source name99, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,seed name4, source name6?, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,seed name5, source name7, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,seed name6, source name8, sg5923,2020-07-09 11:06:21.056,value
-2020-07-09 11:06:21.056,1,i got lazy with names, source name99, sg5923,2020-07-09 11:06:21.056,value
+_fivetran_synced,ticket_id,name,source,source_id,timestamp_instant,value,_fivetran_start,_fivetran_end
+2020-07-09 11:06:21.056,1,seed name, source name, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,seed data is cool, source name1, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,i can make these names whatever i want, source name2, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,seed name1, source name3, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,seed name2, source name4, sg5923,2020-07-09 11:06:21.056,value,2023-01-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,seed name3, source name5, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,stop being lazy with names, source name99, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,seed name4, source name6?, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,seed name5, source name7, sg5923,2020-07-09 11:06:21.056,value,2023-09-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,seed name6, source name8, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
+2020-07-09 11:06:21.056,1,i got lazy with names, source name99, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
diff --git a/macros/get_ticket_property_history_columns.sql b/macros/get_ticket_property_history_columns.sql
@@ -3,6 +3,7 @@
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
     {"name": "_fivetran_start", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_end", "datatype": dbt.type_timestamp()},
     {"name": "ticket_id", "datatype": dbt.type_int()},
     {"name": "name", "datatype": dbt.type_string(), "alias": "field_name"},
     {"name": "source", "datatype": dbt.type_string(), "alias": "change_source"},

diff --git a/models/docs.md b/models/docs.md
@@ -6,6 +6,14 @@ Timestamp of when Fivetran synced a record.
 Boolean indicating whether a record has been deleted in Hubspot and/or inferred deleted in Hubspot by Fivetran; _fivetran_deleted and is_deleted fields are equivalent. 
 {% enddocs %}
 
+{% docs _fivetran_end %}
+The (UTC DATETIME) when a record became inactive in the source database.
+{% enddocs %}
+
+{% docs _fivetran_start %}
+The (UTC DATETIME) to keep track of the time when a record is first created or modified in the source database.
+{% enddocs %}
+
 {% docs portal_id %}
 The hub ID.
 {% enddocs %}

diff --git a/models/src_hubspot.yml b/models/src_hubspot.yml
@@ -1274,6 +1274,10 @@ sources:
             description: '{{ doc("history_name") }}'
           - name: value
             description: '{{ doc("history_value") }}'
+          - name: _fivetran_start
+            description: '{{ doc("_fivetran_start") }}'
+          - name: _fivetran_end
+            description: '{{ doc("_fivetran_end") }}'
 
       - name: ticket
         identifier: "{{ var('hubspot_ticket_identifier', 'ticket')}}"

diff --git a/models/stg_hubspot__ticket.yml b/models/stg_hubspot__ticket.yml
@@ -48,7 +48,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
       - name: _fivetran_start
-        description: The (UTC DATETIME) to keep track of the time when a record is first created or modified in the source database.
+        description: '{{ doc("_fivetran_start") }}'
+      - name: _fivetran_end
+        description: '{{ doc("_fivetran_end") }}'
       - name: change_source
         description: '{{ doc("history_source") }}'
       - name: change_source_id