Merge pull request #89 from fivetran/MagicBot/databricks-compatibility

MagicBot/databricks-compatibility

Author: fivetran-catfritz

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -56,4 +56,19 @@ steps:
             - "CI_REDSHIFT_DBT_PASS"
             - "CI_REDSHIFT_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh redshift
+      bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":databricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks

CHANGELOG.md

@@ -1,3 +1,7 @@
+# dbt_salesforce_formula_utils v0.9.0
+## 🎉 Feature Update 🎉
+- Databricks compatibility! ([#89](https://github.com/fivetran/dbt_salesforce_formula_utils/pull/89))
+
 # dbt_salesforce_formula_utils v0.8.2
 ## Features
 - Clarify Step 5 of the README that `sfdc_exclude_formulas` works only with Options 2 and 3. ([#88](https://github.com/fivetran/dbt_salesforce_formula_utils/pull/88))

README.md

@@ -20,15 +20,15 @@ This package includes macros and scipts to be used within a dbt project to accur
 ## Step 1: Prerequisites
 To use this dbt package, you must have the following:
 - At least one Fivetran Salesforce connector syncing data into your destination. 
-- A **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination.
+- A **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
 
 ## Step 2: Install the package
 The [`sfdc_formula_view`](https://github.com/fivetran/dbt_salesforce_formula_utils#sfdc_formula_view-source) macro is intended to be leveraged within a dbt project that uses the source tables from Fivetran's Salesforce connector. To leverage the macro, you will add the below configuration to your `packages.yml` file (if you do not have a `packages.yml` file, create one in your root dbt project).
 > TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
 ```yml
 packages:
   - package: fivetran/salesforce_formula_utils
-    version: [">=0.8.0", "<0.9.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.9.0", "<0.10.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ## Step 3: Define required source tables

dbt_project.yml

@@ -1,4 +1,4 @@
 name: 'salesforce_formula_utils'
-version: '0.8.2'
+version: '0.9.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]

docs/catalog.json

@@ -1 +1 @@
-{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v4.json", "dbt_version": "1.3.0", "generated_at": "2022-12-19T16:52:36.401683Z", "invocation_id": "58406d35-bbcd-4fb3-b60f-0291900c4e39", "env": {}}, "results": [{"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:35.614834Z", "completed_at": "2022-12-19T16:52:35.955070Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:35.955842Z", "completed_at": "2022-12-19T16:52:35.955864Z"}], "thread_id": "Thread-3", "execution_time": 0.36652088165283203, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.opportunity_statement_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:35.614757Z", "completed_at": "2022-12-19T16:52:35.975733Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:35.977801Z", "completed_at": "2022-12-19T16:52:35.977808Z"}], "thread_id": "Thread-2", "execution_time": 0.3926880359649658, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.account_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:35.630293Z", "completed_at": "2022-12-19T16:52:35.976412Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:35.977289Z", "completed_at": "2022-12-19T16:52:35.977307Z"}], "thread_id": "Thread-4", "execution_time": 0.3930974006652832, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.opportunity_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:35.614598Z", "completed_at": "2022-12-19T16:52:35.977031Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:35.978320Z", "completed_at": "2022-12-19T16:52:35.978329Z"}], "thread_id": "Thread-1", "execution_time": 0.3968508243560791, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.account_statement_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:35.981020Z", "completed_at": "2022-12-19T16:52:36.288462Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.288929Z", "completed_at": "2022-12-19T16:52:36.288945Z"}], "thread_id": "Thread-3", "execution_time": 0.3494741916656494, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.user_role_statement_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:36.341067Z", "completed_at": "2022-12-19T16:52:36.346184Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.346743Z", "completed_at": "2022-12-19T16:52:36.346758Z"}], "thread_id": "Thread-3", "execution_time": 0.018234729766845703, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.account"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:36.358416Z", "completed_at": "2022-12-19T16:52:36.360067Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.360229Z", "completed_at": "2022-12-19T16:52:36.360235Z"}], "thread_id": "Thread-3", "execution_time": 0.0029382705688476562, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.fivetran_formula"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:36.360974Z", "completed_at": "2022-12-19T16:52:36.362284Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.362494Z", "completed_at": "2022-12-19T16:52:36.362499Z"}], "thread_id": "Thread-3", "execution_time": 0.002101898193359375, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.fivetran_formula_model"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:36.363454Z", "completed_at": "2022-12-19T16:52:36.364784Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.364927Z", "completed_at": "2022-12-19T16:52:36.364932Z"}], "thread_id": "Thread-3", "execution_time": 0.002151012420654297, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.opportunity"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:36.365714Z", "completed_at": "2022-12-19T16:52:36.367607Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.369248Z", "completed_at": "2022-12-19T16:52:36.369254Z"}], "thread_id": "Thread-3", "execution_time": 0.00421595573425293, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.user"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:36.370161Z", "completed_at": "2022-12-19T16:52:36.371580Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.372048Z", "completed_at": "2022-12-19T16:52:36.372053Z"}], "thread_id": "Thread-3", "execution_time": 0.0026221275329589844, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.user_role"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:36.020971Z", "completed_at": "2022-12-19T16:52:36.339835Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.340749Z", "completed_at": "2022-12-19T16:52:36.340779Z"}], "thread_id": "Thread-1", "execution_time": 0.3615100383758545, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.user_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:36.012464Z", "completed_at": "2022-12-19T16:52:36.371950Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.372785Z", "completed_at": "2022-12-19T16:52:36.372788Z"}], "thread_id": "Thread-4", "execution_time": 0.39004015922546387, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.user_statement_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2022-12-19T16:52:36.011558Z", "completed_at": "2022-12-19T16:52:36.371666Z"}, {"name": "execute", "started_at": "2022-12-19T16:52:36.372135Z", "completed_at": "2022-12-19T16:52:36.372138Z"}], "thread_id": "Thread-2", "execution_time": 0.39188575744628906, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.user_role_view"}], "elapsed_time": 1.6709964275360107, "args": {"write_json": true, "use_colors": true, "printer_width": 80, "version_check": true, "partial_parse": true, "static_parser": true, "profiles_dir": "/Users/joseph.markiewicz/.dbt", "send_anonymous_usage_stats": true, "event_buffer_size": 100000, "quiet": false, "no_print": false, "target": "postgres", "compile": true, "which": "generate", "rpc_method": "docs.generate", "indirect_selection": "eager"}}
+{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v4.json", "dbt_version": "1.5.2", "generated_at": "2023-06-23T14:16:16.636624Z", "invocation_id": "d895caf0-6088-44bd-9ae3-4ff4ed92c0c4", "env": {}}, "results": [{"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:15.741927Z", "completed_at": "2023-06-23T14:16:16.416059Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.417039Z", "completed_at": "2023-06-23T14:16:16.417065Z"}], "thread_id": "Thread-1", "execution_time": 0.7453927993774414, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.account_statement_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:16.486702Z", "completed_at": "2023-06-23T14:16:16.489365Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.490632Z", "completed_at": "2023-06-23T14:16:16.490640Z"}], "thread_id": "Thread-1", "execution_time": 0.008205175399780273, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.account"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:16.505572Z", "completed_at": "2023-06-23T14:16:16.507051Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.509966Z", "completed_at": "2023-06-23T14:16:16.509970Z"}], "thread_id": "Thread-1", "execution_time": 0.009269952774047852, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.fivetran_formula"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:16.512269Z", "completed_at": "2023-06-23T14:16:16.513471Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.513990Z", "completed_at": "2023-06-23T14:16:16.513994Z"}], "thread_id": "Thread-1", "execution_time": 0.0028657913208007812, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.fivetran_formula_model"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:16.515734Z", "completed_at": "2023-06-23T14:16:16.516887Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.517435Z", "completed_at": "2023-06-23T14:16:16.517439Z"}], "thread_id": "Thread-1", "execution_time": 0.002789020538330078, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.opportunity"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:16.519119Z", "completed_at": "2023-06-23T14:16:16.521183Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.521658Z", "completed_at": "2023-06-23T14:16:16.521662Z"}], "thread_id": "Thread-1", "execution_time": 0.0035729408264160156, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.user"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:16.523230Z", "completed_at": "2023-06-23T14:16:16.524296Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.524762Z", "completed_at": "2023-06-23T14:16:16.524766Z"}], "thread_id": "Thread-1", "execution_time": 0.002538919448852539, "adapter_response": {}, "message": null, "failures": null, "unique_id": "seed.salesforce_formula_integration_tests.user_role"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:15.776892Z", "completed_at": "2023-06-23T14:16:16.503667Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.508333Z", "completed_at": "2023-06-23T14:16:16.508338Z"}], "thread_id": "Thread-7", "execution_time": 0.8214058876037598, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.user_statement_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:15.766635Z", "completed_at": "2023-06-23T14:16:16.502726Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.507424Z", "completed_at": "2023-06-23T14:16:16.507430Z"}], "thread_id": "Thread-3", "execution_time": 0.8252599239349365, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.opportunity_statement_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:15.771942Z", "completed_at": "2023-06-23T14:16:16.503364Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.508013Z", "completed_at": "2023-06-23T14:16:16.508018Z"}], "thread_id": "Thread-5", "execution_time": 0.8239889144897461, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.user_role_statement_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:15.769382Z", "completed_at": "2023-06-23T14:16:16.491365Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.500544Z", "completed_at": "2023-06-23T14:16:16.500552Z"}], "thread_id": "Thread-4", "execution_time": 0.8254609107971191, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.opportunity_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:15.763755Z", "completed_at": "2023-06-23T14:16:16.492437Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.501071Z", "completed_at": "2023-06-23T14:16:16.501078Z"}], "thread_id": "Thread-2", "execution_time": 0.8301539421081543, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.account_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:15.779050Z", "completed_at": "2023-06-23T14:16:16.504558Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.509140Z", "completed_at": "2023-06-23T14:16:16.509144Z"}], "thread_id": "Thread-8", "execution_time": 0.8259849548339844, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.user_view"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-06-23T14:16:15.774153Z", "completed_at": "2023-06-23T14:16:16.568917Z"}, {"name": "execute", "started_at": "2023-06-23T14:16:16.569454Z", "completed_at": "2023-06-23T14:16:16.569459Z"}], "thread_id": "Thread-6", "execution_time": 0.8931610584259033, "adapter_response": {}, "message": null, "failures": null, "unique_id": "model.salesforce_formula_integration_tests.user_role_view"}], "elapsed_time": 3.0202150344848633, "args": {"log_level": "info", "version_check": true, "indirect_selection": "eager", "project_dir": "/Users/catherinefritz/Documents/dbt_packages/salesforce_formula_utils/dbt_salesforce_formula_utils/integration_tests", "introspect": true, "exclude": [], "log_format_file": "debug", "select": [], "favor_state": false, "vars": {}, "log_path": "/Users/catherinefritz/Documents/dbt_packages/salesforce_formula_utils/dbt_salesforce_formula_utils/integration_tests/logs", "use_colors": true, "static_parser": true, "print": true, "profiles_dir": "/Users/catherinefritz/.dbt", "which": "generate", "warn_error_options": {"include": [], "exclude": []}, "defer": false, "log_level_file": "debug", "populate_cache": true, "strict_mode": false, "send_anonymous_usage_stats": true, "printer_width": 80, "quiet": false, "enable_legacy_logger": false, "write_json": true, "macro_debugging": false, "log_format": "default", "use_colors_file": true, "partial_parse": true, "cache_selected_only": false, "compile": true}}

docs/index.html

@@ -45,10 +45,10 @@ integration_tests:
       schema: salesforce_formula_integrations_tests
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: salesforce_formula_integrations_tests
-      threads: 2
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

docs/manifest.json

@@ -1,5 +1,5 @@
 name: 'salesforce_formula_integration_tests'
-version: '0.8.2'
+version: '0.9.0'
 profile: 'integration_tests'
 config-version: 2
 vars:
@@ -16,3 +16,7 @@ seeds:
     fivetran_formula:
       +column_types:
         view_sql: "{{ 'varchar(100)'  if target.type in ('redshift','postgres') else 'string'}}"
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']