Merge pull request #103 from fivetran/release/v0.11.0

release/v0.11.0

.buildkite/scripts/run_models.sh

@@ -20,4 +20,6 @@ dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run --vars '{using_schedules: false, using_domain_names: false, using_user_tags: false, using_ticket_form_history: false, using_organization_tags: false}' --target "$db" --full-refresh
-dbt test --target "$db"
+dbt test --target "$db"
+
+dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

.github/PULL_REQUEST_TEMPLATE/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md

@@ -0,0 +1,51 @@
+## PR Overview
+**This PR will address the following Issue/Feature:**
+
+**This PR will result in the following new package version:**
+<!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
+
+**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:**
+
+## PR Checklist
+### Basic Validation
+Please acknowledge that you have successfully performed the following commands locally:
+- [ ] dbt compile
+- [ ] dbt run –full-refresh
+- [ ] dbt run
+- [ ] dbt test
+- [ ] dbt run –vars (if applicable)
+
+Before marking this PR as "ready for review" the following have been applied:
+- [ ] The appropriate issue has been linked and tagged
+- [ ] You are assigned to the corresponding issue and this PR
+- [ ] BuildKite integration tests are passing
+
+### Detailed Validation
+Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review":
+- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature.
+- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages.
+- [ ] You have provided details below around the validation steps performed to gain confidence in these changes.
+<!--- Provide the steps you took to validate your changes below. -->
+
+### Standard Updates
+Please acknowledge that your PR contains the following standard updates:
+- Package versioning has been appropriately indexed in the following locations:
+    - [ ] indexed within dbt_project.yml
+    - [ ] indexed within integration_tests/dbt_project.yml
+- [ ] CHANGELOG has individual entries for each respective change in this PR
+    <!--- If there is a parallel upstream change, remember to reference the corresponding CHANGELOG as an individual entry.  -->
+- [ ] README updates have been applied (if applicable)
+    <!--- Remember to check the following README locations for common updates. →
+        <!--- Suggested install range (needed for breaking changes) →
+        <!--- Dependency matrix is appropriately updated (if applicable) →
+        <!--- New variable documentation (if applicable) -->
+- [ ] DECISIONLOG updates have been updated (if applicable)
+- [ ] Appropriate yml documentation has been added (if applicable)
+
+### dbt Docs
+Please acknowledge that after the above were all completed the below were applied to your branch:
+- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
+
+### If you had to summarize this PR in an emoji, which would it be?
+<!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
+:dancer:

.github/pull_request_template.md

@@ -1,52 +1,33 @@
-**Are you a current Fivetran customer?** 
-<!--- Please tell us your name, title and company -->
+**Please provide your name and company**
 
-**What change(s) does this PR introduce?** 
-<!--- Describe what changes your PR introduces to the package and how to leverage this new feature. -->
+**Link the issue/feature request which this PR is meant to address**
+<!--- If an issue was not created, please create one first so we may discuss the PR prior to opening one. -->
 
-**Did you update the CHANGELOG?** 
+**Detail what changes this PR introduces and how this addresses the issue/feature request linked above.**
+
+**How did you validate the changes introduced within this PR?**
+
+**Which warehouse did you use to develop these changes?**
+
+**Did you update the CHANGELOG?**
 <!--- Please update the new package version’s CHANGELOG entry detailing the changes included in this PR. -->
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
 - [ ] Yes
 
-**Does this PR introduce a breaking change?**
-<!--- Does this PR introduce changes that will cause current package users' jobs to fail or require a `--full-refresh`? -->
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Yes (please provide breaking change details below.)
-- [ ] No  (please provide an explanation as to how the change is non-breaking below.)
-
-**Did you update the dbt_project.yml files with the version upgrade (please leverage standard semantic versioning)? (In both your main project and integration_tests)** 
+**Did you update the dbt_project.yml files with the version upgrade (please leverage standard semantic versioning)? (In both your main project and integration_tests)**
 <!--- The dbt_project.yml and the integration_tests/dbt_project.yml files contain the version number. Be sure to upgrade it accordingly -->
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
 - [ ] Yes
 
-**Is this PR in response to a previously created Bug or Feature Request**
-<!--- If an Issue was created it is helpful to track the progress by linking it in the PR. -->
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Yes, Issue/Feature [link bug/feature number here]
-- [ ] No 
-
-**How did you test the PR changes?** 
-<!--- Proof of testing is required in order for the PR to be approved. -->
-<!--- To check a box, remove the space and insert an x in the box (eg. [x] Buildkite). --> 
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Buildkite <!--- Buildkite testing is only applicable to Fivetran employees. --> 
-- [ ] Local (please provide additional testing details below)
-
-**Select which warehouse(s) were used to test the PR**
-<!--- To check a warehouse remove the space and insert an x in the box (eg. [x] Bigquery). --> 
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] BigQuery
-- [ ] Redshift
-- [ ] Snowflake
-- [ ] Postgres
-- [ ] Databricks
-- [ ] Other (provide details below)
-
 **Provide an emoji that best describes your current mood**
 <!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
 :dancer:
 
 **Feedback**
 
 We are so excited you decided to contribute to the Fivetran community dbt package! We continue to work to improve the packages and would greatly appreciate your [feedback](https://www.surveymonkey.com/r/DQ7K7WW) on our existing dbt packages or what you'd like to see next.
+
+**PR Template** 
+- [Community Pull Request Template](?expand=1&template=pull_request_template.md) (default)
+
+- [Maintainer Pull Request Template](?expand=1&template=maintainer_pull_request_template.md) (to be used by maintainers)

CHANGELOG.md

@@ -1,9 +1,41 @@
+# dbt_zendesk v0.11.0
+## Feature Updates:
+- Added support of the new `schedule_holiday` table in the `schedule_spine` intermediate model in order to properly capture how holidays impact ticket schedules and their respective SLAs. ([PR #98](https://github.com/fivetran/dbt_zendesk/pull/98))
+- Made relevant downstream changes within the following models to capture proper business hour metrics when taking into account holiday schedules: ([PR #98](https://github.com/fivetran/dbt_zendesk/pull/98))
+  - `int_zendesk__agent_work_time_business_hours`
+  - `int_zendesk__reply_time_business_hours`
+  - `int_zendesk__reply_time_combined`
+  - `int_zendesk__requester_wait_time_business_hours`
+  - `zendesk__sla_policies`
+- Added `open_status_duration_in_business_minutes` and `new_status_duration_in_business_minutes` columns to the `int_zendesk__ticket_work_time_business` and `zendesk__ticket_metrics` models. These are counterparts to the already existing `open_status_duration_in_calendar_minutes` and `new_status_duration_in_calendar_minutes` columns. ([PR #97](https://github.com/fivetran/dbt_zendesk/pull/97)) 
+
+## Fixes:
+- Added coalesce to `0` statements to the following fields in the `zendesk__ticket_metrics` model. This is necessary as some tickets may have responses entirely outside of business hours which will not count towards business minute metrics. As such, a coalesce to `0` is more representative to the metric as opposed to a `null` record: ([PR #103](https://github.com/fivetran/dbt_zendesk/pull/103))
+  - `first_resolution_business_minutes`
+  - `full_resolution_business_minutes`
+  - `first_reply_time_business_minutes`
+  - `agent_wait_time_in_business_minutes`
+  - `requester_wait_time_in_business_minutes`
+  - `agent_work_time_in_business_minutes`
+  - `on_hold_time_in_business_minutes`
+- Fixed the `total_agent_replies` field in `zendesk__ticket_metrics` so the value is derived from public agent comments logic, and also ignores ticket creation comments from an agent, matching the Zendesk definition. ([PR #102](https://github.com/fivetran/dbt_zendesk/pull/102))
+
+## Under the Hood:
+- Leveraged `dbt_date.week_start` in place of `dbt.date_trunc` for business hour metrics to more consistently capture the start of the week across warehouses. ([PR #98](https://github.com/fivetran/dbt_zendesk/pull/98))
+- Start of the week is now consistently set to Sunday. ([PR #98](https://github.com/fivetran/dbt_zendesk/pull/98))
+- Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job. ([PR #98](https://github.com/fivetran/dbt_zendesk/pull/98))
+- Updated the pull request templates. ([PR #98](https://github.com/fivetran/dbt_zendesk/pull/98))
+
+## Contributors:
+- [@Tim-Hoare](https://github.com/Tim-Hoare) ([PR #97](https://github.com/fivetran/dbt_zendesk/pull/97)) 
+
 # dbt_zendesk v0.10.2
 [PR #101](https://github.com/fivetran/dbt_zendesk/pull/101) includes the following updates:
 ## Fixes
 - Updated the `group` variable in the `dbt_project.yml` to have properly closed quotes within the variable declaration.
 - Adjusted the `in_zendesk__calendar_spine` to set the return result of `dbt.current_timestamp_backcompat()` as a variable. This ensures that when the variable is being called within the model it can properly establish a dependency within the manifest.
 
+
 # dbt_zendesk v0.10.1
 ## Bug Fixes
 - Modified the `int_zendesk__ticket_schedules` model to have the execute statement reference the source `schedule` table as opposed to the `stg_zendesk__schedule` model so the package may successfully compile before being run for the first time. ([#90](https://github.com/fivetran/dbt_zendesk/pull/90))

README.md

@@ -59,7 +59,7 @@ Include the following zendesk package version in your `packages.yml` file:
 ```yml
 packages:
   - package: fivetran/zendesk
-    version: [">=0.10.0", "<0.11.0"]
+    version: [">=0.11.0", "<0.12.0"]
 
 ```
 > **Note**: Do not include the Zendesk source package. The Zendesk transform package already has a dependency on the source in its own `packages.yml` file.
@@ -177,7 +177,7 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
     - package: fivetran/zendesk_source
-      version: [">=0.8.0", "<0.9.0"]
+      version: [">=0.9.0", "<0.10.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'zendesk'
-version: '0.10.2'
+version: '0.11.0'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
@@ -35,6 +35,7 @@ vars:
     organization_tag: "{{ ref('stg_zendesk__organization_tag') }}"
     organization: "{{ ref('stg_zendesk__organization') }}"
     schedule: "{{ ref('stg_zendesk__schedule') }}"
+    schedule_holiday: "{{ ref('stg_zendesk__schedule_holiday') }}"
     ticket: "{{ ref('stg_zendesk__ticket') }}"
     ticket_form_history: "{{ ref('stg_zendesk__ticket_form_history') }}"
     ticket_comment: "{{ ref('stg_zendesk__ticket_comment') }}"

integration_tests/.gitignore

@@ -3,4 +3,5 @@ target/
 dbt_modules/
 logs/
 .DS_Store
-dbt_packages/
+dbt_packages/
+env/

integration_tests/dbt_project.yml

@@ -1,8 +1,7 @@
 config-version: 2
 
 name: 'zendesk_integration_tests'
-version: '0.10.2'
-
+version: '0.11.0'
 
 profile: 'integration_tests'
 
@@ -11,6 +10,7 @@ vars:
   zendesk_source:
     zendesk_organization_identifier:           "organization_data"
     zendesk_schedule_identifier:               "schedule_data"
+    zendesk_schedule_holiday_identifier:       "schedule_holiday_data"
     zendesk_ticket_identifier:                 "ticket_data"
     zendesk_ticket_comment_identifier:         "ticket_comment_data"
     zendesk_ticket_field_history_identifier:   "ticket_field_history_data"
@@ -49,6 +49,10 @@ seeds:
         end_time_utc: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
         start_time_utc: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
         created_at: timestamp
+    schedule_holiday_data:
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        schedule_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
     ticket_comment_data:
       +column_types:
         id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"

integration_tests/requirements.txt

@@ -1,6 +1,6 @@
 dbt-snowflake>=1.3.0,<2.0.0
 dbt-bigquery>=1.3.0,<2.0.0
-dbt-redshift>=1.3.0,<2.0.0
+dbt-redshift>=1.3.0,<1.5.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0

integration_tests/seeds/schedule_holiday_data.csv

@@ -0,0 +1,3 @@
+id,schedule_id,_fivetran_deleted,_fivetran_synced,end_date,name,start_date
+11630286047764,11630270684308,FALSE,2023-01-19 21:53:06.281,2022-12-27,Test Holiday,2022-12-26
+11630270740884,11630270684308,FALSE,2023-01-19 21:53:06.281,2022-12-29,Test Holiday 2,2022-12-29

models/agent_work_time/int_zendesk__ticket_work_time_business.sql

@@ -113,7 +113,11 @@ with ticket_historical_status as (
       case when ticket_status in ('new', 'open') then scheduled_minutes
           else 0 end as agent_work_time_in_minutes,
       case when ticket_status in ('hold') then scheduled_minutes
-          else 0 end as on_hold_time_in_minutes
+          else 0 end as on_hold_time_in_minutes,
+      case when ticket_status = 'new' then scheduled_minutes
+          else 0 end as new_status_duration_minutes,
+      case when ticket_status = 'open' then scheduled_minutes
+          else 0 end as open_status_duration_minutes
     from intercepted_periods
 
 )
@@ -123,6 +127,8 @@ with ticket_historical_status as (
       sum(agent_wait_time_in_minutes) as agent_wait_time_in_business_minutes,
       sum(requester_wait_time_in_minutes) as requester_wait_time_in_business_minutes,
       sum(agent_work_time_in_minutes) as agent_work_time_in_business_minutes,
-      sum(on_hold_time_in_minutes) as on_hold_time_in_business_minutes
+      sum(on_hold_time_in_minutes) as on_hold_time_in_business_minutes,
+      sum(new_status_duration_minutes) as new_status_duration_in_business_minutes,
+      sum(open_status_duration_minutes) as open_status_duration_in_business_minutes
     from business_minutes
     group by 1

models/intermediate/int_zendesk__comment_metrics.sql

@@ -31,7 +31,11 @@ comment_counts as (
         count(*) as total_comments,
         count(distinct case when commenter_role = 'internal_comment'
             then user_id
-                end) as count_ticket_handoffs
+                end) as count_ticket_handoffs,
+        sum(case when commenter_role = 'internal_comment' and is_public = true and previous_commenter_role != 'first_comment'
+            then 1
+            else 0
+                end) as count_agent_replies
     from ticket_comments
 
     group by 1, 2