Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bug/work wait time filter #163

Closed
wants to merge 6 commits into from
Closed

Bug/work wait time filter #163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
338ae3a
cast to date
fivetran-catfritz Aug 19, 2024
a72165f
initial commit
fivetran-catfritz Aug 20, 2024
0fe373c
add dst to agent and requester
fivetran-catfritz Aug 20, 2024
fd4618b
update ticket_schedules
fivetran-catfritz Aug 21, 2024
0bf297e
add config and move to int
fivetran-catfritz Aug 21, 2024
76a6b96
add config and move to int
fivetran-catfritz Aug 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 9 additions & 83 deletions models/intermediate/int_zendesk__schedule_spine.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ with timezone as (
select *
from {{ var('time_zone') }}

), daylight_time as (
), timezones_with_dst as (

select *
from {{ var('daylight_time') }}
from {{ ref('int_zendesk__timezones_with_dst') }}

), schedule as (

Expand All @@ -36,80 +36,6 @@ with timezone as (
on holiday_start_date_at <= cast(date_day as {{ dbt.type_timestamp() }} )
and holiday_end_date_at >= cast(date_day as {{ dbt.type_timestamp() }} )

), timezone_with_dt as (

select
timezone.*,
daylight_time.daylight_start_utc,
daylight_time.daylight_end_utc,
daylight_time.daylight_offset_minutes

from timezone
left join daylight_time
on timezone.time_zone = daylight_time.time_zone

), order_timezone_dt as (

select
*,
-- will be null for timezones without any daylight savings records (and the first entry)
-- we will coalesce the first entry date with .... the X years ago
lag(daylight_end_utc, 1) over (partition by time_zone order by daylight_end_utc asc) as last_daylight_end_utc,
-- will be null for timezones without any daylight savings records (and the last entry)
-- we will coalesce the last entry date with the current date
lead(daylight_start_utc, 1) over (partition by time_zone order by daylight_start_utc asc) as next_daylight_start_utc

from timezone_with_dt

), split_timezones as (

-- standard schedule (includes timezones without DT)
-- starts: when the last Daylight Savings ended
-- ends: when the next Daylight Savings starts
select
time_zone,
standard_offset_minutes as offset_minutes,

-- last_daylight_end_utc is null for the first record of the time_zone's daylight time, or if the TZ doesn't use DT
coalesce(last_daylight_end_utc, cast('1970-01-01' as date)) as valid_from,

-- daylight_start_utc is null for timezones that don't use DT
coalesce(daylight_start_utc, cast( {{ dbt.dateadd('year', 1, dbt.current_timestamp_backcompat()) }} as date)) as valid_until

from order_timezone_dt

union all

-- DT schedule (excludes timezones without it)
-- starts: when this Daylight Savings started
-- ends: when this Daylight Savings ends
select
time_zone,
-- Pacific Time is -8h during standard time and -7h during DT
standard_offset_minutes + daylight_offset_minutes as offset_minutes,
daylight_start_utc as valid_from,
daylight_end_utc as valid_until

from order_timezone_dt
where daylight_offset_minutes is not null

union all

select
time_zone,
standard_offset_minutes as offset_minutes,

-- Get the latest daylight_end_utc time and set that as the valid_from
max(daylight_end_utc) as valid_from,

-- If the latest_daylight_end_time_utc is less than todays timestamp, that means DST has ended. Therefore, we will make the valid_until in the future.
cast( {{ dbt.dateadd('year', 1, dbt.current_timestamp_backcompat()) }} as date) as valid_until

from order_timezone_dt
group by 1, 2
-- We only want to apply this logic to time_zone's that had daylight saving time and it ended at a point. For example, Hong Kong ended DST in 1979.
having cast(max(daylight_end_utc) as date) < cast({{ dbt.current_timestamp_backcompat() }} as date)

), calculate_schedules as (

select
Expand All @@ -119,16 +45,16 @@ with timezone as (
schedule.end_time,
schedule.created_at,
schedule.schedule_name,
schedule.start_time - coalesce(split_timezones.offset_minutes, 0) as start_time_utc,
schedule.end_time - coalesce(split_timezones.offset_minutes, 0) as end_time_utc,
coalesce(split_timezones.offset_minutes, 0) as offset_minutes_to_add,
schedule.start_time - coalesce(timezones_with_dst.offset_minutes, 0) as start_time_utc,
schedule.end_time - coalesce(timezones_with_dst.offset_minutes, 0) as end_time_utc,
coalesce(timezones_with_dst.offset_minutes, 0) as offset_minutes_to_add,
-- we'll use these to determine which schedule version to associate tickets with
cast(split_timezones.valid_from as {{ dbt.type_timestamp() }}) as valid_from,
cast(split_timezones.valid_until as {{ dbt.type_timestamp() }}) as valid_until
cast(timezones_with_dst.valid_from as {{ dbt.type_timestamp() }}) as valid_from,
cast(timezones_with_dst.valid_until as {{ dbt.type_timestamp() }}) as valid_until

from schedule
left join split_timezones
on split_timezones.time_zone = schedule.time_zone
left join timezones_with_dst
on timezones_with_dst.time_zone = schedule.time_zone

-- Now we need take holiday's into consideration and perform the following transformations to account for Holidays in existing schedules
), holiday_start_end_times as (
Expand Down
30 changes: 28 additions & 2 deletions models/intermediate/int_zendesk__ticket_schedules.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,20 @@ with ticket as (
select *
from {{ ref('stg_zendesk__ticket_schedule') }}

), schedule as (
), schedules as (

select *
from {{ ref('stg_zendesk__schedule') }}

), timezones_with_dst as (

select *
from {{ ref('int_zendesk__timezones_with_dst') }}

), daylight_time as (

select *
from {{ var('daylight_time') }}

), default_schedule_events as (
-- Goal: understand the working schedules applied to tickets, so that we can then determine the applicable business hours/schedule.
Expand Down Expand Up @@ -79,7 +88,24 @@ with ticket as (
, {{ fivetran_utils.timestamp_add("hour", 1000, "" ~ dbt.current_timestamp_backcompat() ~ "") }} ) as schedule_invalidated_at
from schedule_events

), ticket_schedules_with_timezone_offset as (
select
ticket_schedules.ticket_id,
ticket_schedules.schedule_id,
ticket_schedules.schedule_created_at,
ticket_schedules.schedule_invalidated_at,
coalesce(timezones_with_dst.offset_minutes, 0) as offset_minutes,
timezones_with_dst.valid_from,
timezones_with_dst.valid_until
from ticket_schedules
left join schedules
on ticket_schedules.schedule_id = schedules.schedule_id
left join timezones_with_dst
on schedules.time_zone = timezones_with_dst.time_zone
and cast(ticket_schedules.schedule_created_at as date) >= cast(timezones_with_dst.valid_from as date)
and cast(ticket_schedules.schedule_created_at as date) < cast(timezones_with_dst.valid_until as date)
{{ dbt_utils.group_by(n=7) }}
)
select
*
from ticket_schedules
from ticket_schedules_with_timezone_offset
89 changes: 89 additions & 0 deletions models/intermediate/int_zendesk__timezones_with_dst.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
{{ config(enabled=var('using_schedules', True)) }}

with timezone as (

select *
from {{ var('time_zone') }}

), daylight_time as (

select *
from {{ var('daylight_time') }}

), timezone_with_dt as (

select
timezone.*,
daylight_time.daylight_start_utc,
daylight_time.daylight_end_utc,
daylight_time.daylight_offset_minutes

from timezone
left join daylight_time
on timezone.time_zone = daylight_time.time_zone

), order_timezone_dt as (

select
*,
-- will be null for timezones without any daylight savings records (and the first entry)
-- we will coalesce the first entry date with .... the X years ago
lag(daylight_end_utc, 1) over (partition by time_zone order by daylight_end_utc asc) as last_daylight_end_utc,
-- will be null for timezones without any daylight savings records (and the last entry)
-- we will coalesce the last entry date with the current date
lead(daylight_start_utc, 1) over (partition by time_zone order by daylight_start_utc asc) as next_daylight_start_utc

from timezone_with_dt

), split_timezones as (

-- standard schedule (includes timezones without DT)
-- starts: when the last Daylight Savings ended
-- ends: when the next Daylight Savings starts
select
time_zone,
standard_offset_minutes as offset_minutes,

-- last_daylight_end_utc is null for the first record of the time_zone's daylight time, or if the TZ doesn't use DT
coalesce(last_daylight_end_utc, cast('1970-01-01' as date)) as valid_from,

-- daylight_start_utc is null for timezones that don't use DT
coalesce(daylight_start_utc, cast( {{ dbt.dateadd('year', 1, dbt.current_timestamp_backcompat()) }} as date)) as valid_until

from order_timezone_dt

union all

-- DT schedule (excludes timezones without it)
-- starts: when this Daylight Savings started
-- ends: when this Daylight Savings ends
select
time_zone,
-- Pacific Time is -8h during standard time and -7h during DT
standard_offset_minutes + daylight_offset_minutes as offset_minutes,
daylight_start_utc as valid_from,
daylight_end_utc as valid_until

from order_timezone_dt
where daylight_offset_minutes is not null

union all

select
time_zone,
standard_offset_minutes as offset_minutes,

-- Get the latest daylight_end_utc time and set that as the valid_from
max(daylight_end_utc) as valid_from,

-- If the latest_daylight_end_time_utc is less than todays timestamp, that means DST has ended. Therefore, we will make the valid_until in the future.
cast( {{ dbt.dateadd('year', 1, dbt.current_timestamp_backcompat()) }} as date) as valid_until

from order_timezone_dt
group by 1, 2
-- We only want to apply this logic to time_zone's that had daylight saving time and it ended at a point. For example, Hong Kong ended DST in 1979.
having cast(max(daylight_end_utc) as date) < cast({{ dbt.current_timestamp_backcompat() }} as date)
)

select *
from split_timezones
25 changes: 15 additions & 10 deletions models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ with agent_work_time_filtered_statuses as (

select *
from {{ ref('int_zendesk__ticket_schedules') }}

-- cross schedules with work time
), ticket_status_crossed_with_schedule as (

Expand All @@ -29,6 +29,7 @@ with agent_work_time_filtered_statuses as (
agent_work_time_filtered_statuses.target,
agent_work_time_filtered_statuses.sla_policy_name,
ticket_schedules.schedule_id,
ticket_schedules.offset_minutes,

-- take the intersection of the intervals in which the status and the schedule were both active, for calculating the business minutes spent working on the ticket
greatest(valid_starting_at, schedule_created_at) as valid_starting_at,
Expand All @@ -41,6 +42,8 @@ with agent_work_time_filtered_statuses as (
from agent_work_time_filtered_statuses
left join ticket_schedules
on agent_work_time_filtered_statuses.ticket_id = ticket_schedules.ticket_id
and cast(agent_work_time_filtered_statuses.valid_starting_at as date) >= cast(ticket_schedules.valid_from as date)
and cast(agent_work_time_filtered_statuses.valid_starting_at as date) < cast(ticket_schedules.valid_until as date)
where {{ dbt.datediff(
'greatest(valid_starting_at, schedule_created_at)',
'least(valid_ending_at, schedule_invalidated_at)',
Expand All @@ -61,13 +64,14 @@ with agent_work_time_filtered_statuses as (
({{ dbt.datediff(
"cast(" ~ dbt_date.week_start('ticket_status_crossed_with_schedule.valid_starting_at','UTC') ~ "as " ~ dbt.type_timestamp() ~ ")",
"cast(ticket_status_crossed_with_schedule.valid_starting_at as " ~ dbt.type_timestamp() ~ ")",
'second') }} /60
'second') }}
/60 - offset_minutes
) as valid_starting_at_in_minutes_from_week,
({{ dbt.datediff(
'ticket_status_crossed_with_schedule.valid_starting_at',
'ticket_status_crossed_with_schedule.valid_ending_at',
'second') }} /60
) as raw_delta_in_minutes,
({{ dbt.datediff(
'ticket_status_crossed_with_schedule.valid_starting_at',
'ticket_status_crossed_with_schedule.valid_ending_at',
'second') }} /60
) as raw_delta_in_minutes,
{{ dbt_date.week_start('ticket_status_crossed_with_schedule.valid_starting_at','UTC') }} as start_week_date

from ticket_status_crossed_with_schedule
Expand Down Expand Up @@ -124,13 +128,14 @@ with agent_work_time_filtered_statuses as (
schedule.end_time_utc as schedule_end_time,
least(ticket_week_end_time_minute, schedule.end_time_utc) - greatest(weekly_period_agent_work_time.ticket_week_start_time_minute, schedule.start_time_utc) as scheduled_minutes
from weekly_period_agent_work_time
join schedule on ticket_week_start_time_minute <= schedule.end_time_utc
join schedule
on ticket_week_start_time_minute <= schedule.end_time_utc
and ticket_week_end_time_minute >= schedule.start_time_utc
and weekly_period_agent_work_time.schedule_id = schedule.schedule_id
-- this chooses the Daylight Savings Time or Standard Time version of the schedule
-- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week
and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }})
and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }})
and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as date) > cast(schedule.valid_from as date)
and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time_minute', from_date_or_timestamp='start_week_date') }} as date) < cast(schedule.valid_until as date)

), intercepted_periods_with_running_total as (

Expand Down
16 changes: 10 additions & 6 deletions models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ with requester_wait_time_filtered_statuses as (
requester_wait_time_filtered_statuses.target,
requester_wait_time_filtered_statuses.sla_policy_name,
ticket_schedules.schedule_id,
ticket_schedules.offset_minutes,

-- take the intersection of the intervals in which the status and the schedule were both active, for calculating the business minutes spent working on the ticket
greatest(valid_starting_at, schedule_created_at) as valid_starting_at,
Expand All @@ -41,6 +42,8 @@ with requester_wait_time_filtered_statuses as (
from requester_wait_time_filtered_statuses
left join ticket_schedules
on requester_wait_time_filtered_statuses.ticket_id = ticket_schedules.ticket_id
and cast(requester_wait_time_filtered_statuses.valid_starting_at as date) >= cast(ticket_schedules.valid_from as date)
and cast(requester_wait_time_filtered_statuses.valid_starting_at as date) < cast(ticket_schedules.valid_until as date)
where {{ dbt.datediff(
'greatest(valid_starting_at, schedule_created_at)',
'least(valid_ending_at, schedule_invalidated_at)',
Expand All @@ -59,10 +62,11 @@ with requester_wait_time_filtered_statuses as (
status_valid_starting_at,
status_valid_ending_at,
({{ dbt.datediff(
"cast(" ~ dbt_date.week_start('ticket_status_crossed_with_schedule.valid_starting_at','UTC') ~ "as " ~ dbt.type_timestamp() ~ ")",
"cast(ticket_status_crossed_with_schedule.valid_starting_at as " ~ dbt.type_timestamp() ~ ")",
'second') }} /60
) as valid_starting_at_in_minutes_from_week,
"cast(" ~ dbt_date.week_start('ticket_status_crossed_with_schedule.valid_starting_at','UTC') ~ "as " ~ dbt.type_timestamp() ~ ")",
"cast(ticket_status_crossed_with_schedule.valid_starting_at as " ~ dbt.type_timestamp() ~ ")",
'second') }}
/60 - offset_minutes
) as valid_starting_at_in_minutes_from_week,
({{ dbt.datediff(
'ticket_status_crossed_with_schedule.valid_starting_at',
'ticket_status_crossed_with_schedule.valid_ending_at',
Expand Down Expand Up @@ -129,8 +133,8 @@ with requester_wait_time_filtered_statuses as (
and weekly_period_requester_wait_time.schedule_id = schedule.schedule_id
-- this chooses the Daylight Savings Time or Standard Time version of the schedule
-- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week
and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }})
and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }})
and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as date) > cast(schedule.valid_from as date)
and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time_minute', from_date_or_timestamp='start_week_date') }} as date) < cast(schedule.valid_until as date)

), intercepted_periods_with_running_total as (

Expand Down