From 609693433b5e6bb73907058f83f2abdcab570543 Mon Sep 17 00:00:00 2001 From: Erika Pacheco Date: Thu, 10 Oct 2024 18:13:38 -0700 Subject: [PATCH 1/5] Add validation and move ROUND at the end of calculation to fix division by zero error on NTD Validation Service Check, and fix information on value_checks and description. [#3483] --- .../fct_ntd_rr20_service_checks.sql | 386 ++++++++++-------- 1 file changed, 222 insertions(+), 164 deletions(-) diff --git a/warehouse/models/mart/ntd_validation/fct_ntd_rr20_service_checks.sql b/warehouse/models/mart/ntd_validation/fct_ntd_rr20_service_checks.sql index c1477fa4ef..25e1d352d1 100644 --- a/warehouse/models/mart/ntd_validation/fct_ntd_rr20_service_checks.sql +++ b/warehouse/models/mart/ntd_validation/fct_ntd_rr20_service_checks.sql @@ -1,4 +1,3 @@ ---- We do identical CASE WHEN clauses in each CTE. -- The results determine 2 different column values but one can only specify 1 col/statement {% set cph_threshold = 0.3 %} {% set mpv_threshold = 0.2 %} @@ -11,319 +10,378 @@ {% set last_year = this_year - 1 %} WITH rr20f_005 as ( - select + SELECT organization, "RR20F-005: Cost per Hour change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (cph_this_year IS NULL OR cph_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN COALESCE(cph_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN (cph_last_year IS NULL OR cph_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN COALESCE(cph_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(cph_this_year) - ROUND(cph_last_year) / ROUND(cph_last_year)) >= {{cph_threshold}} THEN "Fail" + WHEN COALESCE(cph_last_year, 0) = 0 OR COALESCE(cph_this_year, 0) = 0 + THEN "No Data" + WHEN ROUND(ABS((cph_this_year - cph_last_year) / cph_last_year), 2) >= {{cph_threshold}} + THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(cph_this_year,1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(cph_last_year,1) AS STRING), - "chg = ", - CAST(ROUND((ROUND(cph_this_year,1) - ROUND(cph_last_year,1))/ABS(ROUND(cph_this_year,1)) * 100,1) AS STRING), - "%" + CONCAT("{{this_year}} = ", IF(cph_this_year IS NULL, ' ', CAST(ROUND(cph_this_year, 2) AS STRING)), + ", {{last_year}} = ", IF(cph_last_year IS NULL, ' ', CAST(ROUND(cph_last_year, 2) AS STRING)), + IF(COALESCE(cph_this_year, 0) = 0 OR COALESCE(cph_last_year, 0) = 0, + ' ', + CONCAT(", chg = ", CAST(ROUND(ABS((cph_this_year - cph_last_year) / cph_last_year) * 100, 2) AS STRING), "%") + ) ) as value_checked, - CASE WHEN (cph_this_year IS NULL OR cph_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ". Did not yet check for cost per hr.") - WHEN (cph_last_year IS NULL OR cph_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ". Did not yet check for cost per hr.") - WHEN ABS(ROUND(cph_this_year) - ROUND(cph_last_year) / ROUND(cph_last_year)) >= {{cph_threshold}} + CASE WHEN COALESCE(cph_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{this_year}}. Did not yet check for cost per hr." + WHEN COALESCE(cph_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{last_year}}. Did not yet check for cost per hr." + WHEN COALESCE(cph_last_year, 0) = 0 AND COALESCE(cph_this_year, 0) = 0 + THEN "No data for {{last_year}} and {{this_year}}." + WHEN COALESCE(cph_last_year, 0) = 0 + THEN "No data for {{last_year}}." + WHEN COALESCE(cph_this_year, 0) = 0 + THEN "No data for {{this_year}}." + WHEN ROUND(ABS((cph_this_year - cph_last_year) / cph_last_year), 2) >= {{cph_threshold}} THEN CONCAT("The cost per hour for this mode has changed from last year by > = ", - {{cph_threshold}} * 100, - "%, please provide a narrative justification.") + {{cph_threshold}} * 100, + "%, please provide a narrative justification.") ELSE "" END AS description, "" as Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_service_3ratios_wide') }} + FROM {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), rr20f_146 as ( - select + SELECT organization, "RR20F-146: Miles per Vehicle change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (mpv_this_year IS NULL OR mpv_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN COALESCE(mpv_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN (mpv_last_year IS NULL OR mpv_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN COALESCE(mpv_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(mpv_this_year) - ROUND(mpv_last_year) / ROUND(mpv_last_year)) >= {{mpv_threshold}} THEN "Fail" + WHEN COALESCE(mpv_last_year, 0) = 0 OR COALESCE(mpv_this_year, 0) = 0 + THEN "No Data" + WHEN ROUND(ABS((mpv_this_year - mpv_last_year) / mpv_last_year), 2) >= {{mpv_threshold}} + THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(mpv_this_year,1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(mpv_last_year,1) AS STRING), - "chg = ", - CAST(ROUND((ROUND(mpv_this_year,1) - ROUND(mpv_last_year,1))/ABS(ROUND(mpv_this_year,1)) * 100,1) AS STRING), - "%" + CONCAT("{{this_year}} = ", IF(mpv_this_year IS NULL, ' ', CAST(ROUND(mpv_this_year, 2) AS STRING)), + ", {{last_year}} = ", IF(mpv_last_year IS NULL, ' ', CAST(ROUND(mpv_last_year, 2) AS STRING)), + IF(COALESCE(mpv_this_year, 0) = 0 OR COALESCE(mpv_last_year, 0) = 0, + ' ', + CONCAT(", chg = ", CAST(ROUND(ABS((mpv_this_year - mpv_last_year) / mpv_last_year) * 100, 2) AS STRING), "%") + ) ) as value_checked, - CASE WHEN (mpv_this_year IS NULL OR mpv_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ". Did not yet check for miles per vehicle.") - WHEN (mpv_last_year IS NULL OR mpv_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ". Did not yet check for miles per vehicle.") - WHEN ABS(ROUND(mpv_this_year) - ROUND(mpv_last_year) / ROUND(mpv_last_year)) >= {{mpv_threshold}} + CASE WHEN COALESCE(mpv_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{this_year}}. Did not yet check for miles per vehicle." + WHEN COALESCE(mpv_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{last_year}}. Did not yet check for miles per vehicle." + WHEN COALESCE(mpv_last_year, 0) = 0 AND COALESCE(mpv_this_year, 0) = 0 + THEN "No data for {{last_year}} and {{this_year}}." + WHEN COALESCE(mpv_last_year, 0) = 0 + THEN "No data for {{last_year}}." + WHEN COALESCE(mpv_this_year, 0) = 0 + THEN "No data for {{this_year}}." + WHEN ROUND(ABS((mpv_this_year - mpv_last_year) / mpv_last_year), 2) >= {{mpv_threshold}} THEN CONCAT("The miles per vehicle for this mode has changed from last year by >= ", - {{mpv_threshold}} * 100, - "%, please provide a narrative justification.") + {{mpv_threshold}} * 100, + "%, please provide a narrative justification.") ELSE "" END AS description, "" as Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_service_3ratios_wide') }} + FROM {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), rr20f_179 as ( - select + SELECT organization, "RR20F-179: Missing Service Data check" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN ((vrm_this_year IS NULL OR vrm_this_year = 0) OR (vrh_this_year IS NULL OR vrh_this_year = 0) OR - (upt_this_year IS NULL OR upt_this_year = 0) OR (voms_this_year IS NULL OR voms_this_year = 0)) - AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Fail" - WHEN ((vrm_last_year IS NULL OR vrm_last_year = 0) OR (vrh_last_year IS NULL OR vrh_last_year = 0) OR - (upt_last_year IS NULL OR upt_last_year = 0) OR (voms_last_year IS NULL OR voms_last_year = 0)) - AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Fail" - WHEN ((vrm_this_year IS NULL OR vrm_this_year = 0) OR (vrh_this_year IS NULL OR vrh_this_year = 0) OR - (upt_this_year IS NULL OR upt_this_year = 0) OR (voms_this_year IS NULL OR voms_this_year = 0)) - AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Didn't run" - WHEN ((vrm_last_year IS NULL OR vrm_last_year = 0) OR (vrh_last_year IS NULL OR vrh_last_year = 0) OR - (upt_last_year IS NULL OR upt_last_year = 0) OR (voms_last_year IS NULL OR voms_last_year = 0)) - AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Didn't run" + CASE WHEN (COALESCE(vrm_this_year, 0) = 0 OR COALESCE(vrh_this_year, 0) = 0 OR + COALESCE(upt_this_year, 0) = 0 OR COALESCE(voms_this_year, 0) = 0) + AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST("{{this_year}}-10-31" AS DATE) + THEN "Fail" + WHEN (COALESCE(vrm_last_year, 0) = 0 OR COALESCE(vrh_last_year, 0) = 0 OR + COALESCE(upt_last_year, 0) = 0 OR COALESCE(voms_last_year, 0) = 0) + AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST("{{last_year}}-10-31" AS DATE) + THEN "Fail" + WHEN (COALESCE(vrm_this_year, 0) = 0 OR COALESCE(vrh_this_year, 0) = 0 OR + COALESCE(upt_this_year, 0) = 0 OR COALESCE(voms_this_year, 0) = 0) + AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "Didn't run" + WHEN (COALESCE(vrm_last_year, 0) = 0 OR COALESCE(vrh_last_year, 0) = 0 OR + COALESCE(upt_last_year, 0) = 0 OR COALESCE(voms_last_year, 0) = 0) + AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "Didn't run" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " Service data: VRM=", vrm_this_year, " VRH=", vrh_this_year, " UPT=", upt_this_year, - " VOMS=", voms_this_year) as value_checked, - CASE WHEN ((vrm_this_year IS NULL OR vrm_this_year = 0) OR (vrh_this_year IS NULL OR vrh_this_year = 0) OR - (upt_this_year IS NULL OR upt_this_year = 0) OR (voms_this_year IS NULL OR voms_this_year = 0)) - AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CONCAT("{{this_year}} Service data: VRM=", COALESCE(CAST(vrm_this_year AS STRING), ' '), + " VRH=", COALESCE(CAST(vrh_this_year AS STRING), ' '), + " UPT=", COALESCE(CAST(upt_this_year AS STRING), ' '), + " VOMS=", COALESCE(CAST(voms_this_year AS STRING), ' ')) as value_checked, + CASE WHEN (COALESCE(vrm_this_year, 0) = 0 OR COALESCE(vrh_this_year, 0) = 0 OR + COALESCE(upt_this_year, 0) = 0 OR COALESCE(voms_this_year, 0) = 0) + AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST("{{this_year}}-10-31" AS DATE) THEN "One or more service data fields are are missing for Vehicle Revenue Miles, Vehicle Revenue Hours, Unlinked Passenger Trips, or VOMS. Please revise your submission to report it if the transit unit operated revenue service during the fiscal year." - WHEN ((vrm_last_year IS NULL OR vrm_last_year = 0) OR (vrh_last_year IS NULL OR vrh_last_year = 0) OR - (upt_last_year IS NULL OR upt_last_year = 0) OR (voms_last_year IS NULL OR voms_last_year = 0)) - AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (COALESCE(vrm_last_year, 0) = 0 OR COALESCE(vrh_last_year, 0) = 0 OR + COALESCE(upt_last_year, 0) = 0 OR COALESCE(voms_last_year, 0) = 0) + AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST("{{last_year}}-10-31" AS DATE) THEN "One or more service data fields are are missing for Vehicle Revenue Miles, Vehicle Revenue Hours, Unlinked Passenger Trips, or VOMS. Please revise your submission to report it if the transit unit operated revenue service during the fiscal year." - WHEN ((vrm_this_year IS NULL OR vrm_this_year = 0) OR (vrh_this_year IS NULL OR vrh_this_year = 0) OR - (upt_this_year IS NULL OR upt_this_year = 0) OR (voms_this_year IS NULL OR voms_this_year = 0)) - AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ". Did not yet check for missing service data.") - WHEN ((vrm_last_year IS NULL OR vrm_last_year = 0) OR (vrh_last_year IS NULL OR vrh_last_year = 0) OR - (upt_last_year IS NULL OR upt_last_year = 0) OR (voms_last_year IS NULL OR voms_last_year = 0)) - AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ". Did not yet check for missing service data.") + WHEN (COALESCE(vrm_this_year, 0) = 0 OR COALESCE(vrh_this_year, 0) = 0 OR + COALESCE(upt_this_year, 0) = 0 OR COALESCE(voms_this_year, 0) = 0) + AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{this_year}}. Did not yet check for missing service data." + WHEN (COALESCE(vrm_last_year, 0) = 0 OR COALESCE(vrh_last_year, 0) = 0 OR + COALESCE(upt_last_year, 0) = 0 OR COALESCE(voms_last_year, 0) = 0) + AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{last_year}}. Did not yet check for missing service data." ELSE "Pass" END as description, "" as Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_service_3ratios_wide') }} + FROM {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), rr20f_139 as ( - select + SELECT organization, "RR20F-139: Vehicle Revenue Miles (VRM) % change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (vrm_this_year IS NULL OR vrm_this_year = 0 ) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (vrm_last_year IS NULL OR vrm_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(vrm_this_year) - ROUND(vrm_last_year) / ROUND(vrm_last_year)) >= {{vrm_threshold}} THEN "Fail" + CASE WHEN COALESCE(vrm_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "Did Not Run" + WHEN COALESCE(vrm_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "Did Not Run" + WHEN COALESCE(vrm_this_year, 0) = 0 OR COALESCE(vrm_last_year, 0) = 0 + THEN "No Data" + WHEN ROUND(ABS((vrm_this_year - vrm_last_year) / vrm_last_year), 2) >= {{vrm_threshold}} + THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(vrm_this_year,1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(vrm_last_year,1) AS STRING), - "chg = ", - CAST(ROUND((ROUND(vrm_this_year,1) - ROUND(vrm_last_year,1))/ABS(ROUND(vrm_this_year,1)) * 100,1) AS STRING), - "%" + CONCAT("{{this_year}} = ", IF(vrm_this_year IS NULL, ' ', CAST(ROUND(vrm_this_year, 2) AS STRING)), + ", {{last_year}} = ", IF(vrm_last_year IS NULL, ' ', CAST(ROUND(vrm_last_year, 2) AS STRING)), + IF(COALESCE(vrm_this_year, 0) = 0 OR COALESCE(vrm_last_year, 0) = 0, + ' ', + CONCAT(", chg = ", CAST(ROUND(ABS((vrm_this_year - vrm_last_year) / vrm_last_year) * 100, 2) AS STRING), "%") + ) ) as value_checked, - CASE WHEN (vrm_this_year IS NULL OR vrm_this_year = 0 ) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ".") - WHEN (vrm_last_year IS NULL OR vrm_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ".") - WHEN ABS(ROUND(vrm_this_year) - ROUND(vrm_last_year) / ROUND(vrm_last_year)) >= {{vrm_threshold}} + CASE WHEN COALESCE(vrm_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{this_year}}." + WHEN COALESCE(vrm_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{last_year}}." + WHEN COALESCE(vrm_last_year, 0) = 0 AND COALESCE(vrm_this_year, 0) = 0 + THEN "No data for {{last_year}} and {{this_year}}." + WHEN COALESCE(vrm_last_year, 0) = 0 + THEN "No data for {{last_year}}." + WHEN COALESCE(vrm_this_year, 0) = 0 + THEN "No data for {{this_year}}." + WHEN ROUND(ABS((vrm_this_year - vrm_last_year) / vrm_last_year), 2) >= {{vrm_threshold}} THEN CONCAT("The annual vehicle revenue miles for this mode has changed from last year by >= ", - {{vrm_threshold}} * 100, - "%, please provide a narrative justification.") + {{vrm_threshold}} * 100, + "%, please provide a narrative justification.") ELSE "" END AS description, "" as Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_service_3ratios_wide') }} + FROM {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), rr20f_008 as ( - select + SELECT organization, "RR20F-139: Fare Revenue Per Trip change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (frpt_this_year IS NULL OR frpt_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (frpt_last_year IS NULL OR frpt_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(frpt_this_year) - ROUND(frpt_last_year) / ROUND(frpt_last_year)) >= {{frpt_threshold}} THEN "Fail" + CASE WHEN COALESCE(frpt_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "Did Not Run" + WHEN COALESCE(frpt_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "Did Not Run" + WHEN COALESCE(frpt_this_year, 0) = 0 OR COALESCE(frpt_last_year, 0) = 0 + THEN "No Data" + WHEN ROUND(ABS((frpt_this_year - frpt_last_year) / frpt_last_year), 2) >= {{frpt_threshold}} + THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(frpt_this_year,1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(frpt_last_year,1) AS STRING), - "chg = ", - CAST(ROUND((ROUND(frpt_this_year,1) - ROUND(frpt_last_year,1))/ABS(ROUND(frpt_this_year,1)) * 100,1) AS STRING), - "%" + CONCAT("{{this_year}} = ", IF(frpt_this_year IS NULL, ' ', CAST(ROUND(frpt_this_year, 2) AS STRING)), + ", {{last_year}} = ", IF(frpt_last_year IS NULL, ' ', CAST(ROUND(frpt_last_year, 2) AS STRING)), + IF(COALESCE(frpt_this_year, 0) = 0 OR COALESCE(frpt_last_year, 0) = 0, + ' ', + CONCAT(", chg = ", CAST(ROUND(ABS((frpt_this_year - frpt_last_year) / frpt_last_year) * 100, 2) AS STRING), "%") + ) ) as value_checked, - CASE WHEN (frpt_this_year IS NULL OR frpt_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ".") - WHEN (frpt_last_year IS NULL OR frpt_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ".") - WHEN ABS(ROUND(frpt_this_year) - ROUND(frpt_last_year) / ROUND(frpt_last_year)) >= {{frpt_threshold}} - THEN CONCAT("The fare revenues per unlinked passenger trip for this mode has changed from last year by >= ", - {{frpt_threshold}} * 100, - "%, please provide a narrative justification.") + CASE WHEN COALESCE(frpt_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{this_year}}." + WHEN COALESCE(frpt_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{last_year}}." + WHEN COALESCE(frpt_last_year, 0) = 0 AND COALESCE(frpt_this_year, 0) = 0 + THEN "No data for {{last_year}} and {{this_year}}." + WHEN COALESCE(frpt_last_year, 0) = 0 + THEN "No data for {{last_year}}." + WHEN COALESCE(frpt_this_year, 0) = 0 + THEN "No data for {{this_year}}." + WHEN ROUND(ABS((frpt_this_year - frpt_last_year) / frpt_last_year), 2) >= {{frpt_threshold}} + THEN CONCAT("The fare revenues per unlinked passenger trip for this mode has changed from last year by >= ", + {{frpt_threshold}} * 100, + "%, please provide a narrative justification.") ELSE "" END AS description, "" as Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_service_3ratios_wide') }} + FROM {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), rr20f_137 as ( - select + SELECT organization, "RR20F-139: Revenue Speed change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (rev_speed_this_year IS NULL OR rev_speed_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN COALESCE(rev_speed_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN (rev_speed_last_year IS NULL OR rev_speed_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN COALESCE(rev_speed_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(rev_speed_this_year) - ROUND(rev_speed_last_year) / ROUND(rev_speed_last_year)) >= {{rs_threshold}} THEN "Fail" + WHEN ROUND(ABS((rev_speed_this_year - rev_speed_last_year) / rev_speed_last_year), 2) >= {{rs_threshold}} + THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(rev_speed_this_year,1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(rev_speed_last_year,1) AS STRING), - "chg = ", - CAST(ROUND(ROUND(rev_speed_this_year) - ROUND(rev_speed_last_year) / ROUND(rev_speed_last_year)*100, 1) AS STRING), - "%" + CONCAT("{{this_year}} = ", IF(rev_speed_this_year IS NULL, ' ', CAST(ROUND(rev_speed_this_year, 2) AS STRING)), + ", {{last_year}} = ", IF(rev_speed_last_year IS NULL, ' ', CAST(ROUND(rev_speed_last_year, 2) AS STRING)), + IF(COALESCE(rev_speed_this_year, 0) = 0 OR COALESCE(rev_speed_last_year, 0) = 0, + ' ', + CONCAT(", chg = ", CAST(ROUND(ABS((rev_speed_this_year - rev_speed_last_year) / rev_speed_last_year) * 100, 2) AS STRING), "%") + ) ) as value_checked, - CASE WHEN (rev_speed_this_year IS NULL OR rev_speed_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ".") - WHEN (rev_speed_last_year IS NULL OR rev_speed_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ".") - WHEN ABS(ROUND(rev_speed_this_year) - ROUND(rev_speed_last_year) / ROUND(rev_speed_last_year)) >= {{rs_threshold}} + CASE WHEN COALESCE(rev_speed_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{this_year}}." + WHEN COALESCE(rev_speed_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{last_year}}." + WHEN COALESCE(rev_speed_last_year, 0) = 0 AND COALESCE(rev_speed_this_year, 0) = 0 + THEN "No data for {{last_year}} and {{this_year}}." + WHEN COALESCE(rev_speed_last_year, 0) = 0 + THEN "No data for {{last_year}}." + WHEN COALESCE(rev_speed_this_year, 0) = 0 + THEN "No data for {{this_year}}." + WHEN ROUND(ABS((rev_speed_this_year - rev_speed_last_year) / rev_speed_last_year), 2) >= {{rs_threshold}} THEN CONCAT("The revenue speed, the avg speed of your vehicles while in revenue service, for this mode has changed from last year by >= ", - {{rs_threshold}} * 100, - "%, please provide a narrative justification.") + {{rs_threshold}} * 100, + "%, please provide a narrative justification.") ELSE "" END AS description, "" as Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_service_3ratios_wide') }} + FROM {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), rr20f_154 as ( - select + SELECT organization, "RR20F-154: Trips per Hour change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (tph_this_year IS NULL OR tph_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN COALESCE(tph_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN (tph_last_year IS NULL OR tph_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN COALESCE(tph_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(tph_this_year) - ROUND(tph_last_year) / ROUND(tph_last_year)) >= {{tph_threshold}} + WHEN COALESCE(tph_this_year, 0) = 0 OR COALESCE(tph_last_year, 0) = 0 + THEN "No Data" + WHEN ROUND(ABS((tph_this_year - tph_last_year) / tph_last_year), 2) >= {{tph_threshold}} THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(tph_this_year,1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(tph_last_year,1) AS STRING), - "chg = ", - CAST(ROUND((ROUND(tph_this_year,1) - ROUND(tph_last_year,1))/ABS(ROUND(tph_this_year,1)) * 100,1) AS STRING), - "%" + CONCAT("{{this_year}} = ", IF(tph_this_year IS NULL, ' ', CAST(ROUND(tph_this_year, 2) AS STRING)), + ", {{last_year}} = ", IF(tph_last_year IS NULL, ' ', CAST(ROUND(tph_last_year, 2) AS STRING)), + IF(COALESCE(tph_this_year, 0) = 0 OR COALESCE(tph_last_year, 0) = 0, + ' ', + CONCAT(", chg = ", CAST(ROUND(ABS((tph_this_year - tph_last_year) / tph_last_year) * 100, 2) AS STRING), "%") + ) ) as value_checked, - CASE WHEN (tph_this_year IS NULL OR tph_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ".") - WHEN (tph_last_year IS NULL OR tph_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ".") - WHEN ABS(ROUND(tph_this_year) - ROUND(tph_last_year) / ROUND(tph_last_year)) >= {{tph_threshold}} + CASE WHEN COALESCE(tph_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "No data but this check ran before the NTD submission due date in {{this_year}}." + WHEN COALESCE(tph_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "No data but this check ran before the NTD submission due date in {{last_year}}." + WHEN tph_last_year IS NULL OR tph_last_year = 0 + THEN "No data for {{last_year}}." + WHEN tph_this_year IS NULL OR tph_this_year = 0 + THEN "No data for {{this_year}}." + WHEN ROUND(ABS((tph_this_year - tph_last_year) / tph_last_year), 2) >= {{tph_threshold}} THEN CONCAT("The calculated trips per hour for this mode has changed from last year by >= ", - {{tph_threshold}} * 100, - "%, please provide a narrative justification.") + {{tph_threshold}} * 100, + "%, please provide a narrative justification.") ELSE "" END AS description, "" as Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_service_3ratios_wide') }} + FROM {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), rr20f_171 as ( - select + SELECT organization, "RR20F-171: Vehicles of Maximum Service (VOMS) change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (ROUND(voms_this_year) =0 AND ROUND(voms_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) + CASE WHEN (ROUND(voms_this_year) =0 AND ROUND(voms_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST("{{this_year}}-10-31" AS DATE)) OR (ROUND(voms_this_year) !=0 AND ROUND(voms_this_year) IS NOT NULL AND ROUND(voms_last_year) = 0) THEN "Fail" - WHEN (voms_this_year IS NULL OR voms_this_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + WHEN COALESCE(voms_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN (voms_last_year IS NULL OR voms_last_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN COALESCE(voms_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) THEN "Did Not Run" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(voms_this_year) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(voms_last_year) AS STRING), - "chg = ", - CAST(ROUND((ROUND(voms_this_year,1) - ROUND(voms_last_year,1))/ABS(ROUND(voms_this_year,1)) * 100,1) AS STRING), - "%" + CONCAT("{{this_year}} = ", IF(voms_this_year IS NULL, ' ', CAST(ROUND(voms_this_year, 2) AS STRING)), + ", {{last_year}} = ", IF(voms_last_year IS NULL, ' ', CAST(ROUND(voms_last_year, 2) AS STRING)) ) as value_checked, - CASE WHEN (ROUND(voms_this_year) =0 AND ROUND(voms_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) + CASE WHEN (ROUND(voms_this_year) =0 AND ROUND(voms_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST("{{this_year}}-10-31" AS DATE)) OR (ROUND(voms_this_year) !=0 AND ROUND(voms_this_year) IS NOT NULL AND ROUND(voms_last_year) = 0) THEN "The Vehicles of Maximum Service (VOMS) for this mode has changed either to or from 0 compared to last year, please provide a narrative justification." - WHEN (voms_this_year IS NULL OR voms_this_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, " for VOMS.") - WHEN (voms_last_year IS NULL OR voms_last_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, " for VOMS.") + WHEN COALESCE(voms_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{this_year}} for VOMS." + WHEN COALESCE(voms_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{last_year}} for VOMS." ELSE "" END AS description, "" as Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_service_3ratios_wide') }} + FROM {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), rr20f_143 as ( - select + SELECT organization, "RR20F-143: Vehicle Revenue Miles (VRM) change from zero" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (ROUND(vrm_this_year) =0 AND ROUND(vrm_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) + CASE WHEN (ROUND(vrm_this_year) =0 AND ROUND(vrm_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST("{{this_year}}-10-31" AS DATE)) OR (ROUND(vrm_this_year) !=0 AND ROUND(vrm_this_year) IS NOT NULL AND ROUND(voms_last_year) = 0) THEN "Fail" - WHEN (vrm_this_year IS NULL OR vrm_this_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + WHEN COALESCE(vrm_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) THEN "Did Not Run" - WHEN (vrm_last_year IS NULL OR vrm_last_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN COALESCE(vrm_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) THEN "Did Not Run" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(vrm_this_year) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(vrm_last_year) AS STRING), - "chg = ", - CAST(ROUND((ROUND(vrm_this_year,1) - ROUND(vrm_last_year,1))/ABS(ROUND(vrm_this_year,1)) * 100,1) AS STRING), - "%" + CONCAT("{{this_year}} = ", IF(vrm_this_year IS NULL, ' ', CAST(ROUND(vrm_this_year, 2) AS STRING)), + ", {{last_year}} = ", IF(vrm_last_year IS NULL, ' ', CAST(ROUND(vrm_last_year, 2) AS STRING)) ) as value_checked, - CASE WHEN (ROUND(vrm_this_year) =0 AND ROUND(vrm_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) + CASE WHEN (ROUND(vrm_this_year) =0 AND ROUND(vrm_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST("{{this_year}}-10-31" AS DATE)) OR (ROUND(vrm_this_year) !=0 AND ROUND(vrm_this_year) IS NOT NULL AND ROUND(vrm_last_year) = 0) THEN "The Vehicle Revenue Miles (VRM) for this mode has changed either to or from 0 compared to last year, please provide a narrative justification." - WHEN (vrm_this_year IS NULL OR vrm_this_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, " for VOMS.") - WHEN (vrm_last_year IS NULL OR vrm_last_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) - THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, " for VOMS.") + WHEN COALESCE(vrm_this_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{this_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{this_year}} for VOMS." + WHEN COALESCE(vrm_last_year, 0) = 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST("{{last_year}}-10-31" AS DATE) + THEN "No data but this check was run before the NTD submission due date in {{last_year}} for VOMS." ELSE "" END AS description, "" as Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_service_3ratios_wide') }} + FROM {{ ref('int_ntd_rr20_service_3ratios_wide') }} ) From 2bed915c224c42bb8c19114ce086f3048ebe3908 Mon Sep 17 00:00:00 2001 From: Erika Pacheco Date: Mon, 14 Oct 2024 11:12:00 -0700 Subject: [PATCH 2/5] Add column year of the data and fix missing information on value_checked column for NTD Validation VOMS and Vins Checks [#3483] --- .../ntd_validation/int_ntd_a30_voms_vins_totals.sql | 5 ++++- .../models/mart/ntd_validation/fct_ntd_a30_vomscheck.sql | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_a30_voms_vins_totals.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_a30_voms_vins_totals.sql index efd4f6df41..8f8a22682a 100644 --- a/warehouse/models/intermediate/ntd_validation/int_ntd_a30_voms_vins_totals.sql +++ b/warehouse/models/intermediate/ntd_validation/int_ntd_a30_voms_vins_totals.sql @@ -18,7 +18,10 @@ vins_a30 as ( GROUP BY organization, fiscal_year ) -select voms_rr20.*, vins_a30.a30_vin_n +SELECT COALESCE(voms_rr20.organization, vins_a30.organization) as organization, + COALESCE(voms_rr20.fiscal_year, vins_a30.fiscal_year) as fiscal_year, + voms_rr20.rr20_voms, + vins_a30.a30_vin_n FROM voms_rr20 FULL OUTER JOIN vins_a30 ON voms_rr20.organization = vins_a30.organization diff --git a/warehouse/models/mart/ntd_validation/fct_ntd_a30_vomscheck.sql b/warehouse/models/mart/ntd_validation/fct_ntd_a30_vomscheck.sql index af06559cee..123a4c2a9a 100644 --- a/warehouse/models/mart/ntd_validation/fct_ntd_a30_vomscheck.sql +++ b/warehouse/models/mart/ntd_validation/fct_ntd_a30_vomscheck.sql @@ -4,12 +4,14 @@ WITH rr20f_180 as ( SELECT organization, "RR20F-180: VOMS across forms" as name_of_check, + fiscal_year as year_of_data, CASE WHEN ROUND(rr20_voms, 1) > ROUND(a30_vin_n, 1) THEN "Fail" ELSE "Pass" END as check_status, - CONCAT("RR-20 VOMS = ", CAST(ROUND(rr20_voms, 1) AS STRING), - "# A-30 VINs = ", CAST(ROUND(a30_vin_n, 1) AS STRING)) AS value_checked, + CONCAT("RR-20 VOMS = ", IF(rr20_voms IS NULL, ' ', CAST(ROUND(rr20_voms, 1) AS STRING)), + ", A-30 VINs = ", IF(a30_vin_n IS NULL, ' ', CAST(ROUND(a30_vin_n, 1) AS STRING)) + ) AS value_checked, CASE WHEN ROUND(rr20_voms, 1) > ROUND(a30_vin_n, 1) THEN "Total VOMS is greater than total A-30 vehicles reported. Please clarify." ELSE "VOMS & A-30 vehicles reported are equal to and/or lower than active inventory." From fea8f930075257fb55489eeb8542692fbbba19f1 Mon Sep 17 00:00:00 2001 From: Erika Pacheco Date: Mon, 14 Oct 2024 16:15:02 -0700 Subject: [PATCH 3/5] Add new columns on external table All_NTDReports [#3483] --- .../external_table_all_ntdreports.yml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/airflow/dags/create_external_tables/ntd_report_validation/external_table_all_ntdreports.yml b/airflow/dags/create_external_tables/ntd_report_validation/external_table_all_ntdreports.yml index 748cbe1279..88675d4c4a 100644 --- a/airflow/dags/create_external_tables/ntd_report_validation/external_table_all_ntdreports.yml +++ b/airflow/dags/create_external_tables/ntd_report_validation/external_table_all_ntdreports.yml @@ -73,6 +73,8 @@ schema_fields: type: INTEGER - name: Size type: STRING + - name: Type + type: STRING - name: DOTCapitalResponsibility type: FLOAT - name: OrganizationCapitalResponsibility @@ -95,6 +97,10 @@ schema_fields: type: STRING - name: PrivateMode type: STRING + - name: Note + type: STRING + - name: LastModifiedDate + type: TIMESTAMP - name: ntdassetandresourceinfo_data type: RECORD mode: REPEATED @@ -139,6 +145,44 @@ schema_fields: type: STRING - name: LastModifiedDate type: TIMESTAMP + - name: AgencyFleetId + type: STRING + - name: TotalVehicles + type: INTEGER + - name: ActiveVehicles + type: INTEGER + - name: DedicatedFleet + type: BOOLEAN + - name: NoCapitalReplacementResponsibility + type: BOOLEAN + - name: AutomatedorAutonomousVehicles + type: STRING + - name: Manufacturer + type: STRING + - name: DescribeOtherManufacturer + type: STRING + - name: Model + type: STRING + - name: YearRebuilt + type: INTEGER + - name: OtherFuelType + type: STRING + - name: DuelFuelType + type: STRING + - name: StandingCapacity + type: INTEGER + - name: OtherOwnershipType + type: STRING + - name: EmergencyVehicles + type: BOOLEAN + - name: TypeofLastRenewal + type: STRING + - name: UsefulLifeBenchmark + type: INTEGER + - name: MilesThisYear + type: INTEGER + - name: AverageLifetimeMilesPerActiveVehicle + type: INTEGER - name: ntdreportingp10_data type: RECORD mode: REPEATED @@ -320,14 +364,24 @@ schema_fields: type: STRING - name: AnnualVehicleRevMiles type: INTEGER + - name: AnnualVehicleRevMilesComments + type: STRING - name: AnnualVehicleRevHours type: INTEGER + - name: AnnualVehicleRevHoursComments + type: STRING - name: AnnualUnlinkedPassTrips type: INTEGER + - name: AnnualUnlinkedPassTripsComments + type: STRING - name: AnnualVehicleMaxService type: INTEGER + - name: AnnualVehicleMaxServiceComments + type: STRING - name: SponsoredServiceUPT type: INTEGER + - name: SponsoredServiceUPTComments + type: STRING - name: Quantity type: INTEGER - name: LastModifiedDate @@ -372,6 +426,8 @@ schema_fields: type: STRING - name: VehiclesToBeRetiredBeyondULB type: INTEGER + - name: VehiclesToBePurchasesNextYear + type: INTEGER - name: VehiclesPastULBInTAM type: INTEGER - name: LastModifiedDate From 79a41befdacd9407ac6392f207567923a183ebf7 Mon Sep 17 00:00:00 2001 From: V <156962295+vevetron@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:48:10 -0700 Subject: [PATCH 4/5] fix(gtfs_rt_service_alerts): Changes datatype of active_period to strings to accomodate some bad data (#3503) Co-authored-by: V --- .../gtfs_rt_v2/rt_v2_service_alerts.yml | 4 ++-- .../gtfs/int_gtfs_rt__service_alerts_fully_unnested.sql | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airflow/dags/create_external_tables/gtfs_rt_v2/rt_v2_service_alerts.yml b/airflow/dags/create_external_tables/gtfs_rt_v2/rt_v2_service_alerts.yml index 92c85bf696..a84b648f56 100644 --- a/airflow/dags/create_external_tables/gtfs_rt_v2/rt_v2_service_alerts.yml +++ b/airflow/dags/create_external_tables/gtfs_rt_v2/rt_v2_service_alerts.yml @@ -50,9 +50,9 @@ schema_fields: mode: REPEATED fields: - name: start - type: INTEGER + type: STRING - name: end - type: INTEGER + type: STRING - name: informedEntity type: RECORD diff --git a/warehouse/models/intermediate/gtfs/int_gtfs_rt__service_alerts_fully_unnested.sql b/warehouse/models/intermediate/gtfs/int_gtfs_rt__service_alerts_fully_unnested.sql index db61e6b21e..30d324bdee 100644 --- a/warehouse/models/intermediate/gtfs/int_gtfs_rt__service_alerts_fully_unnested.sql +++ b/warehouse/models/intermediate/gtfs/int_gtfs_rt__service_alerts_fully_unnested.sql @@ -28,9 +28,9 @@ int_gtfs_rt__service_alerts_fully_unnested AS ( cause, effect, - -- active periods - unnested_active_period.start AS active_period_start, - unnested_active_period.end AS active_period_end, + -- active periods, converting from STRINGS since some agency has bad data that insn't unnestable as INTs + SAFE_CAST(unnested_active_period.start AS INT) AS active_period_start, + SAFE_CAST(unnested_active_period.end AS INT) AS active_period_end, -- informted entities unnested_informed_entity.agencyId AS agency_id, From f25f237f063b158cd47b4a0baf498ecb9b0a5331 Mon Sep 17 00:00:00 2001 From: V <156962295+vevetron@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:51:09 -0700 Subject: [PATCH 5/5] fix(external_airtable_services): Remove a field holiday_website__from_provider_ that isn't parsing correctly (#3502) Co-authored-by: V --- .../airtable/external_airtable_california_transit_services.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/airflow/dags/create_external_tables/airtable/external_airtable_california_transit_services.yml b/airflow/dags/create_external_tables/airtable/external_airtable_california_transit_services.yml index 5259c8480f..150ada4706 100644 --- a/airflow/dags/create_external_tables/airtable/external_airtable_california_transit_services.yml +++ b/airflow/dags/create_external_tables/airtable/external_airtable_california_transit_services.yml @@ -288,9 +288,6 @@ schema_fields: - name: public_currently_operating_fixed_route type: STRING mode: NULLABLE - - name: holiday_website__from_provider_ - type: STRING - mode: NULLABLE - name: holiday_schedule___veterans_day__observed_ type: STRING mode: NULLABLE