Merge branch 'main' into juypyterhub_3499

cal-itp · Oct 16, 2024 · 6c3acd0 · 6c3acd0
2 parents 7e1f427 + f25f237
commit 6c3acd0
Show file tree

Hide file tree

Showing 7 changed files with 291 additions and 175 deletions.
diff --git a/...ow/dags/create_external_tables/airtable/external_airtable_california_transit_services.yml b/...ow/dags/create_external_tables/airtable/external_airtable_california_transit_services.yml
@@ -288,9 +288,6 @@ schema_fields:
   - name: public_currently_operating_fixed_route
     type: STRING
     mode: NULLABLE
-  - name: holiday_website__from_provider_
-    type: STRING
-    mode: NULLABLE
   - name: holiday_schedule___veterans_day__observed_
     type: STRING
     mode: NULLABLE

diff --git a/airflow/dags/create_external_tables/gtfs_rt_v2/rt_v2_service_alerts.yml b/airflow/dags/create_external_tables/gtfs_rt_v2/rt_v2_service_alerts.yml
@@ -50,9 +50,9 @@ schema_fields:
         mode: REPEATED
         fields:
           - name: start
-            type: INTEGER
+            type: STRING
           - name: end
-            type: INTEGER
+            type: STRING
 
       - name: informedEntity
         type: RECORD

diff --git a/airflow/dags/create_external_tables/ntd_report_validation/external_table_all_ntdreports.yml b/airflow/dags/create_external_tables/ntd_report_validation/external_table_all_ntdreports.yml
@@ -73,6 +73,8 @@ schema_fields:
       type: INTEGER
     - name: Size
       type: STRING
+    - name: Type
+      type: STRING
     - name: DOTCapitalResponsibility
       type: FLOAT
     - name: OrganizationCapitalResponsibility
@@ -95,6 +97,10 @@ schema_fields:
       type: STRING
     - name: PrivateMode
       type: STRING
+    - name: Note
+      type: STRING
+    - name: LastModifiedDate
+      type: TIMESTAMP
   - name: ntdassetandresourceinfo_data
     type: RECORD
     mode: REPEATED
@@ -139,6 +145,44 @@ schema_fields:
       type: STRING
     - name: LastModifiedDate
       type: TIMESTAMP
+    - name: AgencyFleetId
+      type: STRING
+    - name: TotalVehicles
+      type: INTEGER
+    - name: ActiveVehicles
+      type: INTEGER
+    - name: DedicatedFleet
+      type: BOOLEAN
+    - name: NoCapitalReplacementResponsibility
+      type: BOOLEAN
+    - name: AutomatedorAutonomousVehicles
+      type: STRING
+    - name: Manufacturer
+      type: STRING
+    - name: DescribeOtherManufacturer
+      type: STRING
+    - name: Model
+      type: STRING
+    - name: YearRebuilt
+      type: INTEGER
+    - name: OtherFuelType
+      type: STRING
+    - name: DuelFuelType
+      type: STRING
+    - name: StandingCapacity
+      type: INTEGER
+    - name: OtherOwnershipType
+      type: STRING
+    - name: EmergencyVehicles
+      type: BOOLEAN
+    - name: TypeofLastRenewal
+      type: STRING
+    - name: UsefulLifeBenchmark
+      type: INTEGER
+    - name: MilesThisYear
+      type: INTEGER
+    - name: AverageLifetimeMilesPerActiveVehicle
+      type: INTEGER
   - name: ntdreportingp10_data
     type: RECORD
     mode: REPEATED
@@ -320,14 +364,24 @@ schema_fields:
       type: STRING
     - name: AnnualVehicleRevMiles
       type: INTEGER
+    - name: AnnualVehicleRevMilesComments
+      type: STRING
     - name: AnnualVehicleRevHours
       type: INTEGER
+    - name: AnnualVehicleRevHoursComments
+      type: STRING
     - name: AnnualUnlinkedPassTrips
       type: INTEGER
+    - name: AnnualUnlinkedPassTripsComments
+      type: STRING
     - name: AnnualVehicleMaxService
       type: INTEGER
+    - name: AnnualVehicleMaxServiceComments
+      type: STRING
     - name: SponsoredServiceUPT
       type: INTEGER
+    - name: SponsoredServiceUPTComments
+      type: STRING
     - name: Quantity
       type: INTEGER
     - name: LastModifiedDate
@@ -372,6 +426,8 @@ schema_fields:
       type: STRING
     - name: VehiclesToBeRetiredBeyondULB
       type: INTEGER
+    - name: VehiclesToBePurchasesNextYear
+      type: INTEGER
     - name: VehiclesPastULBInTAM
       type: INTEGER
     - name: LastModifiedDate

diff --git a/warehouse/models/intermediate/gtfs/int_gtfs_rt__service_alerts_fully_unnested.sql b/warehouse/models/intermediate/gtfs/int_gtfs_rt__service_alerts_fully_unnested.sql
@@ -28,9 +28,9 @@ int_gtfs_rt__service_alerts_fully_unnested AS (
         cause,
         effect,
 
-        -- active periods
-        unnested_active_period.start AS active_period_start,
-        unnested_active_period.end AS active_period_end,
+        -- active periods, converting from STRINGS since some agency has bad data that insn't unnestable as INTs
+        SAFE_CAST(unnested_active_period.start AS INT) AS active_period_start,
+        SAFE_CAST(unnested_active_period.end AS INT) AS active_period_end,
 
         -- informted entities
         unnested_informed_entity.agencyId AS agency_id,

diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_a30_voms_vins_totals.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_a30_voms_vins_totals.sql
@@ -18,7 +18,10 @@ vins_a30 as (
     GROUP BY organization, fiscal_year
 )
 
-select voms_rr20.*, vins_a30.a30_vin_n
+SELECT COALESCE(voms_rr20.organization, vins_a30.organization) as organization,
+       COALESCE(voms_rr20.fiscal_year, vins_a30.fiscal_year) as fiscal_year,
+       voms_rr20.rr20_voms,
+       vins_a30.a30_vin_n
 FROM voms_rr20
 FULL OUTER JOIN vins_a30
     ON voms_rr20.organization = vins_a30.organization

diff --git a/warehouse/models/mart/ntd_validation/fct_ntd_a30_vomscheck.sql b/warehouse/models/mart/ntd_validation/fct_ntd_a30_vomscheck.sql
@@ -4,12 +4,14 @@
 WITH rr20f_180 as (
     SELECT organization,
     "RR20F-180: VOMS across forms" as name_of_check,
+    fiscal_year as year_of_data,
     CASE WHEN ROUND(rr20_voms, 1) > ROUND(a30_vin_n, 1)
         THEN "Fail"
         ELSE "Pass"
     END as check_status,
-    CONCAT("RR-20 VOMS = ", CAST(ROUND(rr20_voms, 1) AS STRING),
-            "# A-30 VINs = ", CAST(ROUND(a30_vin_n, 1) AS STRING)) AS value_checked,
+    CONCAT("RR-20 VOMS = ", IF(rr20_voms IS NULL, ' ', CAST(ROUND(rr20_voms, 1) AS STRING)),
+            ", A-30 VINs = ", IF(a30_vin_n IS NULL, ' ', CAST(ROUND(a30_vin_n, 1) AS STRING))
+    ) AS value_checked,
     CASE WHEN ROUND(rr20_voms, 1) > ROUND(a30_vin_n, 1)
         THEN "Total VOMS is greater than total A-30 vehicles reported. Please clarify."
         ELSE "VOMS & A-30 vehicles reported are equal to and/or lower than active inventory."