diff --git a/airflow/dags/create_external_tables/ntd_data_products/annual_database_service.yml b/airflow/dags/create_external_tables/ntd_data_products/annual_database_service.yml deleted file mode 100644 index 7a500b52a4..0000000000 --- a/airflow/dags/create_external_tables/ntd_data_products/annual_database_service.yml +++ /dev/null @@ -1,16 +0,0 @@ -operator: operators.ExternalTable -bucket: gs://calitp-ntd-data-products -prefix_bucket: true -post_hook: | - SELECT * - FROM `{{ get_project_id() }}`.external_ntd_data_products.annual_database_service - LIMIT 1; -source_objects: - - "annual-database-service/*.jsonl.gz" -destination_project_dataset_table: "external_ntd_data_products.annual_database_service" -source_format: NEWLINE_DELIMITED_JSON -use_bq_client: true -hive_options: - mode: CUSTOM - require_partition_filter: false - source_uri_prefix: "annual-database-service/{dt:DATE}/{ts:TIMESTAMP}/{year:INTEGER}/" diff --git a/script/scrape_ntd.py b/script/scrape_ntd.py index b001b7a685..64f84a2ec6 100644 --- a/script/scrape_ntd.py +++ b/script/scrape_ntd.py @@ -13,9 +13,6 @@ https://www.transit.dot.gov/ntd/data-product/2021-annual-database-agency-information Contains basic contact and agency information for each NTD reporter. -https://www.transit.dot.gov/ntd/data-product/2022-annual-database-service -Contains operating statistics reported by mode and type of service. Categorized by vehicles operated and vehicles available in maximum service by day and time period. - https://www.transit.dot.gov/ntd/data-product/monthly-module-adjusted-data-release Monthly ridership @@ -28,7 +25,6 @@ then poetry run python scrape_ntd.py annual-database-agency-information 2021 https://www.transit.dot.gov/sites/fta.dot.gov/files/2022-10/2021%20Agency%20Information.xlsx poetry run python scrape_ntd.py monthly-ridership-with-adjustments 2024 https://www.transit.dot.gov/sites/fta.dot.gov/files/2024-04/February%202024%20Complete%20Monthly%20Ridership%20%28with%20adjustments%20and%20estimates%29_240402_0.xlsx -poetry run python scrape_ntd.py annual-database-service 2022 https://www.transit.dot.gov/sites/fta.dot.gov/files/2024-04/2022%20Service.xlsx """ import gzip diff --git a/warehouse/models/staging/ntd/_src.yml b/warehouse/models/staging/ntd/_src.yml index 614f220b52..5984ed77b8 100644 --- a/warehouse/models/staging/ntd/_src.yml +++ b/warehouse/models/staging/ntd/_src.yml @@ -7,7 +7,6 @@ sources: schema: external_ntd_data_products tables: - name: annual_database_agency_information - - name: annual_database_service - name: monthly_ridership_with_adjustments_upt - name: monthly_ridership_with_adjustments_vrm - name: monthly_ridership_with_adjustments_vrh diff --git a/warehouse/models/staging/ntd/_stg_ntd.yml b/warehouse/models/staging/ntd/_stg_ntd.yml index d520b91a0a..c41e5a1a06 100644 --- a/warehouse/models/staging/ntd/_stg_ntd.yml +++ b/warehouse/models/staging/ntd/_stg_ntd.yml @@ -12,7 +12,6 @@ models: - name: ntd_id tests: - not_null - - name: stg_ntd__annual_database_service - name: stg_ntd__monthly_ridership_with_adjustments_upt - name: stg_ntd__monthly_ridership_with_adjustments_vrm - name: stg_ntd__monthly_ridership_with_adjustments_vrh diff --git a/warehouse/models/staging/ntd/stg_ntd__annual_database_service.sql b/warehouse/models/staging/ntd/stg_ntd__annual_database_service.sql deleted file mode 100644 index c36aa3b366..0000000000 --- a/warehouse/models/staging/ntd/stg_ntd__annual_database_service.sql +++ /dev/null @@ -1,54 +0,0 @@ -WITH source AS ( - SELECT * FROM {{ source('ntd_data_products', 'annual_database_service') }} -), - -stg_ntd__annual_database_service AS ( - SELECT - dt as _dt, - ts, - year, - state_parent_ntd_id, - format("%05d", cast(ntd_id as int64)) as ntd_id, - agency_name, - reporter_type, - subrecipient_type, - reporting_module, - mode, - tos, - time_period, - time_service_begins, - time_service_ends, - vehicles_passenger_cars_operated_in_maximum_service, - vehicles_passenger_cars_available_for_maximum_service, - trains_in_operation, - vehicles_passenger_cars_in_operation, - actual_vehicles_passenger_car_miles, - actual_vehicles_passenger_car_revenue_miles, - actual_vehicle_passenger_deadhead_miles, - scheduled_actual_vehicle_passenger_car_revenue_miles, - actual_vehicle_passenger_car_hours, - actual_vehicle_passenger_car_revenue_hours, - actual_vehicle_passenger_car_deadhead_hours, - charter_service_hours, - school_bus_hours, - train_miles, - train_revenue_miles, - train_deadhead_miles, - train_hours, - train_revenue_hours, - train_deadhead_hours, - unlinked_passenger_trips__upt_, - ada_upt, - sponsored_service_upt, - passenger_miles, - days_of_service_operated, - days_not_operated_due_to_strikes, - strike_comment, - days_not_operated_due_to_emergencies, - emergency_comment, - non_statutory_mixed_traffic, - drm_mixed_traffic_row, - FROM source -) - -SELECT * FROM stg_ntd__annual_database_service