converted to use json, fixed gcsfs upload error

cal-itp · Sep 16, 2024 · aa291c8 · aa291c8
1 parent 9f5b4ee
commit aa291c8
Show file tree

Hide file tree

Showing 70 changed files with 282 additions and 391 deletions.
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_capital_expenses_by_capital_use.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_capital_expenses_by_capital_use.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_capital_expenses_by_mode.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_capital_expenses_by_mode.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_capital_expenses_for_existing_service.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_capital_expenses_for_existing_service.yml
diff --git a/.../dags/sync_ntd_data_api/2022_reporting/2022_capital_expenses_for_expansion_of_service.yml b/.../dags/sync_ntd_data_api/2022_reporting/2022_capital_expenses_for_expansion_of_service.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_employees_by_mode_and_employee_type.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_employees_by_mode_and_employee_type.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_fuel_and_energy_by_agency.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_fuel_and_energy_by_agency.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_funding_sources_by_expense_type.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_funding_sources_by_expense_type.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_funding_sources_directly_generated.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_funding_sources_directly_generated.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_funding_sources_federal.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_funding_sources_federal.yml
diff --git a/...low/dags/sync_ntd_data_api/2022_reporting/2022_funding_sources_taxes_levied_by_agency.yml b/...low/dags/sync_ntd_data_api/2022_reporting/2022_funding_sources_taxes_levied_by_agency.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_maintenance_facilities_by_agency.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_maintenance_facilities_by_agency.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_operating_expenses_by_function.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_operating_expenses_by_function.yml
diff --git a/.../dags/sync_ntd_data_api/2022_reporting/2022_operating_expenses_by_function_and_agency.yml b/.../dags/sync_ntd_data_api/2022_reporting/2022_operating_expenses_by_function_and_agency.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_operating_expenses_by_type.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_operating_expenses_by_type.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_operating_expenses_by_type_and_agency.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_operating_expenses_by_type_and_agency.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_service_by_mode_and_time_period.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_service_by_mode_and_time_period.yml
diff --git a/..._ntd_data_api/2022_reporting/2022_stations_and_facilities_by_agency_and_facility_type.yml b/..._ntd_data_api/2022_reporting/2022_stations_and_facilities_by_agency_and_facility_type.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_stations_by_mode_and_age.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_stations_by_mode_and_age.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_track_and_roadway_by_agency.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_track_and_roadway_by_agency.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_track_and_roadway_by_mode.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_track_and_roadway_by_mode.yml
diff --git a/...ags/sync_ntd_data_api/2022_reporting/2022_track_and_roadway_guideway_age_distribution.yml b/...ags/sync_ntd_data_api/2022_reporting/2022_track_and_roadway_guideway_age_distribution.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_vehicles_age_distribution.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_vehicles_age_distribution.yml
diff --git a/airflow/dags/sync_ntd_data_api/2022_reporting/2022_vehicles_type_count_by_agency.yml b/airflow/dags/sync_ntd_data_api/2022_reporting/2022_vehicles_type_count_by_agency.yml
diff --git a/...ta_api/2022_reporting/2022_breakdowns.yml → ..._api/annual_reporting/2022/breakdowns.yml b/...ta_api/2022_reporting/2022_breakdowns.yml → ..._api/annual_reporting/2022/breakdowns.yml
@@ -1,9 +1,7 @@
 operator: operators.NtdDataProductAPIOperator
 
-bucket: 'gs://calitp-ntd-api-products'
 year: '2022'
-product_long: 'Breakdowns'
-product_short: 'breakdowns'
+product: 'breakdowns'
 root_url: 'https://data.transportation.gov/resource/'
 endpoint_id: 'amkt-4ehs'
-file_format: '.csv'
+file_format: '.json'
diff --git a/...2_reporting/2022_breakdowns_by_agency.yml → ...l_reporting/2022/breakdowns_by_agency.yml b/...2_reporting/2022_breakdowns_by_agency.yml → ...l_reporting/2022/breakdowns_by_agency.yml
@@ -1,9 +1,7 @@
 operator: operators.NtdDataProductAPIOperator
 
-bucket: 'gs://calitp-ntd-api-products'
 year: '2022'
-product_long: 'Breakdowns (by Agency)'
-product_short: 'breakdowns_by_agency'
+product: 'breakdowns_by_agency'
 root_url: 'https://data.transportation.gov/resource/'
 endpoint_id: 'fk8n-qvag'
-file_format: '.csv'
+file_format: '.json'
diff --git a/airflow/dags/sync_ntd_data_api/annual_reporting/2022/capital_expenses_by_capital_use.yml b/airflow/dags/sync_ntd_data_api/annual_reporting/2022/capital_expenses_by_capital_use.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'capital_expenses_by_capital_use'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: 'fphd-jyyj'
+file_format: '.json'
diff --git a/airflow/dags/sync_ntd_data_api/annual_reporting/2022/capital_expenses_by_mode.yml b/airflow/dags/sync_ntd_data_api/annual_reporting/2022/capital_expenses_by_mode.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'capital_expenses_by_mode'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: '2667-vitc'
+file_format: '.json'
diff --git a/...ow/dags/sync_ntd_data_api/annual_reporting/2022/capital_expenses_for_existing_service.yml b/...ow/dags/sync_ntd_data_api/annual_reporting/2022/capital_expenses_for_existing_service.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'capital_expenses_for_existing_service'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: '7kqv-yqbn'
+file_format: '.json'
diff --git a/...ags/sync_ntd_data_api/annual_reporting/2022/capital_expenses_for_expansion_of_service.yml b/...ags/sync_ntd_data_api/annual_reporting/2022/capital_expenses_for_expansion_of_service.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'capital_expenses_for_expansion_of_service'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: 'nvgd-g6pj'
+file_format: '.json'
diff --git a/...22_reporting/2022_employees_by_agency.yml → ...al_reporting/2022/employees_by_agency.yml b/...22_reporting/2022_employees_by_agency.yml → ...al_reporting/2022/employees_by_agency.yml
@@ -1,9 +1,7 @@
 operator: operators.NtdDataProductAPIOperator
 
-bucket: 'gs://calitp-ntd-api-products'
 year: '2022'
-product_long: 'Employees (By Agency)'
-product_short: 'employees_by_agency'
+product: 'employees_by_agency'
 root_url: 'https://data.transportation.gov/resource/'
 endpoint_id: 'brbd-9azc'
-file_format: '.csv'
+file_format: '.json'
diff --git a/...2022_reporting/2022_employees_by_mode.yml → ...nual_reporting/2022/employees_by_mode.yml b/...2022_reporting/2022_employees_by_mode.yml → ...nual_reporting/2022/employees_by_mode.yml
@@ -1,9 +1,7 @@
 operator: operators.NtdDataProductAPIOperator
 
-bucket: 'gs://calitp-ntd-api-products'
 year: '2022'
-product_long: 'Employees (By Mode)'
-product_short: 'employees_by_mode'
+product: 'employees_by_mode'
 root_url: 'https://data.transportation.gov/resource/'
 endpoint_id: 'wsxw-2rpq'
-file_format: '.csv'
+file_format: '.json'
diff --git a/airflow/dags/sync_ntd_data_api/annual_reporting/2022/employees_by_mode_and_employee_type.yml b/airflow/dags/sync_ntd_data_api/annual_reporting/2022/employees_by_mode_and_employee_type.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'employees_by_mode_and_employee_type'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: 'uyv8-9jek'
+file_format: '.json'
diff --git a/...i/2022_reporting/2022_fuel_and_energy.yml → ...annual_reporting/2022/fuel_and_energy.yml b/...i/2022_reporting/2022_fuel_and_energy.yml → ...annual_reporting/2022/fuel_and_energy.yml
@@ -1,9 +1,7 @@
 operator: operators.NtdDataProductAPIOperator
 
-bucket: 'gs://calitp-ntd-api-products'
 year: '2022'
-product_long: 'Fuel and Energy'
-product_short: 'fuel_and_energy'
+product: 'fuel_and_energy'
 root_url: 'https://data.transportation.gov/resource/'
 endpoint_id: '8ehq-7his'
-file_format: '.csv'
+file_format: '.json'
diff --git a/airflow/dags/sync_ntd_data_api/annual_reporting/2022/fuel_and_energy_by_agency.yml b/airflow/dags/sync_ntd_data_api/annual_reporting/2022/fuel_and_energy_by_agency.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'fuel_and_energy_by_agency'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: 'wwem-ata9'
+file_format: '.json'
diff --git a/airflow/dags/sync_ntd_data_api/annual_reporting/2022/funding_sources_by_expense_type.yml b/airflow/dags/sync_ntd_data_api/annual_reporting/2022/funding_sources_by_expense_type.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'funding_sources_by_expense_type'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: '4tmr-gwuu'
+file_format: '.json'
diff --git a/airflow/dags/sync_ntd_data_api/annual_reporting/2022/funding_sources_directly_generated.yml b/airflow/dags/sync_ntd_data_api/annual_reporting/2022/funding_sources_directly_generated.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'funding_sources_directly_generated'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: 'yuaq-zdvc'
+file_format: '.json'
diff --git a/airflow/dags/sync_ntd_data_api/annual_reporting/2022/funding_sources_federal.yml b/airflow/dags/sync_ntd_data_api/annual_reporting/2022/funding_sources_federal.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'funding_sources_federal'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: 'qpjk-b3zw'
+file_format: '.json'
diff --git a/..._reporting/2022_funding_sources_local.yml → ..._reporting/2022/funding_sources_local.yml b/..._reporting/2022_funding_sources_local.yml → ..._reporting/2022/funding_sources_local.yml
@@ -1,9 +1,7 @@
 operator: operators.NtdDataProductAPIOperator
 
-bucket: 'gs://calitp-ntd-api-products'
 year: '2022'
-product_long: 'Funding Sources (Local)'
-product_short: 'funding_sources_local'
+product: 'funding_sources_local'
 root_url: 'https://data.transportation.gov/resource/'
 endpoint_id: '8tvb-ywj3'
-file_format: '.csv'
+file_format: '.json'
diff --git a/..._reporting/2022_funding_sources_state.yml → ..._reporting/2022/funding_sources_state.yml b/..._reporting/2022_funding_sources_state.yml → ..._reporting/2022/funding_sources_state.yml
@@ -1,9 +1,7 @@
 operator: operators.NtdDataProductAPIOperator
 
-bucket: 'gs://calitp-ntd-api-products'
 year: '2022'
-product_long: 'Funding Sources (State)'
-product_short: 'funding_sources_state'
+product: 'funding_sources_state'
 root_url: 'https://data.transportation.gov/resource/'
 endpoint_id: 'dd43-h6wv'
-file_format: '.csv'
+file_format: '.json'
diff --git a/...w/dags/sync_ntd_data_api/annual_reporting/2022/funding_sources_taxes_levied_by_agency.yml b/...w/dags/sync_ntd_data_api/annual_reporting/2022/funding_sources_taxes_levied_by_agency.yml
@@ -0,0 +1,7 @@
+operator: operators.NtdDataProductAPIOperator
+
+year: '2022'
+product: 'funding_sources_taxes_levied_by_agency'
+root_url: 'https://data.transportation.gov/resource/'
+endpoint_id: 'c8k8-y2cj'
+file_format: '.json'