Review 2022 NTD external tables column data types and missing optiona…

…l columns [#3403]
cal-itp · Sep 26, 2024 · 83c7843 · 83c7843
1 parent 86ab19e
commit 83c7843
Show file tree

Hide file tree

Showing 43 changed files with 840 additions and 356 deletions.
diff --git a/airflow/dags/create_external_tables/ntd_data_products/2022__breakdowns.yml b/airflow/dags/create_external_tables/ntd_data_products/2022__breakdowns.yml
@@ -20,32 +20,42 @@ schema_fields:
     type: STRING
   - name: major_mechanical_failures
     type: NUMERIC
+  - name: major_mechanical_failures_1
+    type: STRING
   - name: mode
     type: STRING
   - name: mode_name
     type: STRING
   - name: mode_voms
     type: NUMERIC
   - name: ntd_id
-    type: NUMERIC
+    type: STRING
   - name: organization_type
     type: STRING
   - name: other_mechanical_failures
     type: NUMERIC
+  - name: other_mechanical_failures_1
+    type: STRING
   - name: primary_uza_population
     type: NUMERIC
   - name: report_year
-    type: NUMERIC
+    type: STRING
   - name: reporter_type
     type: STRING
   - name: state
     type: STRING
   - name: total_mechanical_failures
     type: NUMERIC
+  - name: total_mechanical_failures_1
+    type: STRING
   - name: train_miles
     type: NUMERIC
+  - name: train_miles_questionable
+    type: STRING
   - name: train_revenue_miles
     type: NUMERIC
+  - name: train_revenue_miles_1
+    type: STRING
   - name: type_of_service
     type: STRING
   - name: uace_code
@@ -54,5 +64,9 @@ schema_fields:
     type: STRING
   - name: vehicle_passenger_car_miles
     type: NUMERIC
+  - name: vehicle_passenger_car_miles_1
+    type: STRING
+  - name: vehicle_passenger_car_miles_2
+    type: STRING
   - name: vehicle_passenger_car_revenue
     type: NUMERIC
diff --git a/airflow/dags/create_external_tables/ntd_data_products/2022__breakdowns_by_agency.yml b/airflow/dags/create_external_tables/ntd_data_products/2022__breakdowns_by_agency.yml
@@ -43,9 +43,9 @@ schema_fields:
   - name: max_uza_name
     type: STRING
   - name: ntd_id
-    type: NUMERIC
+    type: STRING
   - name: report_year
-    type: NUMERIC
+    type: STRING
   - name: sum_major_mechanical_failures
     type: NUMERIC
   - name: sum_other_mechanical_failures

diff --git a/...w/dags/create_external_tables/ntd_data_products/2022__capital_expenses_by_capital_use.yml b/...w/dags/create_external_tables/ntd_data_products/2022__capital_expenses_by_capital_use.yml
@@ -13,6 +13,8 @@ prefix_bucket: false
 post_hook: SELECT * FROM `{{ get_project_id() }}`.external_ntd__annual_reporting.2022__capital_expenses_by_capital_use LIMIT 1;
 schema_fields:
   - name: administrative_buildings
+    type: NUMERIC
+  - name: administrative_buildings_1
     type: STRING
   - name: agency
     type: STRING
@@ -21,14 +23,22 @@ schema_fields:
   - name: city
     type: STRING
   - name: communication_information
+    type: NUMERIC
+  - name: communication_information_1
     type: STRING
   - name: fare_collection_equipment
+    type: NUMERIC
+  - name: fare_collection_equipment_1
     type: STRING
   - name: form_type
     type: STRING
   - name: guideway
-    type: STRING
+    type: NUMERIC
+  - name: guideway_questionable
+    type: NUMERIC
   - name: maintenance_buildings
+    type: NUMERIC
+  - name: maintenance_buildings_1
     type: STRING
   - name: mode_name
     type: STRING
@@ -37,26 +47,44 @@ schema_fields:
   - name: modecd
     type: STRING
   - name: ntd_id
-    type: NUMERIC
+    type: STRING
   - name: organization_type
     type: STRING
   - name: other
+    type: NUMERIC
+  - name: other_questionable
     type: STRING
   - name: other_vehicles
+    type: NUMERIC
+  - name: other_vehicles_questionable
     type: STRING
   - name: passenger_vehicles
+    type: NUMERIC
+  - name: passenger_vehicles_1
     type: STRING
+  - name: primary_uza_population
+    type: NUMERIC
   - name: reduced_reporter
+    type: NUMERIC
+  - name: reduced_reporter_questionable
     type: STRING
   - name: report_year
-    type: NUMERIC
+    type: STRING
   - name: reporter_type
     type: STRING
   - name: state
     type: STRING
   - name: stations
+    type: NUMERIC
+  - name: stations_questionable
     type: STRING
   - name: total
     type: NUMERIC
+  - name: total_questionable
+    type: STRING
   - name: typeofservicecd
     type: STRING
+  - name: uace_code
+    type: STRING
+  - name: uza_name
+    type: STRING
diff --git a/airflow/dags/create_external_tables/ntd_data_products/2022__capital_expenses_by_mode.yml b/airflow/dags/create_external_tables/ntd_data_products/2022__capital_expenses_by_mode.yml
@@ -33,23 +33,29 @@ schema_fields:
   - name: max_agency
     type: STRING
   - name: max_agency_voms
-    type: STRING
+    type: NUMERIC
   - name: max_city
     type: STRING
   - name: max_mode_name
     type: STRING
   - name: max_organization_type
     type: STRING
+  - name: max_primary_uza_population
+    type: NUMERIC
   - name: max_reporter_type
     type: STRING
   - name: max_state
     type: STRING
+  - name: max_uace_code
+    type: STRING
+  - name: max_uza_name
+    type: STRING
   - name: modecd
     type: STRING
   - name: ntd_id
-    type: NUMERIC
+    type: STRING
   - name: report_year
-    type: NUMERIC
+    type: STRING
   - name: sum_administrative_buildings
     type: NUMERIC
   - name: sum_communication_information

diff --git a/.../create_external_tables/ntd_data_products/2022__capital_expenses_for_existing_service.yml b/.../create_external_tables/ntd_data_products/2022__capital_expenses_for_existing_service.yml
@@ -17,13 +17,13 @@ schema_fields:
   - name: max_agency
     type: STRING
   - name: max_agency_voms
-    type: STRING
+    type: NUMERIC
   - name: max_city
     type: STRING
   - name: max_organization_type
     type: STRING
   - name: max_primary_uza_population
-    type: STRING
+    type: NUMERIC
   - name: max_reporter_type
     type: STRING
   - name: max_state
@@ -33,9 +33,9 @@ schema_fields:
   - name: max_uza_name
     type: STRING
   - name: ntd_id
-    type: NUMERIC
+    type: STRING
   - name: report_year
-    type: NUMERIC
+    type: STRING
   - name: sum_administrative_buildings
     type: NUMERIC
   - name: sum_communication_information

diff --git a/...ate_external_tables/ntd_data_products/2022__capital_expenses_for_expansion_of_service.yml b/...ate_external_tables/ntd_data_products/2022__capital_expenses_for_expansion_of_service.yml
@@ -17,7 +17,7 @@ schema_fields:
   - name: max_agency
     type: STRING
   - name: max_agency_voms
-    type: STRING
+    type: NUMERIC
   - name: max_city
     type: STRING
   - name: max_organization_type
@@ -33,9 +33,9 @@ schema_fields:
   - name: max_uza_name
     type: STRING
   - name: ntd_id
-    type: NUMERIC
+    type: STRING
   - name: report_year
-    type: NUMERIC
+    type: STRING
   - name: sum_administrative_buildings
     type: NUMERIC
   - name: sum_communication_information

diff --git a/airflow/dags/create_external_tables/ntd_data_products/2022__employees_by_agency.yml b/airflow/dags/create_external_tables/ntd_data_products/2022__employees_by_agency.yml
@@ -41,21 +41,21 @@ schema_fields:
   - name: count_vehicle_operations_hours_q
     type: NUMERIC
   - name: max_agency_voms_1
-    type: STRING
+    type: NUMERIC
   - name: max_city_1
     type: STRING
   - name: max_mode_voms
     type: NUMERIC
   - name: max_ntd_id
-    type: NUMERIC
-  - name: max_primary_uza_population_1
     type: STRING
+  - name: max_primary_uza_population_1
+    type: NUMERIC
   - name: max_state_1
     type: STRING
   - name: max_uza_name_1
     type: STRING
   - name: report_year
-    type: NUMERIC
+    type: STRING
   - name: sum_total_hours
     type: NUMERIC
   - name: total_employees

diff --git a/airflow/dags/create_external_tables/ntd_data_products/2022__employees_by_mode.yml b/airflow/dags/create_external_tables/ntd_data_products/2022__employees_by_mode.yml
@@ -41,9 +41,9 @@ schema_fields:
   - name: mode
     type: STRING
   - name: ntd_id
-    type: NUMERIC
+    type: STRING
   - name: report_year
-    type: NUMERIC
+    type: STRING
   - name: sum_total_employee_count
     type: NUMERIC
   - name: sum_total_hours

diff --git a/...gs/create_external_tables/ntd_data_products/2022__employees_by_mode_and_employee_type.yml b/...gs/create_external_tables/ntd_data_products/2022__employees_by_mode_and_employee_type.yml
@@ -18,40 +18,56 @@ schema_fields:
     type: NUMERIC
   - name: capital_labor_count
     type: NUMERIC
+  - name: capital_labor_count_q
+    type: STRING
   - name: capital_labor_hours
     type: NUMERIC
+  - name: capital_labor_hours_q
+    type: STRING
   - name: city
     type: STRING
   - name: facility_maintenance_count
     type: NUMERIC
+  - name: facility_maintenance_count_q
+    type: STRING
   - name: facility_maintenance_hours
     type: NUMERIC
+  - name: facility_maintenance_hours_q
+    type: STRING
   - name: full_or_part_time
     type: STRING
   - name: general_administration_count
     type: NUMERIC
+  - name: general_administration_count_q
+    type: STRING
   - name: general_administration_hours
     type: NUMERIC
+  - name: general_administration_hours_q
+    type: STRING
   - name: mode
     type: STRING
   - name: mode_name
     type: STRING
   - name: mode_voms
     type: NUMERIC
   - name: ntd_id
-    type: NUMERIC
+    type: STRING
   - name: organization_type
     type: STRING
   - name: primary_uza_population
     type: NUMERIC
   - name: report_year
-    type: NUMERIC
+    type: STRING
   - name: state
     type: STRING
   - name: total_employee_count
     type: NUMERIC
+  - name: total_employee_count_q
+    type: STRING
   - name: total_hours
     type: NUMERIC
+  - name: total_hours_q
+    type: STRING
   - name: type_of_service
     type: STRING
   - name: uace_code
@@ -60,9 +76,17 @@ schema_fields:
     type: STRING
   - name: vehicle_maintenance_count
     type: NUMERIC
+  - name: vehicle_maintenance_count_q
+    type: STRING
   - name: vehicle_maintenance_hours
     type: NUMERIC
+  - name: vehicle_maintenance_hours_q
+    type: STRING
   - name: vehicle_operations_count
     type: NUMERIC
+  - name: vehicle_operations_count_q
+    type: STRING
   - name: vehicle_operations_hours
     type: NUMERIC
+  - name: vehicle_operations_hours_q
+    type: STRING