use assessment candidates

cal-itp · Jan 27, 2023 · 62f9eed · 62f9eed
1 parent 174bc18
commit 62f9eed
Showing 1 changed file with 31 additions and 32 deletions.
diff --git a/warehouse/models/mart/gtfs/fct_observed_trips.sql b/warehouse/models/mart/gtfs/fct_observed_trips.sql
@@ -9,8 +9,8 @@ WITH
 urls_to_datasets AS (
     SELECT * FROM {{ ref('int_transit_database__urls_to_gtfs_datasets') }}
 ),
-dim_provider_gtfs_data AS (
-    SELECT * FROM {{ ref('dim_provider_gtfs_data') }}
+assessment_entities AS (
+    SELECT * FROM {{ ref('int_gtfs_quality__daily_assessment_candidate_entities') }}
 ),
 trip_updates AS (
     SELECT * FROM {{ ref('int_gtfs_rt__trip_updates_summaries') }}
@@ -25,27 +25,27 @@ service_alerts AS (
 -- get each of these distinct
 trip_updates_to_schedule AS (
     SELECT DISTINCT
-        associated_schedule_gtfs_dataset_key,
-        trip_updates_gtfs_dataset_key,
-        _valid_from,
-        _valid_to,
-    FROM dim_provider_gtfs_data
+        date,
+        gtfs_dataset_key,
+        schedule_to_use_for_rt_validation_gtfs_dataset_key,
+    FROM assessment_entities
+    WHERE gtfs_dataset_type = 'trip_updates'
 ),
 vehicle_positions_to_schedule AS (
     SELECT DISTINCT
-        associated_schedule_gtfs_dataset_key,
-        vehicle_positions_gtfs_dataset_key,
-        _valid_from,
-        _valid_to,
-    FROM dim_provider_gtfs_data
+        date,
+        schedule_to_use_for_rt_validation_gtfs_dataset_key,
+        gtfs_dataset_key,
+    FROM assessment_entities
+    WHERE gtfs_dataset_type = 'vehicle_positions'
 ),
 service_alerts_to_schedule AS (
     SELECT DISTINCT
-        associated_schedule_gtfs_dataset_key,
-        service_alerts_gtfs_dataset_key,
-        _valid_from,
-        _valid_to,
-    FROM dim_provider_gtfs_data
+        date,
+        schedule_to_use_for_rt_validation_gtfs_dataset_key,
+        gtfs_dataset_key,
+    FROM assessment_entities
+    WHERE gtfs_dataset_type = 'service_alerts'
 ),
 
 
@@ -56,53 +56,52 @@ trip_updates_with_associated_schedule AS (
     SELECT
         trip_updates.*,
         {{ dbt_utils.surrogate_key(['trip_id', 'trip_route_id', 'trip_direction_id', 'trip_start_time', 'trip_start_date']) }} AS trip_identifier,
-        trip_updates_to_schedule.associated_schedule_gtfs_dataset_key,
+        trip_updates_to_schedule.schedule_to_use_for_rt_validation_gtfs_dataset_key,
     FROM trip_updates
     LEFT JOIN urls_to_datasets
         ON trip_updates.base64_url = urls_to_datasets.base64_url
         AND TIMESTAMP(trip_updates.dt) BETWEEN urls_to_datasets._valid_from AND urls_to_datasets._valid_to
     LEFT JOIN trip_updates_to_schedule
-        ON urls_to_datasets.gtfs_dataset_key = trip_updates_to_schedule.trip_updates_gtfs_dataset_key
-        AND TIMESTAMP(trip_updates.dt) BETWEEN trip_updates_to_schedule._valid_from AND trip_updates_to_schedule._valid_to
+        ON urls_to_datasets.gtfs_dataset_key = trip_updates_to_schedule.gtfs_dataset_key
+        AND trip_updates.dt = trip_updates_to_schedule.date
 ),
 vehicle_positions_with_associated_schedule AS (
     SELECT
         vehicle_positions.*,
         {{ dbt_utils.surrogate_key(['trip_id', 'trip_route_id', 'trip_direction_id', 'trip_start_time', 'trip_start_date']) }} AS trip_identifier,
-        vehicle_positions_to_schedule.associated_schedule_gtfs_dataset_key,
+        vehicle_positions_to_schedule.schedule_to_use_for_rt_validation_gtfs_dataset_key,
     FROM vehicle_positions
     LEFT JOIN urls_to_datasets
         ON vehicle_positions.base64_url = urls_to_datasets.base64_url
         AND TIMESTAMP(vehicle_positions.dt) BETWEEN urls_to_datasets._valid_from AND urls_to_datasets._valid_to
     LEFT JOIN vehicle_positions_to_schedule
-        ON urls_to_datasets.gtfs_dataset_key = vehicle_positions_to_schedule.vehicle_positions_gtfs_dataset_key
-        AND TIMESTAMP(vehicle_positions.dt) BETWEEN vehicle_positions_to_schedule._valid_from AND vehicle_positions_to_schedule._valid_to
+        ON urls_to_datasets.gtfs_dataset_key = vehicle_positions_to_schedule.gtfs_dataset_key
+        AND vehicle_positions.dt = vehicle_positions_to_schedule.date
 ),
 service_alerts_with_associated_schedule AS (
     SELECT
         service_alerts.*,
         {{ dbt_utils.surrogate_key(['trip_id', 'trip_route_id', 'trip_direction_id', 'trip_start_time', 'trip_start_date']) }} AS trip_identifier,
         urls_to_datasets.gtfs_dataset_key,
-        service_alerts_to_schedule.associated_schedule_gtfs_dataset_key,
+        service_alerts_to_schedule.schedule_to_use_for_rt_validation_gtfs_dataset_key,
     FROM service_alerts
     LEFT JOIN urls_to_datasets
         ON service_alerts.base64_url = urls_to_datasets.base64_url
         AND TIMESTAMP(service_alerts.dt) BETWEEN urls_to_datasets._valid_from AND urls_to_datasets._valid_to
     LEFT JOIN service_alerts_to_schedule
-        ON urls_to_datasets.gtfs_dataset_key = service_alerts_to_schedule.service_alerts_gtfs_dataset_key
-    QUALIFY ROW_NUMBER() OVER (PARTITION BY trip_identifier) = 1
-        AND TIMESTAMP(service_alerts.dt) BETWEEN service_alerts_to_schedule._valid_from AND service_alerts_to_schedule._valid_to
+        ON urls_to_datasets.gtfs_dataset_key = service_alerts_to_schedule.gtfs_dataset_key
+        AND service_alerts.dt = service_alerts_to_schedule.date
 ),
 
 fct_observed_trips AS (
     SELECT
         {{ dbt_utils.surrogate_key([
             'dt',
-            'associated_schedule_gtfs_dataset_key',
+            'schedule_to_use_for_rt_validation_gtfs_dataset_key',
             'trip_identifier',
         ]) }} as key,
         dt,
-        associated_schedule_gtfs_dataset_key,
+        schedule_to_use_for_rt_validation_gtfs_dataset_key,
         trip_identifier,
         COALESCE(tu.trip_id, vp.trip_id, sa.trip_id) AS trip_id,
         COALESCE(tu.trip_route_id, vp.trip_route_id, sa.trip_route_id) AS trip_route_id,
@@ -121,10 +120,10 @@ fct_observed_trips AS (
         sa.service_alert_message_keys AS sa_service_alert_message_keys,
     FROM trip_updates_with_associated_schedule AS tu
     FULL OUTER JOIN vehicle_positions_with_associated_schedule AS vp
-        USING (dt, associated_schedule_gtfs_dataset_key, trip_identifier)
+        USING (dt, schedule_to_use_for_rt_validation_gtfs_dataset_key, trip_identifier)
     FULL OUTER JOIN service_alerts_with_associated_schedule AS sa
-        USING (dt, associated_schedule_gtfs_dataset_key, trip_identifier)
-    WHERE associated_schedule_gtfs_dataset_key IS NOT NULL
+        USING (dt, schedule_to_use_for_rt_validation_gtfs_dataset_key, trip_identifier)
+    WHERE schedule_to_use_for_rt_validation_gtfs_dataset_key IS NOT NULL
         AND trip_identifier != 'baea044d32c787353f49b168a215e098' -- the hash of all-null trip identifier fields
 )