Merge pull request #1496 from cal-itp/gtfs-digest

GTFS Digest - Operator Section Refactor

Author: amandaha8

_shared_utils/shared_utils/gtfs_analytics_data.yml

@@ -61,12 +61,13 @@ rt_vs_schedule_tables:
 
 digest_tables:
   dir: ${gcs_paths.RT_SCHED_GCS}
-  # Amanda: delete route_schedule_vp later
-  route_schedule_vp: "digest/schedule_vp_metrics"
   monthly_route_schedule_vp: "digest/schedule_vp_metrics"
-  quarterly_route_schedule_vp: "digest/quarterly_schedule_vp_metrics"
+  monthly_route_schedule_vp_report: ${.monthly_route_schedule_vp}_report
+  #quarterly_route_schedule_vp: "digest/quarterly_schedule_vp_metrics" # doesn't exist yet
   operator_profiles: "digest/operator_profiles"
+  operator_profiles_report: ${.operator_profiles}_report
   operator_routes_map: "digest/operator_routes"
+  operator_routes_map_report: ${.operator_routes_map}_report
   operator_sched_rt: "digest/operator_schedule_rt_category"
   scheduled_service_hours: "digest/total_scheduled_service_hours"
 

_shared_utils/shared_utils/publish_utils.py

@@ -101,12 +101,15 @@ def filter_to_recent_date(df: pd.DataFrame, group_cols: list) -> pd.DataFrame:
     Example: By schedule_gtfs_dataset_name, keep the most recent
     service_date that shows up in scheduled trips.
     """
-    df2 = (
+    most_recent_df = (
         df.groupby(group_cols, group_keys=False)
         .service_date.max()
         .reset_index()
         .sort_values(["service_date"] + group_cols, ascending=[False] + [True for c in group_cols])
         .reset_index(drop=True)
         # .astype({"service_date": "str"})
     )
-    return df2
+
+    subset_df = pd.merge(df, most_recent_df, on=group_cols + ["service_date"], how="inner")
+
+    return subset_df

_shared_utils/shared_utils/shared_data_catalog.yml

@@ -23,30 +23,35 @@ sources:
     args:
       # source: shared_utils/shared_data.py
       urlpath: gs://calitp-analytics-data/data-analyses/shared_data/state_highway_network.parquet
+      use_fsspec: true
   ca_transit_routes:
     driver: geoparquet
     description: CA transit routes with line geometry at the operator-level (open data)
     args:
       # source: open_data/create_routes_data.py
       urlpath: gs://calitp-analytics-data/data-analyses/traffic_ops/ca_transit_routes.parquet
+      use_fsspec: true
   ca_transit_stops:
     driver: geoparquet
     description: CA transit stops with point geometry (open data)
     args:
       # source: open_data/create_stops_data.py
       urlpath: gs://calitp-analytics-data/data-analyses/traffic_ops/ca_transit_stops.parquet
+      use_fsspec: true
   hqta_stops:
     driver: geoparquet
     description: CA high quality transit areas with point geometry (open data)
     args:
       # source: high_quality_transit_areas/D1_assemble_hqta_points.py
       urlpath: gs://calitp-analytics-data/data-analyses/high_quality_transit_areas/hqta_points.parquet
+      use_fsspec: true
   hqta_areas:
     driver: geoparquet
     description: CA high quality transit areas with polygon geometry (open data)
     args:
       # source: high_quality_transit_areas/D2_assemble_hqta_polygons.py
       urlpath: gs://calitp-analytics-data/data-analyses/high_quality_transit_areas/hqta_areas.parquet
+      use_fsspec: true
   us_states:
     driver: geojson
     description: US state polygons
@@ -59,19 +64,22 @@ sources:
     args:
       # source: bus_service_increase/bus_service_utils/generate_calenviroscreen_lehd_data.py
       urlpath: gs://calitp-analytics-data/data-analyses/bus_service_increase/calenviroscreen_lehd_by_tract.parquet
+      use_fsspec: true
   state_highway_network_postmiles:
     driver: geoparquet
     description: Caltrans State Highway Network postmiles (every 0.1 mile) with postmiles as point geometry.
     args:
        # source: https://gisdata-caltrans.opendata.arcgis.com/datasets/c22341fec9c74c6b9488ee4da23dd967_0/about
        # hitting url directly would limit us to 2,000 rows
       urlpath: gs://calitp-analytics-data/data-analyses/shared_data/state_highway_network_postmiles.parquet
+      use_fsspec: true
   state_highway_network_postmile_segments:
     driver: geoparquet
     description: Caltrans State Highway Network postmile segments (postmiles converted to line segments)
     args:
        # source: shared_utils/shared_data.py
       urlpath: gs://calitp-analytics-data/data-analyses/shared_data/state_highway_network_postmile_segments.parquet
+      use_fsspec: true
   assembly_districts:
     driver: geojson
     description: California assembly district polygons