Shows gobble data in the dashboard (#943)

* Shows gobble data in the dashboard

Only shows gobble data when manually navigating to a bus url with a date
past the current maximum

* Fixing rapid parsing

* Fixing format for headways

* Updating based on feedback

common/components/notices/BusDataNotice.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
-import { faMagnifyingGlassChart } from '@fortawesome/free-solid-svg-icons';
+import { faLocationCrosshairs } from '@fortawesome/free-solid-svg-icons';
 import { useDelimitatedRoute } from '../../utils/router';
 
 export const BusDataNotice: React.FC = () => {
@@ -9,8 +9,8 @@ export const BusDataNotice: React.FC = () => {
   if (line === 'line-bus' || linePath === 'bus') {
     return (
       <div className={'flex items-center'}>
-        <FontAwesomeIcon icon={faMagnifyingGlassChart} size={'lg'} />
-        <div className={'m-3 text-sm italic'}>
+        <FontAwesomeIcon icon={faLocationCrosshairs} size={'lg'} />
+        <div className={'mx-3 my-2 text-sm italic'}>
           <p>
             Due to data collection issues, bus data is not guaranteed to be complete for any stop or
             date.

common/components/notices/GobbleDataNotice.tsx

@@ -0,0 +1,61 @@
+import React from 'react';
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import dayjs from 'dayjs';
+import { faChartSimple } from '@fortawesome/free-solid-svg-icons';
+import Link from 'next/link';
+import classNames from 'classnames';
+import { useDelimitatedRoute } from '../../utils/router';
+import { BUS_MAX_DAY } from '../../constants/dates';
+import { lineColorTextHover } from '../../styles/general';
+
+export const GobbleDataNotice: React.FC = () => {
+  const {
+    line,
+    linePath,
+    query: { date, startDate, endDate },
+  } = useDelimitatedRoute();
+
+  const isStartDateAfterBusMaxDay =
+    (startDate !== undefined && dayjs(startDate).isAfter(BUS_MAX_DAY)) ||
+    (date !== undefined && dayjs(date).isAfter(BUS_MAX_DAY));
+  const isEndDateAfterBusMaxDay = endDate !== undefined && dayjs(endDate).isAfter(BUS_MAX_DAY);
+
+  if (
+    (line === 'line-bus' || linePath === 'bus') &&
+    (isStartDateAfterBusMaxDay || isEndDateAfterBusMaxDay)
+  ) {
+    return (
+      <div className={'flex items-center'}>
+        <FontAwesomeIcon icon={faChartSimple} size={'lg'} />
+        <div className={'mx-3 my-2 text-sm italic'}>
+          <p>
+            Data shown here is collected by TransitMatters using the{' '}
+            <Link
+              href="https://www.mbta.com/developers/v3-api/streaming"
+              rel="noopener noreferrer"
+              target="_blank"
+              className={classNames(lineColorTextHover[line ?? 'DEFAULT'])}
+            >
+              MBTA's streaming API
+            </Link>
+            . Unlike other data sources we show, this data is not cleaned or filtered in any way
+            before display. Innacuracies may be present.
+          </p>
+          <p>
+            Official MBTA data will be shown when available. Technical details of our data
+            collection can be found{' '}
+            <Link
+              href="https://github.com/transitmatters/gobble"
+              rel="noopener noreferrer"
+              target="_blank"
+              className={classNames(lineColorTextHover[line ?? 'DEFAULT'])}
+            >
+              here
+            </Link>
+          </p>
+        </div>
+      </div>
+    );
+  }
+  return null;
+};

common/components/notices/SameDayNotice.tsx

@@ -14,7 +14,7 @@ export const SameDayNotice: React.FC = () => {
     return (
       <div className={'flex items-center'}>
         <FontAwesomeIcon icon={faCalendarDay} size={'lg'} />
-        <div className={'mx-3 my-1 text-sm italic'}>
+        <div className={'mx-3 my-2 text-sm italic'}>
           <p>
             Due to data not being cleaned yet, today's data may not be fully accurate and may look
             messy.

common/components/notices/TerminusNotice.tsx

@@ -14,7 +14,7 @@ export const TerminusNotice: React.FC<TerminusNoticeProps> = ({ toStation, fromS
     return (
       <div className={'flex items-center'}>
         <FontAwesomeIcon icon={faMagnifyingGlassChart} size={'lg'} />
-        <div className={'mx-3 my-1 text-sm italic'}>
+        <div className={'mx-3 my-2 text-sm italic'}>
           <p>
             Due to data collection issues at terminus stations, data is not guaranteed to be
             complete.

common/components/widgets/WidgetPage.tsx

@@ -1,6 +1,5 @@
 import classNames from 'classnames';
 import React from 'react';
-import { BusDataNotice } from '../notices/BusDataNotice';
 
 interface WidgetPageProps {
   children?: React.ReactNode;
@@ -10,7 +9,6 @@ export const WidgetPage: React.FC<WidgetPageProps> = ({ children }) => {
   return (
     <div className={classNames('flex w-full flex-1 flex-col items-center gap-y-2 lg:p-0')}>
       {children}
-      <BusDataNotice />
     </div>
   );
 };

common/constants/dates.ts

@@ -36,7 +36,7 @@ const OVERVIEW_TRAIN_MIN_DATE = '2016-02-01';
 const TRAIN_MIN_DATE = '2016-01-15';
 const BUS_MIN_DATE = '2018-08-01';
 export const BUS_MAX_DATE = '2023-12-31';
-const BUS_MAX_DAY = dayjs(BUS_MAX_DATE);
+export const BUS_MAX_DAY = dayjs(BUS_MAX_DATE);
 export const BUS_MAX_DATE_MINUS_ONE_WEEK = dayjs(BUS_MAX_DATE)
   .subtract(7, 'days')
   .format(DATE_FORMAT);

modules/ridership/RidershipDetails.tsx

@@ -18,7 +18,6 @@ export function RidershipDetails() {
   } = useDelimitatedRoute();
   const config = SPEED_RANGE_PARAM_MAP.week;
   const lineId = getRidershipLineId(line, busRoute);
-  const lineOrRoute = busRoute ? `line-${busRoute.replaceAll('/', '')}` : line;
   const enabled = Boolean(startDate && endDate && lineId);
 
   const ridership = useRidershipData(
@@ -29,7 +28,7 @@ export function RidershipDetails() {
     },
     enabled
   );
-  const ridershipDataReady = !ridership.isError && startDate && endDate && lineOrRoute && line;
+  const ridershipDataReady = !ridership.isError && startDate && endDate && line;
 
   return (
     <PageWrapper pageTitle={'Ridership'}>

modules/ridership/RidershipWidget.tsx

@@ -16,13 +16,12 @@ export const RidershipWidget: React.FC = () => {
   const endDate = TODAY_STRING;
   const config = getSpeedGraphConfig(dayjs(startDate), dayjs(endDate));
   const lineId = getRidershipLineId(line, query.busRoute);
-  const lineOrRoute = query.busRoute ? `line-${query.busRoute.replaceAll('/', '')}` : line;
   const ridership = useRidershipData({
     line_id: lineId,
     start_date: startDate,
     end_date: endDate,
   });
-  const serviceReady = !ridership.isError && lineId && line && lineOrRoute;
+  const serviceReady = !ridership.isError && lineId && line;
 
   return (
     <WidgetDiv>

modules/tripexplorer/TripExplorer.tsx

@@ -9,6 +9,8 @@ import { ChartPageDiv } from '../../common/components/charts/ChartPageDiv';
 import { Layout } from '../../common/layouts/layoutTypes';
 import { useDelimitatedRoute } from '../../common/utils/router';
 import { getParentStationForStopId } from '../../common/utils/stations';
+import { BusDataNotice } from '../../common/components/notices/BusDataNotice';
+import { GobbleDataNotice } from '../../common/components/notices/GobbleDataNotice';
 import { useAlertStore } from './AlertStore';
 import { TripGraphs } from './TripGraphs';
 
@@ -37,8 +39,12 @@ export const TripExplorer = () => {
       <ChartPageDiv>
         {alertsForModal?.length ? <AlertNotice /> : null}
         <TripGraphs fromStation={fromStation} toStation={toStation} />
-        <SameDayNotice />
-        <TerminusNotice toStation={toStation} fromStation={fromStation} />
+        <div>
+          <GobbleDataNotice />
+          <BusDataNotice />
+          <SameDayNotice />
+          <TerminusNotice toStation={toStation} fromStation={fromStation} />
+        </div>
       </ChartPageDiv>
     </PageWrapper>
   );

server/chalicelib/s3.py

@@ -28,16 +28,27 @@ def upload(key, bytes, compress=True):
     s3.put_object(Bucket=BUCKET, Key=key, Body=bytes)
 
 
-def is_bus(stop_id):
+def is_bus(stop_id: str):
     return ("-0-" in stop_id) or ("-1-" in stop_id)
 
 
-def download_one_event_file(date, stop_id):
+def get_live_folder(stop_id: str):
+    if is_bus(stop_id):
+        return "daily-bus-data"
+    else:
+        return "daily-rapid-data"
+
+
+def download_one_event_file(date, stop_id: str, use_live_data=False):
     """As advertised: single event file from s3"""
-    year, month = date.year, date.month
+    year, month, day = date.year, date.month, date.day
 
-    folder = "monthly-bus-data" if is_bus(stop_id) else "monthly-data"
-    key = f"Events/{folder}/{stop_id}/Year={year}/Month={month}/events.csv.gz"
+    if use_live_data:
+        folder = get_live_folder(stop_id)
+        key = f"Events-live/{folder}/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv.gz"
+    else:
+        folder = "monthly-bus-data" if is_bus(stop_id) else "monthly-data"
+        key = f"Events/{folder}/{stop_id}/Year={year}/Month={month}/events.csv.gz"
 
     # Download events from S3
     try:
@@ -47,6 +58,8 @@ def download_one_event_file(date, stop_id):
         if ex.response["Error"]["Code"] == "NoSuchKey":
             # raise Exception(f"Data not available on S3 for key {key} ") from None
             print(f"WARNING: No data available on S3 for key: {key}")
+            if not use_live_data and is_bus(stop_id):
+                return download_one_event_file(date, stop_id, use_live_data=True)
             return []
         else:
             raise
@@ -67,8 +80,23 @@ def parallel_download_events(datestop):
     return download_one_event_file(date, stop)
 
 
-def download_events(sdate, edate, stops):
-    datestops = itertools.product(parallel.month_range(sdate, edate), stops)
+def download_events(sdate, edate, stops: list):
+    # This used to be month_range but updated to date_range to support live ranges
+    # If something breaks, this may be why
+    datestops = itertools.product(parallel.date_range(sdate, edate), stops)
     result = parallel_download_events(datestops)
     result = filter(lambda row: sdate.strftime("%Y-%m-%d") <= row["service_date"] <= edate.strftime("%Y-%m-%d"), result)
     return sorted(result, key=lambda row: row["event_time"])
+
+
+def get_all_s3_objects(s3, **base_kwargs):
+    continuation_token = None
+    while True:
+        list_kwargs = dict(MaxKeys=1000, **base_kwargs)
+        if continuation_token:
+            list_kwargs["ContinuationToken"] = continuation_token
+        response = s3.list_objects_v2(**list_kwargs)
+        yield from response.get("Contents", [])
+        if not response.get("IsTruncated"):  # At the end of the list?
+            break
+        continuation_token = response.get("NextContinuationToken")

server/chalicelib/s3_historical.py

@@ -32,7 +32,7 @@ def unique_everseen(iterable, key=None):
                 yield element
 
 
-def dwells(stop_ids, sdate, edate):
+def dwells(stop_ids: list, sdate, edate):
     rows_by_time = s3.download_events(sdate, edate, stop_ids)
 
     dwells = []
@@ -59,7 +59,7 @@ def dwells(stop_ids, sdate, edate):
     return dwells
 
 
-def headways(stop_ids, sdate, edate):
+def headways(stop_ids: list, sdate, edate):
     rows_by_time = s3.download_events(sdate, edate, stop_ids)
 
     only_departures = filter(lambda row: row["event_type"] in EVENT_DEPARTURE, rows_by_time)
@@ -88,7 +88,7 @@ def headways(stop_ids, sdate, edate):
             {
                 "route_id": this["route_id"],
                 "direction": this["direction_id"],
-                "current_dep_dt": this["event_time"],
+                "current_dep_dt": date_utils.return_formatted_date(this_dt),
                 "headway_time_sec": headway_time_sec,
                 "benchmark_headway_time_sec": benchmark_headway,
             }
@@ -97,7 +97,7 @@ def headways(stop_ids, sdate, edate):
     return headways
 
 
-def travel_times(stops_a, stops_b, sdate, edate):
+def travel_times(stops_a: list, stops_b: list, sdate, edate):
     rows_by_time_a = s3.download_events(sdate, edate, stops_a)
     rows_by_time_b = s3.download_events(sdate, edate, stops_b)
 

server/rapid/process_events.py

@@ -21,7 +21,6 @@ def process_events(input_csv, outdir, nozip=False):
         input_csv,
         usecols=columns,
         parse_dates=["service_date"],
-        infer_datetime_format=True,
         dtype={
             "route_id": "str",
             "trip_id": "str",

server/rapid/setup_rapid_input.sh

@@ -14,7 +14,7 @@ wget -N -O data/input/2023.zip https://www.arcgis.com/sharing/rest/content/items
 
 cd data/input
 for i in `seq 2017 2023`; do
-    unzip -d $i $i.zip
+    unzip -o -d $i $i.zip
 done
 
 # The following years only have single csv files