Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Shows gobble data in the dashboard #943

Merged
merged 4 commits into from
Jan 30, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions common/components/notices/BusDataNotice.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import React from 'react';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import { faMagnifyingGlassChart } from '@fortawesome/free-solid-svg-icons';
import { faLocationCrosshairs } from '@fortawesome/free-solid-svg-icons';
import { useDelimitatedRoute } from '../../utils/router';

export const BusDataNotice: React.FC = () => {
Expand All @@ -9,8 +9,8 @@ export const BusDataNotice: React.FC = () => {
if (line === 'line-bus' || linePath === 'bus') {
return (
<div className={'flex items-center'}>
<FontAwesomeIcon icon={faMagnifyingGlassChart} size={'lg'} />
<div className={'m-3 text-sm italic'}>
<FontAwesomeIcon icon={faLocationCrosshairs} size={'lg'} />
<div className={'mx-3 my-2 text-sm italic'}>
<p>
Due to data collection issues, bus data is not guaranteed to be complete for any stop or
date.
Expand Down
61 changes: 61 additions & 0 deletions common/components/notices/GobbleDataNotice.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import React from 'react';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import dayjs from 'dayjs';
import { faChartSimple } from '@fortawesome/free-solid-svg-icons';
import Link from 'next/link';
import classNames from 'classnames';
import { useDelimitatedRoute } from '../../utils/router';
import { BUS_MAX_DAY } from '../../constants/dates';
import { lineColorTextHover } from '../../styles/general';

export const GobbleDataNotice: React.FC = () => {
const {
line,
linePath,
query: { date, startDate, endDate },
} = useDelimitatedRoute();

const isStartDateAfterBusMaxDay =
(startDate !== undefined && dayjs(startDate).isAfter(BUS_MAX_DAY)) ||
(date !== undefined && dayjs(date).isAfter(BUS_MAX_DAY));
const isEndDateAfterBusMaxDay = endDate !== undefined && dayjs(endDate).isAfter(BUS_MAX_DAY);

if (
(line === 'line-bus' || linePath === 'bus') &&
(isStartDateAfterBusMaxDay || isEndDateAfterBusMaxDay)
) {
return (
<div className={'flex items-center'}>
<FontAwesomeIcon icon={faChartSimple} size={'lg'} />
<div className={'mx-3 my-2 text-sm italic'}>
<p>
Data shown here is collected by TransitMatters using the{' '}
<Link
href="https://www.mbta.com/developers/v3-api/streaming"
rel="noopener noreferrer"
target="_blank"
className={classNames(lineColorTextHover[line ?? 'DEFAULT'])}
>
MBTA's streaming API
</Link>
. Unlike other data sources we show, this data is not cleaned or filtered in any way
before display. Innacuracies may be present.
</p>
<p>
Official MBTA data will be shown when available. Technical details of our data
collection can be found{' '}
<Link
href="https://github.com/transitmatters/gobble"
rel="noopener noreferrer"
target="_blank"
className={classNames(lineColorTextHover[line ?? 'DEFAULT'])}
>
here
</Link>
</p>
</div>
</div>
);
}
return null;
};
2 changes: 1 addition & 1 deletion common/components/notices/SameDayNotice.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export const SameDayNotice: React.FC = () => {
return (
<div className={'flex items-center'}>
<FontAwesomeIcon icon={faCalendarDay} size={'lg'} />
<div className={'mx-3 my-1 text-sm italic'}>
<div className={'mx-3 my-2 text-sm italic'}>
<p>
Due to data not being cleaned yet, today's data may not be fully accurate and may look
messy.
Expand Down
2 changes: 1 addition & 1 deletion common/components/notices/TerminusNotice.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export const TerminusNotice: React.FC<TerminusNoticeProps> = ({ toStation, fromS
return (
<div className={'flex items-center'}>
<FontAwesomeIcon icon={faMagnifyingGlassChart} size={'lg'} />
<div className={'mx-3 my-1 text-sm italic'}>
<div className={'mx-3 my-2 text-sm italic'}>
<p>
Due to data collection issues at terminus stations, data is not guaranteed to be
complete.
Expand Down
2 changes: 0 additions & 2 deletions common/components/widgets/WidgetPage.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import classNames from 'classnames';
import React from 'react';
import { BusDataNotice } from '../notices/BusDataNotice';

interface WidgetPageProps {
children?: React.ReactNode;
Expand All @@ -10,7 +9,6 @@ export const WidgetPage: React.FC<WidgetPageProps> = ({ children }) => {
return (
<div className={classNames('flex w-full flex-1 flex-col items-center gap-y-2 lg:p-0')}>
{children}
<BusDataNotice />
</div>
);
};
2 changes: 1 addition & 1 deletion common/constants/dates.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ const OVERVIEW_TRAIN_MIN_DATE = '2016-02-01';
const TRAIN_MIN_DATE = '2016-01-15';
const BUS_MIN_DATE = '2018-08-01';
export const BUS_MAX_DATE = '2023-12-31';
const BUS_MAX_DAY = dayjs(BUS_MAX_DATE);
export const BUS_MAX_DAY = dayjs(BUS_MAX_DATE);
export const BUS_MAX_DATE_MINUS_ONE_WEEK = dayjs(BUS_MAX_DATE)
.subtract(7, 'days')
.format(DATE_FORMAT);
Expand Down
3 changes: 1 addition & 2 deletions modules/ridership/RidershipDetails.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ export function RidershipDetails() {
} = useDelimitatedRoute();
const config = SPEED_RANGE_PARAM_MAP.week;
const lineId = getRidershipLineId(line, busRoute);
const lineOrRoute = busRoute ? `line-${busRoute.replaceAll('/', '')}` : line;
const enabled = Boolean(startDate && endDate && lineId);

const ridership = useRidershipData(
Expand All @@ -29,7 +28,7 @@ export function RidershipDetails() {
},
enabled
);
const ridershipDataReady = !ridership.isError && startDate && endDate && lineOrRoute && line;
const ridershipDataReady = !ridership.isError && startDate && endDate && line;

return (
<PageWrapper pageTitle={'Ridership'}>
Expand Down
3 changes: 1 addition & 2 deletions modules/ridership/RidershipWidget.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,12 @@ export const RidershipWidget: React.FC = () => {
const endDate = TODAY_STRING;
const config = getSpeedGraphConfig(dayjs(startDate), dayjs(endDate));
const lineId = getRidershipLineId(line, query.busRoute);
const lineOrRoute = query.busRoute ? `line-${query.busRoute.replaceAll('/', '')}` : line;
const ridership = useRidershipData({
line_id: lineId,
start_date: startDate,
end_date: endDate,
});
const serviceReady = !ridership.isError && lineId && line && lineOrRoute;
const serviceReady = !ridership.isError && lineId && line;

return (
<WidgetDiv>
Expand Down
10 changes: 8 additions & 2 deletions modules/tripexplorer/TripExplorer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import { ChartPageDiv } from '../../common/components/charts/ChartPageDiv';
import { Layout } from '../../common/layouts/layoutTypes';
import { useDelimitatedRoute } from '../../common/utils/router';
import { getParentStationForStopId } from '../../common/utils/stations';
import { BusDataNotice } from '../../common/components/notices/BusDataNotice';
import { GobbleDataNotice } from '../../common/components/notices/GobbleDataNotice';
import { useAlertStore } from './AlertStore';
import { TripGraphs } from './TripGraphs';

Expand Down Expand Up @@ -37,8 +39,12 @@ export const TripExplorer = () => {
<ChartPageDiv>
{alertsForModal?.length ? <AlertNotice /> : null}
<TripGraphs fromStation={fromStation} toStation={toStation} />
<SameDayNotice />
<TerminusNotice toStation={toStation} fromStation={fromStation} />
<div>
<GobbleDataNotice />
<BusDataNotice />
<SameDayNotice />
<TerminusNotice toStation={toStation} fromStation={fromStation} />
</div>
</ChartPageDiv>
</PageWrapper>
);
Expand Down
29 changes: 22 additions & 7 deletions server/chalicelib/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,27 @@ def upload(key, bytes, compress=True):
s3.put_object(Bucket=BUCKET, Key=key, Body=bytes)


def is_bus(stop_id):
def is_bus(stop_id: str):
return ("-0-" in stop_id) or ("-1-" in stop_id)


def download_one_event_file(date, stop_id):
def get_live_folder(stop_id: str):
if is_bus(stop_id):
return "daily-bus-data"
else:
return "daily-data"

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the other folders in Events-live/ are daily-cr-data/ and daily-rapid-data/! daily-data is a subfolder of Events/

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have this in the other branch #898
I guess I can have all three in this branch



def download_one_event_file(date, stop_id: str, live=False):
devinmatte marked this conversation as resolved.
Show resolved Hide resolved
"""As advertised: single event file from s3"""
year, month = date.year, date.month
year, month, day = date.year, date.month, date.day

folder = "monthly-bus-data" if is_bus(stop_id) else "monthly-data"
key = f"Events/{folder}/{stop_id}/Year={year}/Month={month}/events.csv.gz"
if live:
folder = get_live_folder(stop_id)
key = f"Events-live/{folder}/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv.gz"
else:
folder = "monthly-bus-data" if is_bus(stop_id) else "monthly-data"
key = f"Events/{folder}/{stop_id}/Year={year}/Month={month}/events.csv.gz"

# Download events from S3
try:
Expand All @@ -47,6 +58,8 @@ def download_one_event_file(date, stop_id):
if ex.response["Error"]["Code"] == "NoSuchKey":
# raise Exception(f"Data not available on S3 for key {key} ") from None
print(f"WARNING: No data available on S3 for key: {key}")
if not live and is_bus(stop_id):
return download_one_event_file(date, stop_id, live=True)
return []
else:
raise
Expand All @@ -67,8 +80,10 @@ def parallel_download_events(datestop):
return download_one_event_file(date, stop)


def download_events(sdate, edate, stops):
datestops = itertools.product(parallel.month_range(sdate, edate), stops)
def download_events(sdate, edate, stops: list):
# This used to be month_range but updated to date_range to support live ranges
# If something breaks, this may be why
datestops = itertools.product(parallel.date_range(sdate, edate), stops)
result = parallel_download_events(datestops)
result = filter(lambda row: sdate.strftime("%Y-%m-%d") <= row["service_date"] <= edate.strftime("%Y-%m-%d"), result)
return sorted(result, key=lambda row: row["event_time"])
6 changes: 3 additions & 3 deletions server/chalicelib/s3_historical.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def unique_everseen(iterable, key=None):
yield element


def dwells(stop_ids, sdate, edate):
def dwells(stop_ids: list, sdate, edate):
rows_by_time = s3.download_events(sdate, edate, stop_ids)

dwells = []
Expand All @@ -59,7 +59,7 @@ def dwells(stop_ids, sdate, edate):
return dwells


def headways(stop_ids, sdate, edate):
def headways(stop_ids: list, sdate, edate):
rows_by_time = s3.download_events(sdate, edate, stop_ids)

only_departures = filter(lambda row: row["event_type"] in EVENT_DEPARTURE, rows_by_time)
Expand Down Expand Up @@ -97,7 +97,7 @@ def headways(stop_ids, sdate, edate):
return headways


def travel_times(stops_a, stops_b, sdate, edate):
def travel_times(stops_a: list, stops_b: list, sdate, edate):
rows_by_time_a = s3.download_events(sdate, edate, stops_a)
rows_by_time_b = s3.download_events(sdate, edate, stops_b)

Expand Down
Loading
Loading