From dc58bd766a7a7736db5dcdbe565ebbfead8d45dc Mon Sep 17 00:00:00 2001 From: Preston Mueller Date: Sat, 7 Aug 2021 16:39:54 -0400 Subject: [PATCH 001/152] Bus 2 train tool --- server/bus/bus2train.py | 151 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 server/bus/bus2train.py diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py new file mode 100644 index 000000000..9a9015c73 --- /dev/null +++ b/server/bus/bus2train.py @@ -0,0 +1,151 @@ +import csv +import gzip +import json +import os +import pathlib +import sys +import traceback +from collections import defaultdict +from datetime import date +from operator import itemgetter + +CSV_HEADER = ["service_date", "route_id", "trip_id", "direction_id", "stop_id", "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] +AVAILABILITY_FILE = "bus_available.json" + +def write_row(output_by_stop_and_day, stop_id, day, row): + if (stop_id, day) in output_by_stop_and_day: + output_by_stop_and_day[(stop_id, day)].append(row) + return + + output_by_stop_and_day[(stop_id, day)] = [row] + + +def parse_user_date(user_date): + date_split = user_date.split("-") + [year, month, day] = [int(x) for x in date_split[0:3]] + return date(year=year, month=month, day=day) + + +def merge_availability_dicts(current: defaultdict, from_disk: dict): + for stop_id, directions in from_disk.items(): + current[stop_id] = list(set(current[stop_id] + directions)) + return current + + +def process(input_csv): + output_by_stop_and_day = {} + availability_metadata = defaultdict(list) + + with open(input_csv, newline='') as csv_in: + reader = csv.DictReader(csv_in) + + for row in reader: + try: + if row["standard_type"] != "Headway": + continue + if row["actual"] == "NA": + continue + + """ + "service_date", "route_id", "direction", "half_trip_id", "stop_id", "time_point_id", "time_point_order", "point_type", "standard_type", "scheduled", "actual", "scheduled_headway", "headway" + 2020-01-15, "01", "Inbound", 46374001, 67, "maput", 2, "Midpoint", "Schedule", 1900-01-01 05:08:00, 1900-01-01 05:09:07, -5, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 110, "hhgat", 1, "Startpoint", "Schedule", 1900-01-01 05:05:00, 1900-01-01 05:04:34, 26, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 72, "cntsq", 3, "Midpoint", "Schedule", 1900-01-01 05:11:00, 1900-01-01 05:12:01, -22, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 75, "mit", 4, "Midpoint", "Schedule", 1900-01-01 05:14:00, 1900-01-01 05:14:58, -25, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 79, "hynes", 5, "Midpoint", "Schedule", 1900-01-01 05:18:00, 1900-01-01 05:18:45, 32, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 187, "masta", 6, "Midpoint", "Schedule", 1900-01-01 05:20:00, 1900-01-01 05:21:04, -33, NA,NA + 2020-01-15, "01", "Inbound", 46374045, 110, "hhgat", 1, "Startpoint", "Headway", 1900-01-01 05:20:00, 1900-01-01 05:20:45, NA, 900,971 + """ + + time = row["actual"].split(" ")[1] + service_date = row["service_date"] + route_id = row["route_id"].lstrip("0") + trip_id = row["half_trip_id"] + direction_id = 0 if row["direction"] == "Outbound" else 1 + stop_id = int(row["stop_id"]) + event_type = "ARR" + event_time = f"{service_date} {time}" + + # Debug override + # if service_date != "2020-01-15" or route_id != "1": + # continue + + if direction_id not in availability_metadata[stop_id]: + availability_metadata[stop_id].append(direction_id) + + # Bus has no delineation between departure and arrival, so we write out a departure row and an arrival row that match, + # so it looks like the rapid transit format + # Write out an arrival row + write_row(output_by_stop_and_day, stop_id, service_date, [ + service_date, + route_id, + trip_id, + direction_id, + stop_id, + "", # stop_sequence + "", # vehicle_id + "", # vehicle_label + event_type, + event_time, + ]) + # Write out a departure row + write_row(output_by_stop_and_day, stop_id, service_date, [ + service_date, + route_id, + trip_id, + direction_id, + stop_id, + "", # stop_sequence + "", # vehicle_id + "", # vehicle_label + "DEP", + event_time, + ]) + except Exception: + traceback.print_exc() + return output_by_stop_and_day, availability_metadata + +def to_disk(output_by_stop_and_day, root): + for (stop_id, day), rows in output_by_stop_and_day.items(): + day = parse_user_date(day) + destination_dir = pathlib.Path(root, "Events", "daily-data", str(stop_id), f"Year={day.year}", f"Month={day.month}", f"Day={day.day}") + destination_dir.mkdir(parents=True, exist_ok=True) + destination = pathlib.Path(destination_dir, "events.csv.gz") + with open(destination, "wb") as file_out: + with gzip.open(file_out, "wt") as csv_out: + writer = csv.writer(csv_out, delimiter=",") + writer.writerow(CSV_HEADER) + for row in sorted(rows, key=itemgetter(CSV_HEADER.index("event_time"))): + writer.writerow(row) + + +def main(): + if len(sys.argv) < 3: + print("Usage: python3 bus2train.py INPUT_CSV OUTPUT_DIR", file=sys.stderr) + exit(1) + + input_csv = sys.argv[1] + output_dir = sys.argv[2] + pathlib.Path(output_dir).mkdir(exist_ok=True) + + output, availability_metadata = process(input_csv) + to_disk(output, output_dir) + + availability_path = os.path.join(output_dir, AVAILABILITY_FILE) + try: + with open(availability_path, "r") as file: + from_disk = json.load(file) + availability_metadata = merge_availability_dicts(availability_metadata, from_disk) + except FileNotFoundError: + pass + except: + raise + + with open(availability_path, "w", encoding="utf-8") as file: + json.dump(availability_metadata, file, ensure_ascii=False, indent=4) + file.write("\n") + + + +if __name__ == "__main__": + main() From 07cb98aeeee92279bcc703e5c536f1b14b09b4f0 Mon Sep 17 00:00:00 2001 From: Preston Mueller Date: Sat, 7 Aug 2021 16:51:21 -0400 Subject: [PATCH 002/152] lint --- server/bus/bus2train.py | 202 ++++++++++++++++++++-------------------- 1 file changed, 103 insertions(+), 99 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 9a9015c73..ba608d2c0 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -9,15 +9,17 @@ from datetime import date from operator import itemgetter -CSV_HEADER = ["service_date", "route_id", "trip_id", "direction_id", "stop_id", "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] +CSV_HEADER = ["service_date", "route_id", "trip_id", "direction_id", "stop_id", + "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] AVAILABILITY_FILE = "bus_available.json" + def write_row(output_by_stop_and_day, stop_id, day, row): - if (stop_id, day) in output_by_stop_and_day: - output_by_stop_and_day[(stop_id, day)].append(row) - return + if (stop_id, day) in output_by_stop_and_day: + output_by_stop_and_day[(stop_id, day)].append(row) + return - output_by_stop_and_day[(stop_id, day)] = [row] + output_by_stop_and_day[(stop_id, day)] = [row] def parse_user_date(user_date): @@ -27,26 +29,26 @@ def parse_user_date(user_date): def merge_availability_dicts(current: defaultdict, from_disk: dict): - for stop_id, directions in from_disk.items(): - current[stop_id] = list(set(current[stop_id] + directions)) - return current + for stop_id, directions in from_disk.items(): + current[stop_id] = list(set(current[stop_id] + directions)) + return current def process(input_csv): - output_by_stop_and_day = {} - availability_metadata = defaultdict(list) + output_by_stop_and_day = {} + availability_metadata = defaultdict(list) - with open(input_csv, newline='') as csv_in: - reader = csv.DictReader(csv_in) + with open(input_csv, newline='') as csv_in: + reader = csv.DictReader(csv_in) - for row in reader: - try: - if row["standard_type"] != "Headway": - continue - if row["actual"] == "NA": - continue + for row in reader: + try: + if row["standard_type"] != "Headway": + continue + if row["actual"] == "NA": + continue - """ + """ "service_date", "route_id", "direction", "half_trip_id", "stop_id", "time_point_id", "time_point_order", "point_type", "standard_type", "scheduled", "actual", "scheduled_headway", "headway" 2020-01-15, "01", "Inbound", 46374001, 67, "maput", 2, "Midpoint", "Schedule", 1900-01-01 05:08:00, 1900-01-01 05:09:07, -5, NA,NA 2020-01-15, "01", "Inbound", 46374001, 110, "hhgat", 1, "Startpoint", "Schedule", 1900-01-01 05:05:00, 1900-01-01 05:04:34, 26, NA,NA @@ -57,95 +59,97 @@ def process(input_csv): 2020-01-15, "01", "Inbound", 46374045, 110, "hhgat", 1, "Startpoint", "Headway", 1900-01-01 05:20:00, 1900-01-01 05:20:45, NA, 900,971 """ - time = row["actual"].split(" ")[1] - service_date = row["service_date"] - route_id = row["route_id"].lstrip("0") - trip_id = row["half_trip_id"] - direction_id = 0 if row["direction"] == "Outbound" else 1 - stop_id = int(row["stop_id"]) - event_type = "ARR" - event_time = f"{service_date} {time}" - - # Debug override - # if service_date != "2020-01-15" or route_id != "1": - # continue - - if direction_id not in availability_metadata[stop_id]: - availability_metadata[stop_id].append(direction_id) - - # Bus has no delineation between departure and arrival, so we write out a departure row and an arrival row that match, - # so it looks like the rapid transit format - # Write out an arrival row - write_row(output_by_stop_and_day, stop_id, service_date, [ - service_date, - route_id, - trip_id, - direction_id, - stop_id, - "", # stop_sequence - "", # vehicle_id - "", # vehicle_label - event_type, - event_time, - ]) - # Write out a departure row - write_row(output_by_stop_and_day, stop_id, service_date, [ - service_date, - route_id, - trip_id, - direction_id, - stop_id, - "", # stop_sequence - "", # vehicle_id - "", # vehicle_label - "DEP", - event_time, - ]) - except Exception: - traceback.print_exc() - return output_by_stop_and_day, availability_metadata + time = row["actual"].split(" ")[1] + service_date = row["service_date"] + route_id = row["route_id"].lstrip("0") + trip_id = row["half_trip_id"] + direction_id = 0 if row["direction"] == "Outbound" else 1 + stop_id = int(row["stop_id"]) + event_type = "ARR" + event_time = f"{service_date} {time}" + + # Debug override + # if service_date != "2020-01-15" or route_id != "1": + # continue + + if direction_id not in availability_metadata[stop_id]: + availability_metadata[stop_id].append(direction_id) + + # Bus has no delineation between departure and arrival, so we write out a departure row and an arrival row that match, + # so it looks like the rapid transit format + # Write out an arrival row + write_row(output_by_stop_and_day, stop_id, service_date, [ + service_date, + route_id, + trip_id, + direction_id, + stop_id, + "", # stop_sequence + "", # vehicle_id + "", # vehicle_label + event_type, + event_time, + ]) + # Write out a departure row + write_row(output_by_stop_and_day, stop_id, service_date, [ + service_date, + route_id, + trip_id, + direction_id, + stop_id, + "", # stop_sequence + "", # vehicle_id + "", # vehicle_label + "DEP", + event_time, + ]) + except Exception: + traceback.print_exc() + return output_by_stop_and_day, availability_metadata + def to_disk(output_by_stop_and_day, root): - for (stop_id, day), rows in output_by_stop_and_day.items(): - day = parse_user_date(day) - destination_dir = pathlib.Path(root, "Events", "daily-data", str(stop_id), f"Year={day.year}", f"Month={day.month}", f"Day={day.day}") - destination_dir.mkdir(parents=True, exist_ok=True) - destination = pathlib.Path(destination_dir, "events.csv.gz") - with open(destination, "wb") as file_out: - with gzip.open(file_out, "wt") as csv_out: - writer = csv.writer(csv_out, delimiter=",") - writer.writerow(CSV_HEADER) - for row in sorted(rows, key=itemgetter(CSV_HEADER.index("event_time"))): - writer.writerow(row) + for (stop_id, day), rows in output_by_stop_and_day.items(): + day = parse_user_date(day) + destination_dir = pathlib.Path(root, "Events", "daily-data", str( + stop_id), f"Year={day.year}", f"Month={day.month}", f"Day={day.day}") + destination_dir.mkdir(parents=True, exist_ok=True) + destination = pathlib.Path(destination_dir, "events.csv.gz") + with open(destination, "wb") as file_out: + with gzip.open(file_out, "wt") as csv_out: + writer = csv.writer(csv_out, delimiter=",") + writer.writerow(CSV_HEADER) + for row in sorted(rows, key=itemgetter(CSV_HEADER.index("event_time"))): + writer.writerow(row) def main(): - if len(sys.argv) < 3: - print("Usage: python3 bus2train.py INPUT_CSV OUTPUT_DIR", file=sys.stderr) - exit(1) - - input_csv = sys.argv[1] - output_dir = sys.argv[2] - pathlib.Path(output_dir).mkdir(exist_ok=True) + if len(sys.argv) < 3: + print("Usage: python3 bus2train.py INPUT_CSV OUTPUT_DIR", file=sys.stderr) + exit(1) - output, availability_metadata = process(input_csv) - to_disk(output, output_dir) + input_csv = sys.argv[1] + output_dir = sys.argv[2] + pathlib.Path(output_dir).mkdir(exist_ok=True) - availability_path = os.path.join(output_dir, AVAILABILITY_FILE) - try: - with open(availability_path, "r") as file: - from_disk = json.load(file) - availability_metadata = merge_availability_dicts(availability_metadata, from_disk) - except FileNotFoundError: - pass - except: - raise + output, availability_metadata = process(input_csv) + to_disk(output, output_dir) - with open(availability_path, "w", encoding="utf-8") as file: - json.dump(availability_metadata, file, ensure_ascii=False, indent=4) - file.write("\n") + availability_path = os.path.join(output_dir, AVAILABILITY_FILE) + try: + with open(availability_path, "r") as file: + from_disk = json.load(file) + availability_metadata = merge_availability_dicts( + availability_metadata, from_disk) + except FileNotFoundError: + pass + except: + raise + with open(availability_path, "w", encoding="utf-8") as file: + json.dump(availability_metadata, file, ensure_ascii=False, indent=4) + file.write("\n") if __name__ == "__main__": - main() + main() From d16ec376301b2bfaa2117e446fa09237e7211aa3 Mon Sep 17 00:00:00 2001 From: Preston Mueller Date: Sat, 7 Aug 2021 16:56:04 -0400 Subject: [PATCH 003/152] Oops, another linter thing. --- server/bus/bus2train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index ba608d2c0..02da8187e 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -143,7 +143,7 @@ def main(): availability_metadata, from_disk) except FileNotFoundError: pass - except: + except Exception: raise with open(availability_path, "w", encoding="utf-8") as file: From 33844e25d76b33613d006317f33acd0d39cb050f Mon Sep 17 00:00:00 2001 From: Preston Mueller Date: Sun, 22 Aug 2021 13:52:28 -0400 Subject: [PATCH 004/152] Add route level to availability manifest --- server/bus/bus2train.py | 84 +++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 25 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 02da8187e..552e38954 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -5,7 +5,6 @@ import pathlib import sys import traceback -from collections import defaultdict from datetime import date from operator import itemgetter @@ -27,16 +26,61 @@ def parse_user_date(user_date): [year, month, day] = [int(x) for x in date_split[0:3]] return date(year=year, month=month, day=day) +# Read json file of availability into a set() +# Example: +# (These aren't actually stop ids for the 77) +""" +{ + "77": { + "70061": [ + 0, + 1 + ], + "70062": [ + 1 + ] + } +} + + ==> + ("77", "70061", 0), + ("77", "70061", 1), + etc +""" +def json_to_availability(file_path): + try: + with open(file_path, "r") as file: + flat = set() + nested = json.load(file) + for route, stop in nested.items(): + for direction in stop: + flat.add((route, stop, direction)) + return flat + except FileNotFoundError: + return set() + except Exception: + raise -def merge_availability_dicts(current: defaultdict, from_disk: dict): - for stop_id, directions in from_disk.items(): - current[stop_id] = list(set(current[stop_id] + directions)) - return current - - -def process(input_csv): +# Same thing as above but in reverse +def availability_to_json(availability_set): + to_output = {} + for (route, stop, direction) in availability_set: + if route not in to_output: + to_output[route] = { + stop: [direction] + } + continue + if stop not in to_output[route]: + to_output[route][stop] = [direction] + continue + if direction not in to_output[route][stop]: + to_output[route][stop].append(direction) + return to_output + + +def process(input_csv, availability_path): output_by_stop_and_day = {} - availability_metadata = defaultdict(list) + availability_set = json_to_availability(availability_path) with open(input_csv, newline='') as csv_in: reader = csv.DictReader(csv_in) @@ -72,8 +116,8 @@ def process(input_csv): # if service_date != "2020-01-15" or route_id != "1": # continue - if direction_id not in availability_metadata[stop_id]: - availability_metadata[stop_id].append(direction_id) + # I hate this and I'm sorry + availability_set.add((route_id, str(stop_id), direction_id)) # Bus has no delineation between departure and arrival, so we write out a departure row and an arrival row that match, # so it looks like the rapid transit format @@ -105,7 +149,7 @@ def process(input_csv): ]) except Exception: traceback.print_exc() - return output_by_stop_and_day, availability_metadata + return output_by_stop_and_day, availability_set def to_disk(output_by_stop_and_day, root): @@ -132,22 +176,12 @@ def main(): output_dir = sys.argv[2] pathlib.Path(output_dir).mkdir(exist_ok=True) - output, availability_metadata = process(input_csv) - to_disk(output, output_dir) - availability_path = os.path.join(output_dir, AVAILABILITY_FILE) - try: - with open(availability_path, "r") as file: - from_disk = json.load(file) - availability_metadata = merge_availability_dicts( - availability_metadata, from_disk) - except FileNotFoundError: - pass - except Exception: - raise + output, availability_set = process(input_csv, availability_path) + to_disk(output, output_dir) with open(availability_path, "w", encoding="utf-8") as file: - json.dump(availability_metadata, file, ensure_ascii=False, indent=4) + json.dump(availability_to_json(availability_set), file, ensure_ascii=False, indent=4) file.write("\n") From dc8d90fbac9c842b42a44a9dbc2f925aa65e849b Mon Sep 17 00:00:00 2001 From: Preston Mueller Date: Sun, 22 Aug 2021 14:00:48 -0400 Subject: [PATCH 005/152] Debug for route 57 only --- server/bus/bus2train.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 552e38954..fccad81bc 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -115,6 +115,8 @@ def process(input_csv, availability_path): # Debug override # if service_date != "2020-01-15" or route_id != "1": # continue + if route_id != "57": + continue # I hate this and I'm sorry availability_set.add((route_id, str(stop_id), direction_id)) From c6ca52f0462e59da0795f143156cbf26810a4fa0 Mon Sep 17 00:00:00 2001 From: Preston Mueller Date: Sun, 22 Aug 2021 14:42:53 -0400 Subject: [PATCH 006/152] Finagle the 57 in --- server/.chalice/policy.json | 6 +- server/chalicelib/s3.py | 2 +- src/constants.js | 162 +++++++++++++++++++++++++++++++++++- src/line.js | 3 + src/stations.js | 2 +- 5 files changed, 169 insertions(+), 6 deletions(-) diff --git a/server/.chalice/policy.json b/server/.chalice/policy.json index ec1a90ace..487f74172 100644 --- a/server/.chalice/policy.json +++ b/server/.chalice/policy.json @@ -16,7 +16,7 @@ ], "Effect": "Allow", "Resource": [ - "arn:aws:s3:::tm-mbta-performance" + "arn:aws:s3:::tm-mbta-performance-preston-test" ] }, { @@ -25,7 +25,7 @@ ], "Effect": "Allow", "Resource": [ - "arn:aws:s3:::tm-mbta-performance/*" + "arn:aws:s3:::tm-mbta-performance-preston-test/*" ] }, { @@ -34,7 +34,7 @@ ], "Effect": "Allow", "Resource": [ - "arn:aws:s3:::tm-mbta-performance/Alerts/*" + "arn:aws:s3:::tm-mbta-performance-preston-test/Alerts/*" ] } ] diff --git a/server/chalicelib/s3.py b/server/chalicelib/s3.py index 370964d2b..9aa265ee8 100644 --- a/server/chalicelib/s3.py +++ b/server/chalicelib/s3.py @@ -5,7 +5,7 @@ from chalicelib import parallel -BUCKET = "tm-mbta-performance" +BUCKET = "tm-mbta-performance-preston-test" s3 = boto3.client('s3') diff --git a/src/constants.js b/src/constants.js index eac8fbe07..02b358e1b 100644 --- a/src/constants.js +++ b/src/constants.js @@ -3,6 +3,7 @@ export const colorsForLine = { Orange: '#e66f00', Blue: '#0e3d8c', Green: '#159765', + "57": '#ffc72c', }; export const stations = { @@ -1091,7 +1092,166 @@ export const stations = { "northbound": ["70207"], "southbound": ["70208"] } - }] + }], + "57": [ + { + "stop_name": "Tremont St @ Washington St", + "branches": null, + "station": "979", + "order": 1, + "stops": { + "northbound": ["979"], + "southbound": ["912"], + } + }, + { + "stop_name": "Kenmore", + "branches": null, + "station": "place-kencl", + "order": 2, + "stops": { + "northbound": ["899"], + "southbound": ["899"] + } + }, + + { + "stop_name": "Park St @ Tremont St", + "branches": null, + "station": "987", + "order": 3, + "stops": { + "northbound": ["987"] + } + }, + { + "stop_name": "Park Dr @ Fenway Station", + "branches": null, + "station": "1807", + "order": 4, + "stops": { + "northbound": ["1807"] + } + }, + { + "stop_name": "Brighton Ave @ Commonwealth Ave", + "branches": null, + "station": "931", + "order": 5, + "stops": { + "southbound": ["931"] + } + }, + { + "stop_name": "Washington St @ Chestnut Hill Ave", + "branches": null, + "station": "918", + "order": 6, + "stops": { + "southbound": ["918"] + } + }, + { + "stop_name": "1079 Commonwealth Ave", + "branches": null, + "station": "959", + "order": 7, + "stops": { + "northbound": ["959"] + } + }, + { + "stop_name": "Cambridge St @ N Beacon St", + "branches": null, + "station": "966", + "order": 8, + "stops": { + "northbound": ["966"] + } + }, + { + "stop_name": "Watertown Yard", + "branches": null, + "station": "900", + "order": 9, + "stops": { + "northbound": ["900"], + "southbound": ["900"] + } + }, + { + "stop_name": "Washington St @ Bacon St", + "branches": null, + "station": "903", + "order": 10, + "stops": { + "southbound": ["903"] + } + }, + { + "stop_name": "Commonwealth Ave @ Carlton St", + "branches": null, + "station": "937", + "order": 11, + "stops": { + "southbound": ["937"] + } + }, + { + "stop_name": "Commonwealth Ave @ University Rd", + "branches": null, + "station": "954", + "order": 12, + "stops": { + "northbound": ["954"] + } + }, + { + "stop_name": "Washington St @ Oak Sq", + "branches": null, + "station": "9780", + "order": 13, + "stops": { + "northbound": ["9780"] + } + }, + { + "stop_name": "Brighton Ave @ Cambridge St", + "branches": null, + "station": "926", + "order": 14, + "stops": { + "southbound": ["926"] + } + }, + { + "stop_name": "Washington St @ Breck Ave", + "branches": null, + "station": "913", + "order": 15, + "stops": { + "southbound": ["913"] + } + }, + { + "stop_name": "Washington St @ Market St", + "branches": null, + "station": "973", + "order": 16, + "stops": { + "northbound": ["973"] + } + }, + { + "stop_name": "Ave Louis Pasteur @ Longwood Ave", + "branches": null, + "station": "11780", + "order": 17, + "stops": { + "northbound": ["11780"] + } + }, + ] }; const createConfigPresetValue = (line, fromStationName, toStationName, date_start, date_end = undefined) => { diff --git a/src/line.js b/src/line.js index a26231e38..b134f5274 100644 --- a/src/line.js +++ b/src/line.js @@ -164,6 +164,9 @@ class LineClass extends React.Component { display: true, fontSize: 14, labelString: this.props.yFieldLabel + }, + ticks: { + max: 100, } } ], diff --git a/src/stations.js b/src/stations.js index 03c6b98c0..6b099bf2a 100644 --- a/src/stations.js +++ b/src/stations.js @@ -9,7 +9,7 @@ const lookup_station_by_id = (line, id) => { return undefined; } - return stations[line].find(x => [...x.stops.northbound, ...x.stops.southbound].includes(id)); + return stations[line].find(x => [...x.stops.northbound || [], ...x.stops.southbound || []].includes(id)); }; const options_station = (line) => { From acf4b8a7bfef7c74ff16e676975cd6201ff03a5f Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Fri, 17 Sep 2021 17:32:15 -0400 Subject: [PATCH 007/152] rewrite bus2train in pandas --- server/bus/bus2train.py | 194 +++++++++++++++++----------------------- 1 file changed, 84 insertions(+), 110 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index fccad81bc..6d6bd0df2 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -1,31 +1,19 @@ +import argparse import csv import gzip import json import os import pathlib +import pandas as pd import sys import traceback -from datetime import date +from datetime import date, datetime from operator import itemgetter CSV_HEADER = ["service_date", "route_id", "trip_id", "direction_id", "stop_id", "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] AVAILABILITY_FILE = "bus_available.json" - -def write_row(output_by_stop_and_day, stop_id, day, row): - if (stop_id, day) in output_by_stop_and_day: - output_by_stop_and_day[(stop_id, day)].append(row) - return - - output_by_stop_and_day[(stop_id, day)] = [row] - - -def parse_user_date(user_date): - date_split = user_date.split("-") - [year, month, day] = [int(x) for x in date_split[0:3]] - return date(year=year, month=month, day=day) - # Read json file of availability into a set() # Example: # (These aren't actually stop ids for the 77) @@ -78,108 +66,94 @@ def availability_to_json(availability_set): return to_output -def process(input_csv, availability_path): +def process(input_csv, availability_path, routes): output_by_stop_and_day = {} - availability_set = json_to_availability(availability_path) - - with open(input_csv, newline='') as csv_in: - reader = csv.DictReader(csv_in) - - for row in reader: - try: - if row["standard_type"] != "Headway": - continue - if row["actual"] == "NA": - continue - - """ - "service_date", "route_id", "direction", "half_trip_id", "stop_id", "time_point_id", "time_point_order", "point_type", "standard_type", "scheduled", "actual", "scheduled_headway", "headway" - 2020-01-15, "01", "Inbound", 46374001, 67, "maput", 2, "Midpoint", "Schedule", 1900-01-01 05:08:00, 1900-01-01 05:09:07, -5, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 110, "hhgat", 1, "Startpoint", "Schedule", 1900-01-01 05:05:00, 1900-01-01 05:04:34, 26, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 72, "cntsq", 3, "Midpoint", "Schedule", 1900-01-01 05:11:00, 1900-01-01 05:12:01, -22, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 75, "mit", 4, "Midpoint", "Schedule", 1900-01-01 05:14:00, 1900-01-01 05:14:58, -25, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 79, "hynes", 5, "Midpoint", "Schedule", 1900-01-01 05:18:00, 1900-01-01 05:18:45, 32, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 187, "masta", 6, "Midpoint", "Schedule", 1900-01-01 05:20:00, 1900-01-01 05:21:04, -33, NA,NA - 2020-01-15, "01", "Inbound", 46374045, 110, "hhgat", 1, "Startpoint", "Headway", 1900-01-01 05:20:00, 1900-01-01 05:20:45, NA, 900,971 - """ - - time = row["actual"].split(" ")[1] - service_date = row["service_date"] - route_id = row["route_id"].lstrip("0") - trip_id = row["half_trip_id"] - direction_id = 0 if row["direction"] == "Outbound" else 1 - stop_id = int(row["stop_id"]) - event_type = "ARR" - event_time = f"{service_date} {time}" - - # Debug override - # if service_date != "2020-01-15" or route_id != "1": - # continue - if route_id != "57": - continue - - # I hate this and I'm sorry - availability_set.add((route_id, str(stop_id), direction_id)) - - # Bus has no delineation between departure and arrival, so we write out a departure row and an arrival row that match, - # so it looks like the rapid transit format - # Write out an arrival row - write_row(output_by_stop_and_day, stop_id, service_date, [ - service_date, - route_id, - trip_id, - direction_id, - stop_id, - "", # stop_sequence - "", # vehicle_id - "", # vehicle_label - event_type, - event_time, - ]) - # Write out a departure row - write_row(output_by_stop_and_day, stop_id, service_date, [ - service_date, - route_id, - trip_id, - direction_id, - stop_id, - "", # stop_sequence - "", # vehicle_id - "", # vehicle_label - "DEP", - event_time, - ]) - except Exception: - traceback.print_exc() - return output_by_stop_and_day, availability_set - - -def to_disk(output_by_stop_and_day, root): - for (stop_id, day), rows in output_by_stop_and_day.items(): - day = parse_user_date(day) - destination_dir = pathlib.Path(root, "Events", "daily-data", str( - stop_id), f"Year={day.year}", f"Month={day.month}", f"Day={day.day}") - destination_dir.mkdir(parents=True, exist_ok=True) - destination = pathlib.Path(destination_dir, "events.csv.gz") - with open(destination, "wb") as file_out: - with gzip.open(file_out, "wt") as csv_out: - writer = csv.writer(csv_out, delimiter=",") - writer.writerow(CSV_HEADER) - for row in sorted(rows, key=itemgetter(CSV_HEADER.index("event_time"))): - writer.writerow(row) - + # availability_set = json_to_availability(availability_path) + + # thinking about doing this in pandas to have all the info at once + df = pd.read_csv(input_csv, parse_dates=['service_date', 'scheduled', 'actual']) + + df = df.loc[(df.standard_type == "Headway") & (df.actual.notnull())] + df.route_id = df.route_id.str.lstrip('0') + if routes: + df = df.loc[df.route_id.isin(routes)] + + OFFSET = datetime(1900, 1, 1, 0, 0, 0) + df.scheduled = df.service_date + (df.scheduled - OFFSET) + df.actual = df.service_date + (df.actual - OFFSET) + df.service_date = df.service_date.dt.date + + df.direction_id = df.direction_id.map({"Outbound": 0, "Inbound": 1}) + + """ + "service_date", "route_id", "direction", "half_trip_id", "stop_id", "time_point_id", "time_point_order", "point_type", "standard_type", "scheduled", "actual", "scheduled_headway", "headway" + 2020-01-15, "01", "Inbound", 46374001, 67, "maput", 2, "Midpoint", "Schedule", 1900-01-01 05:08:00, 1900-01-01 05:09:07, -5, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 110, "hhgat", 1, "Startpoint", "Schedule", 1900-01-01 05:05:00, 1900-01-01 05:04:34, 26, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 72, "cntsq", 3, "Midpoint", "Schedule", 1900-01-01 05:11:00, 1900-01-01 05:12:01, -22, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 75, "mit", 4, "Midpoint", "Schedule", 1900-01-01 05:14:00, 1900-01-01 05:14:58, -25, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 79, "hynes", 5, "Midpoint", "Schedule", 1900-01-01 05:18:00, 1900-01-01 05:18:45, 32, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 187, "masta", 6, "Midpoint", "Schedule", 1900-01-01 05:20:00, 1900-01-01 05:21:04, -33, NA,NA + 2020-01-15, "01", "Inbound", 46374045, 110, "hhgat", 1, "Startpoint", "Headway", 1900-01-01 05:20:00, 1900-01-01 05:20:45, NA, 900,971 + """ + available = df[['route_id', 'stop_id', 'direction_id']].drop_duplicates().to_records(index=False) + + + CSV_HEADER = ["service_date", "route_id", "trip_id", "direction_id", "stop_id", + "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] + + df = df.rename(columns={'half_trip_id': 'trip_id', + 'time_point_order': 'stop_sequence', + 'actual': 'event_time'}) + df.drop(columns=['time_point_id','standard_type','scheduled','scheduled_headway','headway']) + df['vehicle_id'] = "" + df['vehicle_label'] = "" + + df['event_type'] = df.point_type.map({"Startpoint": ["DEP"], + "Midpoint": ["ARR", "DEP"], + "Endpoint": ["ARR"]}) + df = df.explode('event_type') + df = df[CSV_HEADER] + + return df, [] + return output_by_stop_and_day, [] + + +def write_file(events, outdir): + service_date, stop_id = events.name + + fname = (pathlib.Path(outdir) / + "Events" / + "daily-data" / + str(stop_id) / + f"Year={service_date.year}" / + f"Month={service_date.month}" / + f"Day={service_date.day}" / + "events.csv.gz") + fname.parent.mkdir(parents=True, exist_ok=True) + events.to_csv(fname, index=False, compression='gzip') + + +def to_disk(df, root): + df.groupby(['service_date', 'stop_id']).apply(lambda e: write_file(e, root)) def main(): - if len(sys.argv) < 3: - print("Usage: python3 bus2train.py INPUT_CSV OUTPUT_DIR", file=sys.stderr) - exit(1) - input_csv = sys.argv[1] - output_dir = sys.argv[2] + parser = argparse.ArgumentParser() + + parser.add_argument('input', metavar='INPUT_CSV') + parser.add_argument('output', metavar='OUTPUT_DIR') + parser.add_argument('--routes', '-r', nargs="*", type=str) + + args = parser.parse_args() + + input_csv = args.input + output_dir = args.output + routes = args.routes + pathlib.Path(output_dir).mkdir(exist_ok=True) availability_path = os.path.join(output_dir, AVAILABILITY_FILE) - output, availability_set = process(input_csv, availability_path) + output, availability_set = process(input_csv, availability_path, routes) to_disk(output, output_dir) with open(availability_path, "w", encoding="utf-8") as file: From 918d6ed1e2ae4a72e64760bf072b94b869341261 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Fri, 17 Sep 2021 18:04:04 -0400 Subject: [PATCH 008/152] update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c4084c2d8..43f4b7877 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,8 @@ build .env.production.local .eslintcache +**/data + npm-debug.log* yarn-debug.log* yarn-error.log* From 34476c08d13db7698cf50866c264ccba5fde69a8 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 19 Sep 2021 22:44:02 -0400 Subject: [PATCH 009/152] this is harder than it should be --- server/bus/bus2train.py | 125 ++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 64 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 6d6bd0df2..052d5af97 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -1,6 +1,4 @@ import argparse -import csv -import gzip import json import os import pathlib @@ -14,59 +12,7 @@ "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] AVAILABILITY_FILE = "bus_available.json" -# Read json file of availability into a set() -# Example: -# (These aren't actually stop ids for the 77) -""" -{ - "77": { - "70061": [ - 0, - 1 - ], - "70062": [ - 1 - ] - } -} - - ==> - ("77", "70061", 0), - ("77", "70061", 1), - etc -""" -def json_to_availability(file_path): - try: - with open(file_path, "r") as file: - flat = set() - nested = json.load(file) - for route, stop in nested.items(): - for direction in stop: - flat.add((route, stop, direction)) - return flat - except FileNotFoundError: - return set() - except Exception: - raise - -# Same thing as above but in reverse -def availability_to_json(availability_set): - to_output = {} - for (route, stop, direction) in availability_set: - if route not in to_output: - to_output[route] = { - stop: [direction] - } - continue - if stop not in to_output[route]: - to_output[route][stop] = [direction] - continue - if direction not in to_output[route][stop]: - to_output[route][stop].append(direction) - return to_output - - -def process(input_csv, availability_path, routes): +def load_data(input_csv, routes): output_by_stop_and_day = {} # availability_set = json_to_availability(availability_path) @@ -95,7 +41,49 @@ def process(input_csv, availability_path, routes): 2020-01-15, "01", "Inbound", 46374001, 187, "masta", 6, "Midpoint", "Schedule", 1900-01-01 05:20:00, 1900-01-01 05:21:04, -33, NA,NA 2020-01-15, "01", "Inbound", 46374045, 110, "hhgat", 1, "Startpoint", "Headway", 1900-01-01 05:20:00, 1900-01-01 05:20:45, NA, 900,971 """ - available = df[['route_id', 'stop_id', 'direction_id']].drop_duplicates().to_records(index=False) + + return df + +def load_checkpoints(checkpoint_file): + if checkpoint_file: + return pd.read_csv(checkpoint_file, index_col="checkpoint_id").to_dict()["checkpoint_name"] + else: + return {} + +def create_manifest(df, checkpoint_file): + station_names = load_checkpoints(checkpoint_file) + + manifest = {} + + timepoints = df[['route_id', 'time_point_id', 'stop_id', 'direction_id', 'time_point_order']].drop_duplicates() + timepoints.stop_id = timepoints.stop_id.astype(str) + # timepoints.time_point_order = timepoints.time_point_order.astype(numpy.int32) + + for rte_id, points in timepoints.groupby('route_id'): + summary = [] + for tp_id, info in points.groupby('time_point_id'): + inbound = info.loc[info.direction_id == 1] + outbound = info.loc[info.direction_id == 0] + + # TODO: this assumes a very linear route. + # alternate route patterns and changes BREAK things. + this_obj = { + "stop_name": station_names.get(tp_id.lower(), ""), + "branches": None, + "station": tp_id.lower(), + "order": inbound.time_point_order.tolist()[0], + "stops": { + "inbound": inbound.stop_id.drop_duplicates().tolist(), + "outbound": outbound.stop_id.drop_duplicates().tolist() + } + } + summary.append(this_obj) + + manifest[rte_id] = sorted(summary, key = lambda x: x['order']) + + return manifest + +def process_events(df): CSV_HEADER = ["service_date", "route_id", "trip_id", "direction_id", "stop_id", @@ -114,11 +102,10 @@ def process(input_csv, availability_path, routes): df = df.explode('event_type') df = df[CSV_HEADER] - return df, [] - return output_by_stop_and_day, [] + return df -def write_file(events, outdir): +def _write_file(events, outdir): service_date, stop_id = events.name fname = (pathlib.Path(outdir) / @@ -134,7 +121,7 @@ def write_file(events, outdir): def to_disk(df, root): - df.groupby(['service_date', 'stop_id']).apply(lambda e: write_file(e, root)) + df.groupby(['service_date', 'stop_id']).apply(lambda e: _write_file(e, root)) def main(): @@ -142,24 +129,34 @@ def main(): parser.add_argument('input', metavar='INPUT_CSV') parser.add_argument('output', metavar='OUTPUT_DIR') + parser.add_argument('--checkpoints', metavar="checkpoints.txt") parser.add_argument('--routes', '-r', nargs="*", type=str) + args = parser.parse_args() input_csv = args.input output_dir = args.output + checkpoint_file = args.checkpoints routes = args.routes pathlib.Path(output_dir).mkdir(exist_ok=True) availability_path = os.path.join(output_dir, AVAILABILITY_FILE) - output, availability_set = process(input_csv, availability_path, routes) - to_disk(output, output_dir) + data = load_data(input_csv, routes) + events = process_events(data) + to_disk(events, output_dir) + + # do something with available and data + manifest = create_manifest(data, checkpoint_file) + with open(availability_path, "w", encoding="utf-8") as fd: + json.dump(manifest, fd, ensure_ascii=False, indent=4) + fd.write("\n") - with open(availability_path, "w", encoding="utf-8") as file: +""" with open(availability_path, "w", encoding="utf-8") as file: json.dump(availability_to_json(availability_set), file, ensure_ascii=False, indent=4) file.write("\n") - + """ if __name__ == "__main__": main() From 683cd1df911a2aab93a524246925d9a78dcd8f0a Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Mon, 20 Sep 2021 20:54:06 -0400 Subject: [PATCH 010/152] remove fixed y-max --- src/line.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/line.js b/src/line.js index b134f5274..a26231e38 100644 --- a/src/line.js +++ b/src/line.js @@ -164,9 +164,6 @@ class LineClass extends React.Component { display: true, fontSize: 14, labelString: this.props.yFieldLabel - }, - ticks: { - max: 100, } } ], From ab15c7ac58389f2e55f4eb17b8072bac589ff47a Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 23 Sep 2021 22:12:43 -0400 Subject: [PATCH 011/152] flexible direction words --- src/App.js | 5 +- src/constants.js | 2365 ++++++++++++++++++++++------------------------ src/stations.js | 12 +- 3 files changed, 1130 insertions(+), 1252 deletions(-) diff --git a/src/App.js b/src/App.js index 51b7b58ce..72f7afffa 100644 --- a/src/App.js +++ b/src/App.js @@ -184,10 +184,11 @@ class App extends React.Component { date_start, date_end, } = this.state.configuration; + const { fromStopIds, toStopIds } = get_stop_ids_for_stations(from, to); const parts = [ line, - from?.stops.southbound, - to?.stops.southbound, + fromStopIds?.[0], + toStopIds?.[0], date_start, date_end, ].map(x => x || "").join(","); diff --git a/src/constants.js b/src/constants.js index 02b358e1b..ddecb48b5 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,1262 +1,1139 @@ export const colorsForLine = { - Red: '#d13434', - Orange: '#e66f00', - Blue: '#0e3d8c', - Green: '#159765', - "57": '#ffc72c', + Red: '#d13434', + Orange: '#e66f00', + Blue: '#0e3d8c', + Green: '#159765', + "57": '#ffc72c', }; export const stations = { - "Red": [{ - "stop_name": "Alewife", - "branches": ["A", "B"], - "station": "place-alfcl", - "order": 1, - "stops": { - "northbound": ["70061"], - "southbound": ["70061"] - } - }, { - "stop_name": "Davis", - "branches": ["A", "B"], - "station": "place-davis", - "order": 2, - "stops": { - "northbound": ["70064"], - "southbound": ["70063"] - } - }, { - "stop_name": "Porter", - "branches": ["A", "B"], - "station": "place-portr", - "order": 3, - "stops": { - "northbound": ["70066"], - "southbound": ["70065"] - } - }, { - "stop_name": "Harvard", - "branches": ["A", "B"], - "station": "place-harsq", - "order": 4, - "stops": { - "northbound": ["70068"], - "southbound": ["70067"] - } - }, { - "stop_name": "Central", - "branches": ["A", "B"], - "station": "place-cntsq", - "order": 5, - "stops": { - "northbound": ["70070"], - "southbound": ["70069"] - } - }, { - "stop_name": "Kendall/MIT", - "branches": ["A", "B"], - "station": "place-knncl", - "order": 6, - "stops": { - "northbound": ["70072"], - "southbound": ["70071"] - } - }, { - "stop_name": "Charles/MGH", - "branches": ["A", "B"], - "station": "place-chmnl", - "order": 7, - "stops": { - "northbound": ["70074"], - "southbound": ["70073"] - } - }, { - "stop_name": "Park Street", - "branches": ["A", "B"], - "station": "place-pktrm", - "order": 8, - "stops": { - "northbound": ["70076"], - "southbound": ["70075"] - } - }, { - "stop_name": "Downtown Crossing", - "branches": ["A", "B"], - "station": "place-dwnxg", - "order": 9, - "stops": { - "northbound": ["70078"], - "southbound": ["70077"] - } - }, { - "stop_name": "South Station", - "branches": ["A", "B"], - "station": "place-sstat", - "order": 10, - "stops": { - "northbound": ["70080"], - "southbound": ["70079"] - } - }, { - "stop_name": "Broadway", - "branches": ["A", "B"], - "station": "place-brdwy", - "order": 11, - "stops": { - "northbound": ["70082"], - "southbound": ["70081"] - } - }, { - "stop_name": "Andrew", - "branches": ["A", "B"], - "station": "place-andrw", - "order": 12, - "stops": { - "northbound": ["70084"], - "southbound": ["70083"] - } - }, { - "stop_name": "JFK/UMass (Ashmont)", - "branches": ["A"], - "station": "place-jfk", - "order": 101, - "stops": { - "northbound": ["70086"], - "southbound": ["70085"] - } - }, { - "stop_name": "Savin Hill", - "branches": ["A"], - "station": "place-shmnl", - "order": 102, - "stops": { - "northbound": ["70088"], - "southbound": ["70087"] - } - }, { - "stop_name": "Fields Corner", - "branches": ["A"], - "station": "place-fldcr", - "order": 103, - "stops": { - "northbound": ["70090"], - "southbound": ["70089"] - } - }, { - "stop_name": "Shawmut", - "branches": ["A"], - "station": "place-smmnl", - "order": 104, - "stops": { - "northbound": ["70092"], - "southbound": ["70091"] - } - }, { - "stop_name": "Ashmont", - "branches": ["A"], - "station": "place-asmnl", - "order": 105, - "stops": { - "northbound": ["70094"], - "southbound": ["70093"] - } - }, { - "stop_name": "JFK/UMass (Braintree)", - "branches": ["B"], - "station": "place-jfk", - "order": 201, - "stops": { - "northbound": ["70096"], - "southbound": ["70095"] - } - }, { - "stop_name": "North Quincy", - "branches": ["B"], - "station": "place-nqncy", - "order": 202, - "stops": { - "northbound": ["70098"], - "southbound": ["70097"] - } - }, { - "stop_name": "Quincy Center", - "branches": ["B"], - "station": "place-qnctr", - "order": 203, - "stops": { - "northbound": ["70102"], - "southbound": ["70101"] - } - }, { - "stop_name": "Quincy Adams", - "branches": ["B"], - "station": "place-qamnl", - "order": 204, - "stops": { - "northbound": ["70104"], - "southbound": ["70103"] - } - }, { - "stop_name": "Braintree", - "branches": ["B"], - "station": "place-brntn", - "order": 205, - "stops": { - "northbound": ["70105"], - "southbound": ["70105"] - } - }], - "Orange": [{ - "stop_name": "Oak Grove", - "branches": null, - "station": "place-ogmnl", - "order": 1, - "stops": { - "northbound": ["70036"], - "southbound": ["70036"] - } - }, { - "stop_name": "Malden Center", - "branches": null, - "station": "place-mlmnl", - "order": 2, - "stops": { - "northbound": ["70035"], - "southbound": ["70034"] - } - }, { - "stop_name": "Wellington", - "branches": null, - "station": "place-welln", - "order": 3, - "stops": { - "northbound": ["70033"], - "southbound": ["70032"] - } - }, { - "stop_name": "Assembly", - "branches": null, - "station": "place-astao", - "order": 4, - "stops": { - "northbound": ["70279"], - "southbound": ["70278"] - } - }, { - "stop_name": "Sullivan Square", - "branches": null, - "station": "place-sull", - "order": 5, - "stops": { - "northbound": ["70031"], - "southbound": ["70030"] - } - }, { - "stop_name": "Community College", - "branches": null, - "station": "place-ccmnl", - "order": 6, - "stops": { - "northbound": ["70029"], - "southbound": ["70028"] - } - }, { - "stop_name": "North Station", - "branches": null, - "station": "place-north", - "order": 7, - "stops": { - "northbound": ["70027"], - "southbound": ["70026"] - } - }, { - "stop_name": "Haymarket", - "branches": null, - "station": "place-haecl", - "order": 8, - "stops": { - "northbound": ["70025"], - "southbound": ["70024"] - } - }, { - "stop_name": "State Street", - "branches": null, - "station": "place-state", - "order": 9, - "stops": { - "northbound": ["70023"], - "southbound": ["70022"] - } - }, { - "stop_name": "Downtown Crossing", - "branches": null, - "station": "place-dwnxg", - "order": 10, - "stops": { - "northbound": ["70021"], - "southbound": ["70020"] - } - }, { - "stop_name": "Chinatown", - "branches": null, - "station": "place-chncl", - "order": 11, - "stops": { - "northbound": ["70019"], - "southbound": ["70018"] - } - }, { - "stop_name": "Tufts Medical Center", - "branches": null, - "station": "place-tumnl", - "order": 12, - "stops": { - "northbound": ["70017"], - "southbound": ["70016"] - } - }, { - "stop_name": "Back Bay", - "branches": null, - "station": "place-bbsta", - "order": 13, - "stops": { - "northbound": ["70015"], - "southbound": ["70014"] - } - }, { - "stop_name": "Massachusetts Avenue", - "branches": null, - "station": "place-masta", - "order": 14, - "stops": { - "northbound": ["70013"], - "southbound": ["70012"] - } - }, { - "stop_name": "Ruggles", - "branches": null, - "station": "place-rugg", - "order": 15, - "stops": { - "northbound": ["70011"], - "southbound": ["70010"] - } - }, { - "stop_name": "Roxbury Crossing", - "branches": null, - "station": "place-rcmnl", - "order": 16, - "stops": { - "northbound": ["70009"], - "southbound": ["70008"] - } - }, { - "stop_name": "Jackson Square", - "branches": null, - "station": "place-jaksn", - "order": 17, - "stops": { - "northbound": ["70007"], - "southbound": ["70006"] - } - }, { - "stop_name": "Stony Brook", - "branches": null, - "station": "place-sbmnl", - "order": 18, - "stops": { - "northbound": ["70005"], - "southbound": ["70004"] - } - }, { - "stop_name": "Green Street", - "branches": null, - "station": "place-grnst", - "order": 19, - "stops": { - "northbound": ["70003"], - "southbound": ["70002"] - } - }, { - "stop_name": "Forest Hills", - "branches": null, - "station": "place-forhl", - "order": 20, - "stops": { - "northbound": ["70001"], - "southbound": ["70001"] - } - }], - "Blue": [{ - "stop_name": "Wonderland", - "branches": null, - "station": "place-wondl", - "order": 1, - "stops": { - "northbound": ["70060"], - "southbound": ["70059"] - } - }, { - "stop_name": "Revere Beach", - "branches": null, - "station": "place-rbmnl", - "order": 2, - "stops": { - "northbound": ["70058"], - "southbound": ["70057"] - } - }, { - "stop_name": "Beachmont", - "branches": null, - "station": "place-bmmnl", - "order": 3, - "stops": { - "northbound": ["70056"], - "southbound": ["70055"] - } - }, { - "stop_name": "Suffolk Downs", - "branches": null, - "station": "place-sdmnl", - "order": 4, - "stops": { - "northbound": ["70054"], - "southbound": ["70053"] - } - }, { - "stop_name": "Orient Heights", - "branches": null, - "station": "place-orhte", - "order": 5, - "stops": { - "northbound": ["70052"], - "southbound": ["70051"] - } - }, { - "stop_name": "Wood Island", - "branches": null, - "station": "place-wimnl", - "order": 6, - "stops": { - "northbound": ["70050"], - "southbound": ["70049"] - } - }, { - "stop_name": "Airport", - "branches": null, - "station": "place-aport", - "order": 7, - "stops": { - "northbound": ["70048"], - "southbound": ["70047"] - } - }, { - "stop_name": "Maverick", - "branches": null, - "station": "place-mvbcl", - "order": 8, - "stops": { - "northbound": ["70046"], - "southbound": ["70045"] - } - }, { - "stop_name": "Aquarium", - "branches": null, - "station": "place-aqucl", - "order": 9, - "stops": { - "northbound": ["70044"], - "southbound": ["70043"] - } - }, { - "stop_name": "State Street", - "branches": null, - "station": "place-state", - "order": 10, - "stops": { - "northbound": ["70042"], - "southbound": ["70041"] - } - }, { - "stop_name": "Government Center", - "branches": null, - "station": "place-gover", - "order": 11, - "stops": { - "northbound": ["70040"], - "southbound": ["70039"] - } - }, { - "stop_name": "Bowdoin", - "branches": null, - "station": "place-bomnl", - "order": 12, - "stops": { - "northbound": ["70038"], - "southbound": ["70838"] - } - }], - "Green": [{ - "stop_name": "Blandford Street", - "branches": ["B"], - "station": "place-bland", - "order": 101, - "stops": { - "northbound": ["70148"], - "southbound": ["70149"] - } - }, { - "stop_name": "Boston University East", - "branches": ["B"], - "station": "place-buest", - "order": 102, - "stops": { - "northbound": ["70146"], - "southbound": ["70147"] - } - }, { - "stop_name": "Boston University Central", - "branches": ["B"], - "station": "place-bucen", - "order": 103, - "stops": { - "northbound": ["70144"], - "southbound": ["70145"] - } - }, { - "stop_name": "Boston University West", - "branches": ["B"], - "station": "place-buwst", - "order": 104, - "stops": { - "northbound": ["70142"], - "southbound": ["70143"] - } - }, { - "stop_name": "Saint Paul Street (B)", - "branches": ["B"], - "station": "place-stplb", - "order": 105, - "stops": { - "northbound": ["70140"], - "southbound": ["70141"] - } - }, { - "stop_name": "Pleasant Street", - "branches": ["B"], - "station": "place-plsgr", - "order": 106, - "stops": { - "northbound": ["70138"], - "southbound": ["70139"] - } - }, { - "stop_name": "Babcock Street", - "branches": ["B"], - "station": "place-babck", - "order": 107, - "stops": { - "northbound": ["70136"], - "southbound": ["70137"] - } - }, { - "stop_name": "Packards Corner", - "branches": ["B"], - "station": "place-brico", - "order": 108, - "stops": { - "northbound": ["70134"], - "southbound": ["70135"] - } - }, { - "stop_name": "Harvard Avenue", - "branches": ["B"], - "station": "place-harvd", - "order": 109, - "stops": { - "northbound": ["70130"], - "southbound": ["70131"] - } - }, { - "stop_name": "Griggs Street", - "branches": ["B"], - "station": "place-grigg", - "order": 110, - "stops": { - "northbound": ["70128"], - "southbound": ["70129"] - } - }, { - "stop_name": "Allston Street", - "branches": ["B"], - "station": "place-alsgr", - "order": 111, - "stops": { - "northbound": ["70126"], - "southbound": ["70127"] - } - }, { - "stop_name": "Warren Street", - "branches": ["B"], - "station": "place-wrnst", - "order": 112, - "stops": { - "northbound": ["70124"], - "southbound": ["70125"] - } - }, { - "stop_name": "Washington Street", - "branches": ["B"], - "station": "place-wascm", - "order": 113, - "stops": { - "northbound": ["70120"], - "southbound": ["70121"] - } - }, { - "stop_name": "Sutherland Road", - "branches": ["B"], - "station": "place-sthld", - "order": 114, - "stops": { - "northbound": ["70116"], - "southbound": ["70117"] - } - }, { - "stop_name": "Chiswick Road", - "branches": ["B"], - "station": "place-chswk", - "order": 115, - "stops": { - "northbound": ["70114"], - "southbound": ["70115"] - } - }, { - "stop_name": "Chestnut Hill Avenue", - "branches": ["B"], - "station": "place-chill", - "order": 116, - "stops": { - "northbound": ["70112"], - "southbound": ["70113"] - } - }, { - "stop_name": "South Street", - "branches": ["B"], - "station": "place-sougr", - "order": 117, - "stops": { - "northbound": ["70110"], - "southbound": ["70111"] - } - }, { - "stop_name": "Boston College", - "branches": ["B"], - "station": "place-lake", - "order": 118, - "stops": { - "northbound": ["70106"], - "southbound": ["70107"] - } - }, { - "stop_name": "Kenmore", - "branches": ["B", "C", "D"], - "station": "place-kencl", - disabled: true, - "order": 11, - "stops": { - "northbound": ["70150", "71150"], - "southbound": ["70151", "71151"] - } - }, { - "stop_name": "Hynes", - "branches": ["B", "C", "D"], - "station": "place-hymnl", - "order": 10, - "stops": { - "northbound": ["70152"], - "southbound": ["70153"] - } - }, { - "stop_name": "Park Street", - "branches": ["B", "C", "D", "E"], - "station": "place-pktrm", - disabled: true, - "order": 6, - "stops": { - "northbound": ["70200"], - "southbound": ["70196", "70197", "70198", "70199"] - } - }, { - "stop_name": "Boylston", - "branches": ["B", "C", "D", "E"], - "station": "place-boyls", - "order": 7, - "stops": { - "northbound": ["70158"], - "southbound": ["70159"] - } - }, { - "stop_name": "Arlington", - "branches": ["B", "C", "D", "E"], - "station": "place-armnl", - "order": 8, - "stops": { - "northbound": ["70156"], - "southbound": ["70157"] - } - }, { - "stop_name": "Copley", - "branches": ["B", "C", "D", "E"], - "station": "place-coecl", - "order": 9, - "stops": { - "northbound": ["70154"], - "southbound": ["70155"] - } - }, { - "stop_name": "Cleveland Circle", - "branches": ["C"], - "station": "place-clmnl", - "order": 213, - "stops": { - "northbound": ["70238"], - "southbound": ["70237"] - } - }, { - "stop_name": "Englewood Avenue", - "branches": ["C"], - "station": "place-engav", - "order": 212, - "stops": { - "northbound": ["70236"], - "southbound": ["70235"] - } - }, { - "stop_name": "Dean Road", - "branches": ["C"], - "station": "place-denrd", - "order": 211, - "stops": { - "northbound": ["70234"], - "southbound": ["70233"] - } - }, { - "stop_name": "Tappan Street", - "branches": ["C"], - "station": "place-tapst", - "order": 210, - "stops": { - "northbound": ["70232"], - "southbound": ["70231"] - } - }, { - "stop_name": "Washington Square", - "branches": ["C"], - "station": "place-bcnwa", - "order": 209, - "stops": { - "northbound": ["70230"], - "southbound": ["70229"] - } - }, { - "stop_name": "Fairbanks Street", - "branches": ["C"], - "station": "place-fbkst", - "order": 208, - "stops": { - "northbound": ["70228"], - "southbound": ["70227"] - } - }, { - "stop_name": "Brandon Hall", - "branches": ["C"], - "station": "place-bndhl", - "order": 207, - "stops": { - "northbound": ["70226"], - "southbound": ["70225"] - } - }, { - "stop_name": "Summit Avenue", - "branches": ["C"], - "station": "place-sumav", - "order": 206, - "stops": { - "northbound": ["70224"], - "southbound": ["70223"] - } - }, { - "stop_name": "Coolidge Corner", - "branches": ["C"], - "station": "place-cool", - "order": 205, - "stops": { - "northbound": ["70220"], - "southbound": ["70219"] - } - }, { - "stop_name": "Saint Paul Street (C)", - "branches": ["C"], - "station": "place-stpul", - "order": 204, - "stops": { - "northbound": ["70218"], - "southbound": ["70217"] - } - }, { - "stop_name": "Kent Street", - "branches": ["C"], - "station": "place-kntst", - "order": 203, - "stops": { - "northbound": ["70216"], - "southbound": ["70215"] - } - }, { - "stop_name": "Hawes Street", - "branches": ["C"], - "station": "place-hwsst", - "order": 202, - "stops": { - "northbound": ["70214"], - "southbound": ["70213"] - } - }, { - "stop_name": "Saint Marys Street", - "branches": ["C"], - "station": "place-smary", - "order": 201, - "stops": { - "northbound": ["70212"], - "southbound": ["70211"] - } - }, { - "stop_name": "Government Center", - "branches": ["C", "D", "E"], - "station": "place-gover", - "order": 5, - "stops": { - "northbound": ["70201"], - "southbound": ["70202"] - } - }, { - "stop_name": "North Station", - "branches": ["C", "E"], - "station": "place-north", - "order": 3, - "stops": { - "northbound": ["70205"], - "southbound": ["70206"] - } - }, { - "stop_name": "Haymarket", - "branches": ["C", "E"], - "station": "place-haecl", - "order": 4, - "stops": { - "northbound": ["70203"], - "southbound": ["70204"] - } - }, { - "stop_name": "Fenway", - "branches": ["D"], - "station": "place-fenwy", - "order": 301, - "stops": { - "northbound": ["70186"], - "southbound": ["70187"] - } - }, { - "stop_name": "Longwood", - "branches": ["D"], - "station": "place-longw", - "order": 302, - "stops": { - "northbound": ["70182"], - "southbound": ["70183"] - } - }, { - "stop_name": "Brookline Village", - "branches": ["D"], - "station": "place-bvmnl", - "order": 303, - "stops": { - "northbound": ["70180"], - "southbound": ["70181"] - } - }, { - "stop_name": "Brookline Hills", - "branches": ["D"], - "station": "place-brkhl", - "order": 304, - "stops": { - "northbound": ["70178"], - "southbound": ["70179"] - } - }, { - "stop_name": "Beaconsfield", - "branches": ["D"], - "station": "place-bcnfd", - "order": 305, - "stops": { - "northbound": ["70176"], - "southbound": ["70177"] - } - }, { - "stop_name": "Reservoir", - "branches": ["D"], - "station": "place-rsmnl", - "order": 306, - "stops": { - "northbound": ["70174"], - "southbound": ["70175"] - } - }, { - "stop_name": "Chestnut Hill", - "branches": ["D"], - "station": "place-chhil", - "order": 307, - "stops": { - "northbound": ["70172"], - "southbound": ["70173"] - } - }, { - "stop_name": "Newton Centre", - "branches": ["D"], - "station": "place-newto", - "order": 308, - "stops": { - "northbound": ["70170"], - "southbound": ["70171"] - } - }, { - "stop_name": "Newton Highlands", - "branches": ["D"], - "station": "place-newtn", - "order": 309, - "stops": { - "northbound": ["70168"], - "southbound": ["70169"] - } - }, { - "stop_name": "Eliot", - "branches": ["D"], - "station": "place-eliot", - "order": 310, - "stops": { - "northbound": ["70166"], - "southbound": ["70167"] - } - }, { - "stop_name": "Waban", - "branches": ["D"], - "station": "place-waban", - "order": 311, - "stops": { - "northbound": ["70164"], - "southbound": ["70165"] - } - }, { - "stop_name": "Woodland", - "branches": ["D"], - "station": "place-woodl", - "order": 312, - "stops": { - "northbound": ["70162"], - "southbound": ["70163"] - } - }, { - "stop_name": "Riverside", - "branches": ["D"], - "station": "place-river", - "order": 313, - "stops": { - "northbound": ["70160"], - "southbound": ["70161"] - } - }, { - "stop_name": "Heath Street", - "branches": ["E"], - "station": "place-hsmnl", - "order": 411, - "stops": { - "northbound": ["70260"], - "southbound": ["70260"] - } - }, { - "stop_name": "Back of the Hill", - "branches": ["E"], - "station": "place-bckhl", - "order": 410, - "stops": { - "northbound": ["70258"], - "southbound": ["70257"] - } - }, { - "stop_name": "Riverway", - "branches": ["E"], - "station": "place-rvrwy", - "order": 409, - "stops": { - "northbound": ["70256"], - "southbound": ["70255"] - } - }, { - "stop_name": "Mission Park", - "branches": ["E"], - "station": "place-mispk", - "order": 408, - "stops": { - "northbound": ["70254"], - "southbound": ["70253"] - } - }, { - "stop_name": "Fenwood Road", - "branches": ["E"], - "station": "place-fenwd", - "order": 407, - "stops": { - "northbound": ["70252"], - "southbound": ["70251"] - } - }, { - "stop_name": "Brigham Circle", - "branches": ["E"], - "station": "place-brmnl", - "order": 406, - "stops": { - "northbound": ["70250"], - "southbound": ["70249"] - } - }, { - "stop_name": "Longwood Medical Area", - "branches": ["E"], - "station": "place-lngmd", - "order": 405, - "stops": { - "northbound": ["70248"], - "southbound": ["70247"] - } - }, { - "stop_name": "Museum of Fine Arts", - "branches": ["E"], - "station": "place-mfa", - "order": 404, - "stops": { - "northbound": ["70246"], - "southbound": ["70245"] - } - }, { - "stop_name": "Northeastern University", - "branches": ["E"], - "station": "place-nuniv", - "order": 403, - "stops": { - "northbound": ["70244"], - "southbound": ["70243"] - } - }, { - "stop_name": "Symphony", - "branches": ["E"], - "station": "place-symcl", - "order": 402, - "stops": { - "northbound": ["70242"], - "southbound": ["70241"] - } - }, { - "stop_name": "Prudential", - "branches": ["E"], - "station": "place-prmnl", - "order": 401, - "stops": { - "northbound": ["70240"], - "southbound": ["70239"] - } - }, { - "stop_name": "Lechmere", - "branches": ["E"], - "station": "place-lech", - "order": 1, - "stops": { - "northbound": ["70209"], - "southbound": ["70210"] - } - }, { - "stop_name": "Science Park", - "branches": ["E"], - "station": "place-spmnl", - "order": 2, - "stops": { - "northbound": ["70207"], - "southbound": ["70208"] - } - }], - "57": [ - { - "stop_name": "Tremont St @ Washington St", - "branches": null, - "station": "979", - "order": 1, - "stops": { - "northbound": ["979"], - "southbound": ["912"], - } - }, - { - "stop_name": "Kenmore", - "branches": null, - "station": "place-kencl", - "order": 2, - "stops": { - "northbound": ["899"], - "southbound": ["899"] - } - }, - - { - "stop_name": "Park St @ Tremont St", - "branches": null, - "station": "987", - "order": 3, - "stops": { - "northbound": ["987"] - } - }, - { - "stop_name": "Park Dr @ Fenway Station", - "branches": null, - "station": "1807", - "order": 4, - "stops": { - "northbound": ["1807"] - } - }, - { - "stop_name": "Brighton Ave @ Commonwealth Ave", - "branches": null, - "station": "931", - "order": 5, - "stops": { - "southbound": ["931"] - } - }, - { - "stop_name": "Washington St @ Chestnut Hill Ave", - "branches": null, - "station": "918", - "order": 6, - "stops": { - "southbound": ["918"] - } - }, - { - "stop_name": "1079 Commonwealth Ave", - "branches": null, - "station": "959", - "order": 7, - "stops": { - "northbound": ["959"] - } - }, - { - "stop_name": "Cambridge St @ N Beacon St", - "branches": null, - "station": "966", - "order": 8, - "stops": { - "northbound": ["966"] - } + "Red": { + "type": "rapidtransit", + "direction": { + "0": "northbound", + "1": "southbound" }, - { - "stop_name": "Watertown Yard", - "branches": null, - "station": "900", - "order": 9, - "stops": { - "northbound": ["900"], - "southbound": ["900"] - } - }, - { - "stop_name": "Washington St @ Bacon St", - "branches": null, - "station": "903", - "order": 10, - "stops": { - "southbound": ["903"] - } - }, - { - "stop_name": "Commonwealth Ave @ Carlton St", - "branches": null, - "station": "937", - "order": 11, - "stops": { - "southbound": ["937"] - } - }, - { - "stop_name": "Commonwealth Ave @ University Rd", - "branches": null, - "station": "954", - "order": 12, - "stops": { - "northbound": ["954"] - } - }, - { - "stop_name": "Washington St @ Oak Sq", - "branches": null, - "station": "9780", - "order": 13, - "stops": { - "northbound": ["9780"] - } - }, - { - "stop_name": "Brighton Ave @ Cambridge St", - "branches": null, - "station": "926", - "order": 14, - "stops": { - "southbound": ["926"] + "stations": [ + { + "stop_name": "Alewife", + "branches": ["A", "B"], + "station": "place-alfcl", + "order": 1, + "stops": { + "0": ["70061"], + "1": ["70061"] + } + }, { + "stop_name": "Davis", + "branches": ["A", "B"], + "station": "place-davis", + "order": 2, + "stops": { + "0": ["70064"], + "1": ["70063"] + } + }, { + "stop_name": "Porter", + "branches": ["A", "B"], + "station": "place-portr", + "order": 3, + "stops": { + "0": ["70066"], + "1": ["70065"] + } + }, { + "stop_name": "Harvard", + "branches": ["A", "B"], + "station": "place-harsq", + "order": 4, + "stops": { + "0": ["70068"], + "1": ["70067"] + } + }, { + "stop_name": "Central", + "branches": ["A", "B"], + "station": "place-cntsq", + "order": 5, + "stops": { + "0": ["70070"], + "1": ["70069"] + } + }, { + "stop_name": "Kendall/MIT", + "branches": ["A", "B"], + "station": "place-knncl", + "order": 6, + "stops": { + "0": ["70072"], + "1": ["70071"] + } + }, { + "stop_name": "Charles/MGH", + "branches": ["A", "B"], + "station": "place-chmnl", + "order": 7, + "stops": { + "0": ["70074"], + "1": ["70073"] + } + }, { + "stop_name": "Park Street", + "branches": ["A", "B"], + "station": "place-pktrm", + "order": 8, + "stops": { + "0": ["70076"], + "1": ["70075"] + } + }, { + "stop_name": "Downtown Crossing", + "branches": ["A", "B"], + "station": "place-dwnxg", + "order": 9, + "stops": { + "0": ["70078"], + "1": ["70077"] + } + }, { + "stop_name": "South Station", + "branches": ["A", "B"], + "station": "place-sstat", + "order": 10, + "stops": { + "0": ["70080"], + "1": ["70079"] + } + }, { + "stop_name": "Broadway", + "branches": ["A", "B"], + "station": "place-brdwy", + "order": 11, + "stops": { + "0": ["70082"], + "1": ["70081"] + } + }, { + "stop_name": "Andrew", + "branches": ["A", "B"], + "station": "place-andrw", + "order": 12, + "stops": { + "0": ["70084"], + "1": ["70083"] + } + }, { + "stop_name": "JFK/UMass (Ashmont)", + "branches": ["A"], + "station": "place-jfk", + "order": 101, + "stops": { + "0": ["70086"], + "1": ["70085"] + } + }, { + "stop_name": "Savin Hill", + "branches": ["A"], + "station": "place-shmnl", + "order": 102, + "stops": { + "0": ["70088"], + "1": ["70087"] + } + }, { + "stop_name": "Fields Corner", + "branches": ["A"], + "station": "place-fldcr", + "order": 103, + "stops": { + "0": ["70090"], + "1": ["70089"] + } + }, { + "stop_name": "Shawmut", + "branches": ["A"], + "station": "place-smmnl", + "order": 104, + "stops": { + "0": ["70092"], + "1": ["70091"] + } + }, { + "stop_name": "Ashmont", + "branches": ["A"], + "station": "place-asmnl", + "order": 105, + "stops": { + "0": ["70094"], + "1": ["70093"] + } + }, { + "stop_name": "JFK/UMass (Braintree)", + "branches": ["B"], + "station": "place-jfk", + "order": 201, + "stops": { + "0": ["70096"], + "1": ["70095"] + } + }, { + "stop_name": "North Quincy", + "branches": ["B"], + "station": "place-nqncy", + "order": 202, + "stops": { + "0": ["70098"], + "1": ["70097"] + } + }, { + "stop_name": "Quincy Center", + "branches": ["B"], + "station": "place-qnctr", + "order": 203, + "stops": { + "0": ["70102"], + "1": ["70101"] + } + }, { + "stop_name": "Quincy Adams", + "branches": ["B"], + "station": "place-qamnl", + "order": 204, + "stops": { + "0": ["70104"], + "1": ["70103"] + } + }, { + "stop_name": "Braintree", + "branches": ["B"], + "station": "place-brntn", + "order": 205, + "stops": { + "0": ["70105"], + "1": ["70105"] + } } + ] + }, + "Orange": { + "type": "rapidtransit", + "direction": { + "0": "northbound", + "1": "southbound" }, - { - "stop_name": "Washington St @ Breck Ave", - "branches": null, - "station": "913", - "order": 15, - "stops": { - "southbound": ["913"] + "stations": [ + { + "stop_name": "Oak Grove", + "branches": null, + "station": "place-ogmnl", + "order": 1, + "stops": { + "0": ["70036"], + "1": ["70036"] + } + }, { + "stop_name": "Malden Center", + "branches": null, + "station": "place-mlmnl", + "order": 2, + "stops": { + "0": ["70035"], + "1": ["70034"] + } + }, { + "stop_name": "Wellington", + "branches": null, + "station": "place-welln", + "order": 3, + "stops": { + "0": ["70033"], + "1": ["70032"] + } + }, { + "stop_name": "Assembly", + "branches": null, + "station": "place-astao", + "order": 4, + "stops": { + "0": ["70279"], + "1": ["70278"] + } + }, { + "stop_name": "Sullivan Square", + "branches": null, + "station": "place-sull", + "order": 5, + "stops": { + "0": ["70031"], + "1": ["70030"] + } + }, { + "stop_name": "Community College", + "branches": null, + "station": "place-ccmnl", + "order": 6, + "stops": { + "0": ["70029"], + "1": ["70028"] + } + }, { + "stop_name": "North Station", + "branches": null, + "station": "place-north", + "order": 7, + "stops": { + "0": ["70027"], + "1": ["70026"] + } + }, { + "stop_name": "Haymarket", + "branches": null, + "station": "place-haecl", + "order": 8, + "stops": { + "0": ["70025"], + "1": ["70024"] + } + }, { + "stop_name": "State Street", + "branches": null, + "station": "place-state", + "order": 9, + "stops": { + "0": ["70023"], + "1": ["70022"] + } + }, { + "stop_name": "Downtown Crossing", + "branches": null, + "station": "place-dwnxg", + "order": 10, + "stops": { + "0": ["70021"], + "1": ["70020"] + } + }, { + "stop_name": "Chinatown", + "branches": null, + "station": "place-chncl", + "order": 11, + "stops": { + "0": ["70019"], + "1": ["70018"] + } + }, { + "stop_name": "Tufts Medical Center", + "branches": null, + "station": "place-tumnl", + "order": 12, + "stops": { + "0": ["70017"], + "1": ["70016"] + } + }, { + "stop_name": "Back Bay", + "branches": null, + "station": "place-bbsta", + "order": 13, + "stops": { + "0": ["70015"], + "1": ["70014"] + } + }, { + "stop_name": "Massachusetts Avenue", + "branches": null, + "station": "place-masta", + "order": 14, + "stops": { + "0": ["70013"], + "1": ["70012"] + } + }, { + "stop_name": "Ruggles", + "branches": null, + "station": "place-rugg", + "order": 15, + "stops": { + "0": ["70011"], + "1": ["70010"] + } + }, { + "stop_name": "Roxbury Crossing", + "branches": null, + "station": "place-rcmnl", + "order": 16, + "stops": { + "0": ["70009"], + "1": ["70008"] + } + }, { + "stop_name": "Jackson Square", + "branches": null, + "station": "place-jaksn", + "order": 17, + "stops": { + "0": ["70007"], + "1": ["70006"] + } + }, { + "stop_name": "Stony Brook", + "branches": null, + "station": "place-sbmnl", + "order": 18, + "stops": { + "0": ["70005"], + "1": ["70004"] + } + }, { + "stop_name": "Green Street", + "branches": null, + "station": "place-grnst", + "order": 19, + "stops": { + "0": ["70003"], + "1": ["70002"] + } + }, { + "stop_name": "Forest Hills", + "branches": null, + "station": "place-forhl", + "order": 20, + "stops": { + "0": ["70001"], + "1": ["70001"] + } } + ] + }, + "Blue": { + "type": "rapidtransit", + "direction": { + "0": "northbound", + "1": "southbound" }, - { - "stop_name": "Washington St @ Market St", - "branches": null, - "station": "973", - "order": 16, - "stops": { - "northbound": ["973"] + "stations": [ + { + "stop_name": "Wonderland", + "branches": null, + "station": "place-wondl", + "order": 1, + "stops": { + "0": ["70060"], + "1": ["70059"] + } + }, { + "stop_name": "Revere Beach", + "branches": null, + "station": "place-rbmnl", + "order": 2, + "stops": { + "0": ["70058"], + "1": ["70057"] + } + }, { + "stop_name": "Beachmont", + "branches": null, + "station": "place-bmmnl", + "order": 3, + "stops": { + "0": ["70056"], + "1": ["70055"] + } + }, { + "stop_name": "Suffolk Downs", + "branches": null, + "station": "place-sdmnl", + "order": 4, + "stops": { + "0": ["70054"], + "1": ["70053"] + } + }, { + "stop_name": "Orient Heights", + "branches": null, + "station": "place-orhte", + "order": 5, + "stops": { + "0": ["70052"], + "1": ["70051"] + } + }, { + "stop_name": "Wood Island", + "branches": null, + "station": "place-wimnl", + "order": 6, + "stops": { + "0": ["70050"], + "1": ["70049"] + } + }, { + "stop_name": "Airport", + "branches": null, + "station": "place-aport", + "order": 7, + "stops": { + "0": ["70048"], + "1": ["70047"] + } + }, { + "stop_name": "Maverick", + "branches": null, + "station": "place-mvbcl", + "order": 8, + "stops": { + "0": ["70046"], + "1": ["70045"] + } + }, { + "stop_name": "Aquarium", + "branches": null, + "station": "place-aqucl", + "order": 9, + "stops": { + "0": ["70044"], + "1": ["70043"] + } + }, { + "stop_name": "State Street", + "branches": null, + "station": "place-state", + "order": 10, + "stops": { + "0": ["70042"], + "1": ["70041"] + } + }, { + "stop_name": "Government Center", + "branches": null, + "station": "place-gover", + "order": 11, + "stops": { + "0": ["70040"], + "1": ["70039"] + } + }, { + "stop_name": "Bowdoin", + "branches": null, + "station": "place-bomnl", + "order": 12, + "stops": { + "0": ["70038"], + "1": ["70838"] + } } + ] + }, + "Green": { + "type": "rapidtransit", + "direction": { + "0": "eastbound", + "1": "westbound" }, - { - "stop_name": "Ave Louis Pasteur @ Longwood Ave", - "branches": null, - "station": "11780", - "order": 17, - "stops": { - "northbound": ["11780"] + "stations": [ + { + "stop_name": "Blandford Street", + "branches": ["B"], + "station": "place-bland", + "order": 101, + "stops": { + "0": ["70148"], + "1": ["70149"] + } + }, { + "stop_name": "Boston University East", + "branches": ["B"], + "station": "place-buest", + "order": 102, + "stops": { + "0": ["70146"], + "1": ["70147"] + } + }, { + "stop_name": "Boston University Central", + "branches": ["B"], + "station": "place-bucen", + "order": 103, + "stops": { + "0": ["70144"], + "1": ["70145"] + } + }, { + "stop_name": "Boston University West", + "branches": ["B"], + "station": "place-buwst", + "order": 104, + "stops": { + "0": ["70142"], + "1": ["70143"] + } + }, { + "stop_name": "Saint Paul Street (B)", + "branches": ["B"], + "station": "place-stplb", + "order": 105, + "stops": { + "0": ["70140"], + "1": ["70141"] + } + }, { + "stop_name": "Pleasant Street", + "branches": ["B"], + "station": "place-plsgr", + "order": 106, + "stops": { + "0": ["70138"], + "1": ["70139"] + } + }, { + "stop_name": "Babcock Street", + "branches": ["B"], + "station": "place-babck", + "order": 107, + "stops": { + "0": ["70136"], + "1": ["70137"] + } + }, { + "stop_name": "Packards Corner", + "branches": ["B"], + "station": "place-brico", + "order": 108, + "stops": { + "0": ["70134"], + "1": ["70135"] + } + }, { + "stop_name": "Harvard Avenue", + "branches": ["B"], + "station": "place-harvd", + "order": 109, + "stops": { + "0": ["70130"], + "1": ["70131"] + } + }, { + "stop_name": "Griggs Street", + "branches": ["B"], + "station": "place-grigg", + "order": 110, + "stops": { + "0": ["70128"], + "1": ["70129"] + } + }, { + "stop_name": "Allston Street", + "branches": ["B"], + "station": "place-alsgr", + "order": 111, + "stops": { + "0": ["70126"], + "1": ["70127"] + } + }, { + "stop_name": "Warren Street", + "branches": ["B"], + "station": "place-wrnst", + "order": 112, + "stops": { + "0": ["70124"], + "1": ["70125"] + } + }, { + "stop_name": "Washington Street", + "branches": ["B"], + "station": "place-wascm", + "order": 113, + "stops": { + "0": ["70120"], + "1": ["70121"] + } + }, { + "stop_name": "Sutherland Road", + "branches": ["B"], + "station": "place-sthld", + "order": 114, + "stops": { + "0": ["70116"], + "1": ["70117"] + } + }, { + "stop_name": "Chiswick Road", + "branches": ["B"], + "station": "place-chswk", + "order": 115, + "stops": { + "0": ["70114"], + "1": ["70115"] + } + }, { + "stop_name": "Chestnut Hill Avenue", + "branches": ["B"], + "station": "place-chill", + "order": 116, + "stops": { + "0": ["70112"], + "1": ["70113"] + } + }, { + "stop_name": "South Street", + "branches": ["B"], + "station": "place-sougr", + "order": 117, + "stops": { + "0": ["70110"], + "1": ["70111"] + } + }, { + "stop_name": "Boston College", + "branches": ["B"], + "station": "place-lake", + "order": 118, + "stops": { + "0": ["70106"], + "1": ["70107"] + } + }, { + "stop_name": "Kenmore", + "branches": ["B", "C", "D"], + "station": "place-kencl", + disabled: true, + "order": 11, + "stops": { + "0": ["70150", "71150"], + "1": ["70151", "71151"] + } + }, { + "stop_name": "Hynes", + "branches": ["B", "C", "D"], + "station": "place-hymnl", + "order": 10, + "stops": { + "0": ["70152"], + "1": ["70153"] + } + }, { + "stop_name": "Park Street", + "branches": ["B", "C", "D", "E"], + "station": "place-pktrm", + disabled: true, + "order": 6, + "stops": { + "0": ["70200"], + "1": ["70196", "70197", "70198", "70199"] + } + }, { + "stop_name": "Boylston", + "branches": ["B", "C", "D", "E"], + "station": "place-boyls", + "order": 7, + "stops": { + "0": ["70158"], + "1": ["70159"] + } + }, { + "stop_name": "Arlington", + "branches": ["B", "C", "D", "E"], + "station": "place-armnl", + "order": 8, + "stops": { + "0": ["70156"], + "1": ["70157"] + } + }, { + "stop_name": "Copley", + "branches": ["B", "C", "D", "E"], + "station": "place-coecl", + "order": 9, + "stops": { + "0": ["70154"], + "1": ["70155"] + } + }, { + "stop_name": "Cleveland Circle", + "branches": ["C"], + "station": "place-clmnl", + "order": 213, + "stops": { + "0": ["70238"], + "1": ["70237"] + } + }, { + "stop_name": "Englewood Avenue", + "branches": ["C"], + "station": "place-engav", + "order": 212, + "stops": { + "0": ["70236"], + "1": ["70235"] + } + }, { + "stop_name": "Dean Road", + "branches": ["C"], + "station": "place-denrd", + "order": 211, + "stops": { + "0": ["70234"], + "1": ["70233"] + } + }, { + "stop_name": "Tappan Street", + "branches": ["C"], + "station": "place-tapst", + "order": 210, + "stops": { + "0": ["70232"], + "1": ["70231"] + } + }, { + "stop_name": "Washington Square", + "branches": ["C"], + "station": "place-bcnwa", + "order": 209, + "stops": { + "0": ["70230"], + "1": ["70229"] + } + }, { + "stop_name": "Fairbanks Street", + "branches": ["C"], + "station": "place-fbkst", + "order": 208, + "stops": { + "0": ["70228"], + "1": ["70227"] + } + }, { + "stop_name": "Brandon Hall", + "branches": ["C"], + "station": "place-bndhl", + "order": 207, + "stops": { + "0": ["70226"], + "1": ["70225"] + } + }, { + "stop_name": "Summit Avenue", + "branches": ["C"], + "station": "place-sumav", + "order": 206, + "stops": { + "0": ["70224"], + "1": ["70223"] + } + }, { + "stop_name": "Coolidge Corner", + "branches": ["C"], + "station": "place-cool", + "order": 205, + "stops": { + "0": ["70220"], + "1": ["70219"] + } + }, { + "stop_name": "Saint Paul Street (C)", + "branches": ["C"], + "station": "place-stpul", + "order": 204, + "stops": { + "0": ["70218"], + "1": ["70217"] + } + }, { + "stop_name": "Kent Street", + "branches": ["C"], + "station": "place-kntst", + "order": 203, + "stops": { + "0": ["70216"], + "1": ["70215"] + } + }, { + "stop_name": "Hawes Street", + "branches": ["C"], + "station": "place-hwsst", + "order": 202, + "stops": { + "0": ["70214"], + "1": ["70213"] + } + }, { + "stop_name": "Saint Marys Street", + "branches": ["C"], + "station": "place-smary", + "order": 201, + "stops": { + "0": ["70212"], + "1": ["70211"] + } + }, { + "stop_name": "Government Center", + "branches": ["C", "D", "E"], + "station": "place-gover", + "order": 5, + "stops": { + "0": ["70201"], + "1": ["70202"] + } + }, { + "stop_name": "North Station", + "branches": ["C", "E"], + "station": "place-north", + "order": 3, + "stops": { + "0": ["70205"], + "1": ["70206"] + } + }, { + "stop_name": "Haymarket", + "branches": ["C", "E"], + "station": "place-haecl", + "order": 4, + "stops": { + "0": ["70203"], + "1": ["70204"] + } + }, { + "stop_name": "Fenway", + "branches": ["D"], + "station": "place-fenwy", + "order": 301, + "stops": { + "0": ["70186"], + "1": ["70187"] + } + }, { + "stop_name": "Longwood", + "branches": ["D"], + "station": "place-longw", + "order": 302, + "stops": { + "0": ["70182"], + "1": ["70183"] + } + }, { + "stop_name": "Brookline Village", + "branches": ["D"], + "station": "place-bvmnl", + "order": 303, + "stops": { + "0": ["70180"], + "1": ["70181"] + } + }, { + "stop_name": "Brookline Hills", + "branches": ["D"], + "station": "place-brkhl", + "order": 304, + "stops": { + "0": ["70178"], + "1": ["70179"] + } + }, { + "stop_name": "Beaconsfield", + "branches": ["D"], + "station": "place-bcnfd", + "order": 305, + "stops": { + "0": ["70176"], + "1": ["70177"] + } + }, { + "stop_name": "Reservoir", + "branches": ["D"], + "station": "place-rsmnl", + "order": 306, + "stops": { + "0": ["70174"], + "1": ["70175"] + } + }, { + "stop_name": "Chestnut Hill", + "branches": ["D"], + "station": "place-chhil", + "order": 307, + "stops": { + "0": ["70172"], + "1": ["70173"] + } + }, { + "stop_name": "Newton Centre", + "branches": ["D"], + "station": "place-newto", + "order": 308, + "stops": { + "0": ["70170"], + "1": ["70171"] + } + }, { + "stop_name": "Newton Highlands", + "branches": ["D"], + "station": "place-newtn", + "order": 309, + "stops": { + "0": ["70168"], + "1": ["70169"] + } + }, { + "stop_name": "Eliot", + "branches": ["D"], + "station": "place-eliot", + "order": 310, + "stops": { + "0": ["70166"], + "1": ["70167"] + } + }, { + "stop_name": "Waban", + "branches": ["D"], + "station": "place-waban", + "order": 311, + "stops": { + "0": ["70164"], + "1": ["70165"] + } + }, { + "stop_name": "Woodland", + "branches": ["D"], + "station": "place-woodl", + "order": 312, + "stops": { + "0": ["70162"], + "1": ["70163"] + } + }, { + "stop_name": "Riverside", + "branches": ["D"], + "station": "place-river", + "order": 313, + "stops": { + "0": ["70160"], + "1": ["70161"] + } + }, { + "stop_name": "Heath Street", + "branches": ["E"], + "station": "place-hsmnl", + "order": 411, + "stops": { + "0": ["70260"], + "1": ["70260"] + } + }, { + "stop_name": "Back of the Hill", + "branches": ["E"], + "station": "place-bckhl", + "order": 410, + "stops": { + "0": ["70258"], + "1": ["70257"] + } + }, { + "stop_name": "Riverway", + "branches": ["E"], + "station": "place-rvrwy", + "order": 409, + "stops": { + "0": ["70256"], + "1": ["70255"] + } + }, { + "stop_name": "Mission Park", + "branches": ["E"], + "station": "place-mispk", + "order": 408, + "stops": { + "0": ["70254"], + "1": ["70253"] + } + }, { + "stop_name": "Fenwood Road", + "branches": ["E"], + "station": "place-fenwd", + "order": 407, + "stops": { + "0": ["70252"], + "1": ["70251"] + } + }, { + "stop_name": "Brigham Circle", + "branches": ["E"], + "station": "place-brmnl", + "order": 406, + "stops": { + "0": ["70250"], + "1": ["70249"] + } + }, { + "stop_name": "Longwood Medical Area", + "branches": ["E"], + "station": "place-lngmd", + "order": 405, + "stops": { + "0": ["70248"], + "1": ["70247"] + } + }, { + "stop_name": "Museum of Fine Arts", + "branches": ["E"], + "station": "place-mfa", + "order": 404, + "stops": { + "0": ["70246"], + "1": ["70245"] + } + }, { + "stop_name": "Northeastern University", + "branches": ["E"], + "station": "place-nuniv", + "order": 403, + "stops": { + "0": ["70244"], + "1": ["70243"] + } + }, { + "stop_name": "Symphony", + "branches": ["E"], + "station": "place-symcl", + "order": 402, + "stops": { + "0": ["70242"], + "1": ["70241"] + } + }, { + "stop_name": "Prudential", + "branches": ["E"], + "station": "place-prmnl", + "order": 401, + "stops": { + "0": ["70240"], + "1": ["70239"] + } + }, { + "stop_name": "Lechmere", + "branches": ["E"], + "station": "place-lech", + "order": 1, + "stops": { + "0": ["70209"], + "1": ["70210"] + } + }, { + "stop_name": "Science Park", + "branches": ["E"], + "station": "place-spmnl", + "order": 2, + "stops": { + "0": ["70207"], + "1": ["70208"] + } } - }, - ] + ] + } }; const createConfigPresetValue = (line, fromStationName, toStationName, date_start, date_end = undefined) => { - const fromStation = stations[line].find(s => s.stop_name === fromStationName); - const toStation = stations[line].find(s => s.stop_name === toStationName); + const fromStation = stations[line].stations.find(s => s.stop_name === fromStationName); + const toStation = stations[line].stations.find(s => s.stop_name === toStationName); return { line, date_start, diff --git a/src/stations.js b/src/stations.js index 6b099bf2a..0de7d6cfa 100644 --- a/src/stations.js +++ b/src/stations.js @@ -9,31 +9,31 @@ const lookup_station_by_id = (line, id) => { return undefined; } - return stations[line].find(x => [...x.stops.northbound || [], ...x.stops.southbound || []].includes(id)); + return stations[line].stations.find(x => [...x.stops["0"] || [], ...x.stops["1"] || []].includes(id)); }; const options_station = (line) => { if (!line) { return []; } - return stations[line]; + return stations[line].stations; }; const station_direction = (from, to, line) => { if (from.order === to.order) { return ""; } - return from.order > to.order ? "northbound" : "southbound"; + return from.order > to.order ? stations[line].direction["0"] : stations[line].direction["1"]; } const get_stop_ids_for_stations = (from, to) => { if (!from || !to) { return { fromStopId: null, toStopId: null }; } - const isSouthbound = from.order < to.order; + const isDirection1 = from.order < to.order; return { - fromStopIds: isSouthbound ? from.stops.southbound : from.stops.northbound, - toStopIds: isSouthbound ? to.stops.southbound : to.stops.northbound, + fromStopIds: isDirection1 ? from.stops["1"] : from.stops["0"], + toStopIds: isDirection1 ? to.stops["1"] : to.stops["0"], } } From f84602acd404992667a8fe252b7311accb4f6462 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 29 Sep 2021 13:20:08 -0400 Subject: [PATCH 012/152] Incorporate bus constants for route 1 --- server/bus/bus2train.py | 39 ++++++++---- server/chalicelib/s3.py | 12 ++-- src/constants_bus.js | 137 ++++++++++++++++++++++++++++++++++++++++ src/stations.js | 5 +- 4 files changed, 170 insertions(+), 23 deletions(-) create mode 100644 src/constants_bus.js diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 052d5af97..3a4a59f75 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -19,7 +19,8 @@ def load_data(input_csv, routes): # thinking about doing this in pandas to have all the info at once df = pd.read_csv(input_csv, parse_dates=['service_date', 'scheduled', 'actual']) - df = df.loc[(df.standard_type == "Headway") & (df.actual.notnull())] + # AHHHHHH we need the "Headway" AND "Schedule" ones. + df = df.loc[df.actual.notnull()] df.route_id = df.route_id.str.lstrip('0') if routes: df = df.loc[df.route_id.isin(routes)] @@ -73,13 +74,20 @@ def create_manifest(df, checkpoint_file): "station": tp_id.lower(), "order": inbound.time_point_order.tolist()[0], "stops": { - "inbound": inbound.stop_id.drop_duplicates().tolist(), - "outbound": outbound.stop_id.drop_duplicates().tolist() + "1": inbound.stop_id.drop_duplicates().tolist(), + "0": outbound.stop_id.drop_duplicates().tolist() } } summary.append(this_obj) - manifest[rte_id] = sorted(summary, key = lambda x: x['order']) + manifest[rte_id] = { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": sorted(summary, key = lambda x: x['order']) + } return manifest @@ -131,6 +139,7 @@ def main(): parser.add_argument('output', metavar='OUTPUT_DIR') parser.add_argument('--checkpoints', metavar="checkpoints.txt") parser.add_argument('--routes', '-r', nargs="*", type=str) + parser.add_argument('--manifest', action='store_true') args = parser.parse_args() @@ -147,16 +156,18 @@ def main(): events = process_events(data) to_disk(events, output_dir) - # do something with available and data - manifest = create_manifest(data, checkpoint_file) - with open(availability_path, "w", encoding="utf-8") as fd: - json.dump(manifest, fd, ensure_ascii=False, indent=4) - fd.write("\n") - -""" with open(availability_path, "w", encoding="utf-8") as file: - json.dump(availability_to_json(availability_set), file, ensure_ascii=False, indent=4) - file.write("\n") - """ + #TODO: this should be moved to a different utility probably + if args.manifest: + # do something with available and data + manifest = create_manifest(data, checkpoint_file) + with open(availability_path, "w", encoding="utf-8") as fd: + json.dump(manifest, fd, ensure_ascii=False, indent=4) + fd.write("\n") + + """ with open(availability_path, "w", encoding="utf-8") as file: + json.dump(availability_to_json(availability_set), file, ensure_ascii=False, indent=4) + file.write("\n") + """ if __name__ == "__main__": main() diff --git a/server/chalicelib/s3.py b/server/chalicelib/s3.py index 9aa265ee8..4d135e7ae 100644 --- a/server/chalicelib/s3.py +++ b/server/chalicelib/s3.py @@ -13,7 +13,8 @@ def download(key, encoding="utf8"): obj = s3.get_object(Bucket=BUCKET, Key=key) s3_data = obj["Body"].read() - decompressed = zlib.decompress(s3_data).decode(encoding) + # 32 should detect zlib vs gzip + decompressed = zlib.decompress(s3_data, zlib.MAX_WBITS|32).decode(encoding) return decompressed @@ -28,13 +29,8 @@ def download_one_event_file(date, stop_id): # Download events from S3 try: - key = f"Events/daily-data/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv.gz" - obj = s3.get_object(Bucket=BUCKET, Key=key) - s3_data = obj["Body"].read() - # Uncompress - decompressed = zlib.decompress( - s3_data, wbits=zlib.MAX_WBITS | 16).decode("ascii").split("\r\n") + decompressed = download(key, 'ascii') except ClientError as ex: if ex.response['Error']['Code'] == 'NoSuchKey': @@ -46,7 +42,7 @@ def download_one_event_file(date, stop_id): # Parse CSV rows = [] - for row in csv.DictReader(decompressed): + for row in csv.DictReader(decompressed.splitlines()): rows.append(row) # sort diff --git a/src/constants_bus.js b/src/constants_bus.js new file mode 100644 index 000000000..1b1eb6f4c --- /dev/null +++ b/src/constants_bus.js @@ -0,0 +1,137 @@ +export const bus_stations = { + "1": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Harvard", + "branches": null, + "station": "hhgat", + "order": 1, + "stops": { + "1": [ + "110" + ], + "0": [ + "110" + ] + } + }, + { + "stop_name": "Massachusetts Avenue & Putnam Avenue", + "branches": null, + "station": "maput", + "order": 2, + "stops": { + "1": [ + "67" + ], + "0": [ + "108" + ] + } + }, + { + "stop_name": "Central Square (Cambridge)", + "branches": null, + "station": "cntsq", + "order": 3, + "stops": { + "1": [ + "72" + ], + "0": [ + "102" + ] + } + }, + { + "stop_name": "MIT @ Massachusetts Avenue", + "branches": null, + "station": "mit", + "order": 4, + "stops": { + "1": [ + "75" + ], + "0": [ + "97" + ] + } + }, + { + "stop_name": "Hynes Station", + "branches": null, + "station": "hynes", + "order": 5, + "stops": { + "1": [ + "79" + ], + "0": [ + "93" + ] + } + }, + { + "stop_name": "Massachusetts Avenue (Orange Line)", + "branches": null, + "station": "masta", + "order": 6, + "stops": { + "1": [ + "187" + ], + "0": [ + "188" + ] + } + }, + { + "stop_name": "Massachusetts Avenue @ Washington", + "branches": null, + "station": "wasma", + "order": 7, + "stops": { + "1": [ + "59" + ], + "0": [ + "10590" + ] + } + }, + { + "stop_name": "Melnea Cass Boulevard @ Washington Street", + "branches": null, + "station": "melwa", + "order": 8, + "stops": { + "1": [ + "62" + ], + "0": [ + "2" + ] + } + }, + { + "stop_name": "Nubian Station", + "branches": null, + "station": "nubn", + "order": 9, + "stops": { + "1": [ + "64" + ], + "0": [ + "64" + ] + } + } + ] + } +} diff --git a/src/stations.js b/src/stations.js index 0de7d6cfa..06e95c783 100644 --- a/src/stations.js +++ b/src/stations.js @@ -1,4 +1,7 @@ -import { stations } from './constants'; +import { stations as rt_stations } from './constants'; +import { bus_stations } from './constants_bus'; + +const stations = {...rt_stations, ...bus_stations}; const all_lines = () => { return Object.keys(stations); From 2fa0c7dd137974c526b25f971fd57033af7cb11a Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 29 Sep 2021 17:07:48 -0400 Subject: [PATCH 013/152] switch back to main s3 folder --- server/.chalice/policy.json | 6 +++--- server/chalicelib/s3.py | 2 +- src/constants.js | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/.chalice/policy.json b/server/.chalice/policy.json index 487f74172..ec1a90ace 100644 --- a/server/.chalice/policy.json +++ b/server/.chalice/policy.json @@ -16,7 +16,7 @@ ], "Effect": "Allow", "Resource": [ - "arn:aws:s3:::tm-mbta-performance-preston-test" + "arn:aws:s3:::tm-mbta-performance" ] }, { @@ -25,7 +25,7 @@ ], "Effect": "Allow", "Resource": [ - "arn:aws:s3:::tm-mbta-performance-preston-test/*" + "arn:aws:s3:::tm-mbta-performance/*" ] }, { @@ -34,7 +34,7 @@ ], "Effect": "Allow", "Resource": [ - "arn:aws:s3:::tm-mbta-performance-preston-test/Alerts/*" + "arn:aws:s3:::tm-mbta-performance/Alerts/*" ] } ] diff --git a/server/chalicelib/s3.py b/server/chalicelib/s3.py index 4d135e7ae..668cff6fa 100644 --- a/server/chalicelib/s3.py +++ b/server/chalicelib/s3.py @@ -5,7 +5,7 @@ from chalicelib import parallel -BUCKET = "tm-mbta-performance-preston-test" +BUCKET = "tm-mbta-performance" s3 = boto3.client('s3') diff --git a/src/constants.js b/src/constants.js index ddecb48b5..a50fe3e14 100644 --- a/src/constants.js +++ b/src/constants.js @@ -3,7 +3,7 @@ export const colorsForLine = { Orange: '#e66f00', Blue: '#0e3d8c', Green: '#159765', - "57": '#ffc72c', + bus: '#ffc72c', }; export const stations = { From d8c9309977599bf8edac76da452def931b11c86b Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 29 Sep 2021 19:00:28 -0400 Subject: [PATCH 014/152] don't use performance api for buses --- server/chalicelib/data_funcs.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/server/chalicelib/data_funcs.py b/server/chalicelib/data_funcs.py index df833e725..1ee678fef 100644 --- a/server/chalicelib/data_funcs.py +++ b/server/chalicelib/data_funcs.py @@ -13,9 +13,12 @@ def stamp_to_dt(stamp): return dt.strftime(DATE_FORMAT) -def use_S3(date): - return (date.today() - date).days >= 90 +def use_S3(date, stops): + archival = (date.today() - date).days >= 90 + stop_id = int(stops[0]) + bus = not stop_id in range(70000, 72000) or stop_id in [70618, 71391, 71855] + return archival or bus def partition_S3_dates(start_date, end_date): CUTOFF = datetime.date.today() - datetime.timedelta(days=90) @@ -36,7 +39,7 @@ def partition_S3_dates(start_date, end_date): def headways(sdate, stops, edate=None): if edate is None: - if use_S3(sdate): + if use_S3(sdate, stops): return s3_historical.headways(stops, sdate, sdate) else: return process_mbta_headways(stops, sdate) @@ -93,7 +96,7 @@ def process_mbta_headways(stops, sdate, edate=None): def travel_times(sdate, from_stops, to_stops, edate=None): if edate is None: - if use_S3(sdate): + if use_S3(sdate, from_stops): return s3_historical.travel_times(from_stops[0], to_stops[0], sdate, sdate) else: return process_mbta_travel_times(from_stops, to_stops, sdate) @@ -143,7 +146,7 @@ def process_mbta_travel_times(from_stops, to_stops, sdate, edate=None): def dwells(sdate, stops, edate=None): if edate is None: - if use_S3(sdate): + if use_S3(sdate, stops): return s3_historical.dwells(stops, sdate, sdate) else: return process_mbta_dwells(stops, sdate) From 04bfaeb298923f441c88322a15e92d25a31942d5 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 30 Sep 2021 11:30:54 -0400 Subject: [PATCH 015/152] clean up bus2train (factor out manifest code) --- server/bus/bus2train.py | 155 ++++++++++++++-------------------------- 1 file changed, 52 insertions(+), 103 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 3a4a59f75..3b1f8029a 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -1,26 +1,35 @@ import argparse -import json -import os import pathlib import pandas as pd -import sys -import traceback -from datetime import date, datetime -from operator import itemgetter +from datetime import datetime -CSV_HEADER = ["service_date", "route_id", "trip_id", "direction_id", "stop_id", - "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] -AVAILABILITY_FILE = "bus_available.json" def load_data(input_csv, routes): - output_by_stop_and_day = {} - # availability_set = json_to_availability(availability_path) + """ + Loads in the below format and makes some adjustments for processing. + - Filter only points with actual trip data + - Trim leading 0s from route_id + - Select only route_ids in `routes` + - Set scheduled/actual times to be on service_date, not 1900-01-01 + - Map direction_id (Outbound -> 0, Inbound -> 1) + """ + """ + "service_date", "route_id", "direction", "half_trip_id", "stop_id", "time_point_id", "time_point_order", "point_type", "standard_type", "scheduled", "actual", "scheduled_headway", "headway" + 2020-01-15, "01", "Inbound", 46374001, 67, "maput", 2, "Midpoint", "Schedule", 1900-01-01 05:08:00, 1900-01-01 05:09:07, -5, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 110, "hhgat", 1, "Startpoint", "Schedule", 1900-01-01 05:05:00, 1900-01-01 05:04:34, 26, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 72, "cntsq", 3, "Midpoint", "Schedule", 1900-01-01 05:11:00, 1900-01-01 05:12:01, -22, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 75, "mit", 4, "Midpoint", "Schedule", 1900-01-01 05:14:00, 1900-01-01 05:14:58, -25, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 79, "hynes", 5, "Midpoint", "Schedule", 1900-01-01 05:18:00, 1900-01-01 05:18:45, 32, NA,NA + 2020-01-15, "01", "Inbound", 46374001, 187, "masta", 6, "Midpoint", "Schedule", 1900-01-01 05:20:00, 1900-01-01 05:21:04, -33, NA,NA + 2020-01-15, "01", "Inbound", 46374045, 110, "hhgat", 1, "Startpoint", "Headway", 1900-01-01 05:20:00, 1900-01-01 05:20:45, NA, 900,971 + """ # thinking about doing this in pandas to have all the info at once df = pd.read_csv(input_csv, parse_dates=['service_date', 'scheduled', 'actual']) - - # AHHHHHH we need the "Headway" AND "Schedule" ones. + + # We need to keep both "Headway" AND "Schedule": both can have timepoint data. df = df.loc[df.actual.notnull()] + df.route_id = df.route_id.str.lstrip('0') if routes: df = df.loc[df.route_id.isin(routes)] @@ -32,71 +41,20 @@ def load_data(input_csv, routes): df.direction_id = df.direction_id.map({"Outbound": 0, "Inbound": 1}) - """ - "service_date", "route_id", "direction", "half_trip_id", "stop_id", "time_point_id", "time_point_order", "point_type", "standard_type", "scheduled", "actual", "scheduled_headway", "headway" - 2020-01-15, "01", "Inbound", 46374001, 67, "maput", 2, "Midpoint", "Schedule", 1900-01-01 05:08:00, 1900-01-01 05:09:07, -5, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 110, "hhgat", 1, "Startpoint", "Schedule", 1900-01-01 05:05:00, 1900-01-01 05:04:34, 26, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 72, "cntsq", 3, "Midpoint", "Schedule", 1900-01-01 05:11:00, 1900-01-01 05:12:01, -22, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 75, "mit", 4, "Midpoint", "Schedule", 1900-01-01 05:14:00, 1900-01-01 05:14:58, -25, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 79, "hynes", 5, "Midpoint", "Schedule", 1900-01-01 05:18:00, 1900-01-01 05:18:45, 32, NA,NA - 2020-01-15, "01", "Inbound", 46374001, 187, "masta", 6, "Midpoint", "Schedule", 1900-01-01 05:20:00, 1900-01-01 05:21:04, -33, NA,NA - 2020-01-15, "01", "Inbound", 46374045, 110, "hhgat", 1, "Startpoint", "Headway", 1900-01-01 05:20:00, 1900-01-01 05:20:45, NA, 900,971 - """ - return df -def load_checkpoints(checkpoint_file): - if checkpoint_file: - return pd.read_csv(checkpoint_file, index_col="checkpoint_id").to_dict()["checkpoint_name"] - else: - return {} - -def create_manifest(df, checkpoint_file): - station_names = load_checkpoints(checkpoint_file) - - manifest = {} - - timepoints = df[['route_id', 'time_point_id', 'stop_id', 'direction_id', 'time_point_order']].drop_duplicates() - timepoints.stop_id = timepoints.stop_id.astype(str) - # timepoints.time_point_order = timepoints.time_point_order.astype(numpy.int32) - - for rte_id, points in timepoints.groupby('route_id'): - summary = [] - for tp_id, info in points.groupby('time_point_id'): - inbound = info.loc[info.direction_id == 1] - outbound = info.loc[info.direction_id == 0] - - # TODO: this assumes a very linear route. - # alternate route patterns and changes BREAK things. - this_obj = { - "stop_name": station_names.get(tp_id.lower(), ""), - "branches": None, - "station": tp_id.lower(), - "order": inbound.time_point_order.tolist()[0], - "stops": { - "1": inbound.stop_id.drop_duplicates().tolist(), - "0": outbound.stop_id.drop_duplicates().tolist() - } - } - summary.append(this_obj) - - manifest[rte_id] = { - "type": "bus", - "direction": { - "0": "outbound", - "1": "inbound" - }, - "stations": sorted(summary, key = lambda x: x['order']) - } - - return manifest def process_events(df): - - + """ + Take the tidied input data and rearrange the columns to match rapidtransit format. + - Rename columns (trip_id, stop_sequence, event_time) + - Remove extra columns + - Add empty vehicle columns + - Calculate event_type column with ARR and DEP entries + """ CSV_HEADER = ["service_date", "route_id", "trip_id", "direction_id", "stop_id", - "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] - + "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] + df = df.rename(columns={'half_trip_id': 'trip_id', 'time_point_order': 'stop_sequence', 'actual': 'event_time'}) @@ -108,66 +66,57 @@ def process_events(df): "Midpoint": ["ARR", "DEP"], "Endpoint": ["ARR"]}) df = df.explode('event_type') - df = df[CSV_HEADER] + df = df[CSV_HEADER] # reorder return df def _write_file(events, outdir): + """ + This is a helper that will write the events to disk. + It will be called on each "groupby" object, grouping stop_id and service_date + """ service_date, stop_id = events.name - fname = (pathlib.Path(outdir) / - "Events" / - "daily-data" / - str(stop_id) / - f"Year={service_date.year}" / - f"Month={service_date.month}" / - f"Day={service_date.day}" / - "events.csv.gz") + fname = pathlib.Path(outdir, + "Events", + "daily-data", + str(stop_id), + f"Year={service_date.year}", + f"Month={service_date.month}", + f"Day={service_date.day}", + "events.csv.gz") fname.parent.mkdir(parents=True, exist_ok=True) events.to_csv(fname, index=False, compression='gzip') def to_disk(df, root): + """ + For each service_date/stop_id group, we call the helper that will write it to disk. + """ df.groupby(['service_date', 'stop_id']).apply(lambda e: _write_file(e, root)) -def main(): +def main(): parser = argparse.ArgumentParser() parser.add_argument('input', metavar='INPUT_CSV') parser.add_argument('output', metavar='OUTPUT_DIR') - parser.add_argument('--checkpoints', metavar="checkpoints.txt") - parser.add_argument('--routes', '-r', nargs="*", type=str) - parser.add_argument('--manifest', action='store_true') - + parser.add_argument('--routes', '-r', nargs="*", type=str, + help="One note here: we should always be additive with our route set \ + in case 2 lines share the same stop id: we need both in the result file.") args = parser.parse_args() - input_csv = args.input output_dir = args.output - checkpoint_file = args.checkpoints routes = args.routes pathlib.Path(output_dir).mkdir(exist_ok=True) - - availability_path = os.path.join(output_dir, AVAILABILITY_FILE) + data = load_data(input_csv, routes) events = process_events(data) to_disk(events, output_dir) - #TODO: this should be moved to a different utility probably - if args.manifest: - # do something with available and data - manifest = create_manifest(data, checkpoint_file) - with open(availability_path, "w", encoding="utf-8") as fd: - json.dump(manifest, fd, ensure_ascii=False, indent=4) - fd.write("\n") - - """ with open(availability_path, "w", encoding="utf-8") as file: - json.dump(availability_to_json(availability_set), file, ensure_ascii=False, indent=4) - file.write("\n") - """ if __name__ == "__main__": main() From 6a64b98c36a1bad07c05404a377f97e9db3a7d2e Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 30 Sep 2021 12:48:04 -0400 Subject: [PATCH 016/152] add script to create bus station json (best guess) --- server/bus/manifest.py | 86 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 server/bus/manifest.py diff --git a/server/bus/manifest.py b/server/bus/manifest.py new file mode 100644 index 000000000..d5da76719 --- /dev/null +++ b/server/bus/manifest.py @@ -0,0 +1,86 @@ +import argparse +import pandas as pd +import json + +from bus2train import load_data + + +def load_checkpoints(checkpoint_file): + """ + input: path to checkpoints.txt from GTFS + output: dict(checkpoint_id -> station_name) or {} + """ + if checkpoint_file: + return pd.read_csv(checkpoint_file, index_col="checkpoint_id").to_dict()["checkpoint_name"] + else: + return {} + + +def create_manifest(df, checkpoint_file): + station_names = load_checkpoints(checkpoint_file) + + manifest = {} + + timepoints = df.groupby(['route_id', 'time_point_id', 'stop_id', 'direction_id'])['time_point_order'].agg(pd.Series.mode).reset_index() + timepoints.stop_id = timepoints.stop_id.astype(str) + + for rte_id, points in timepoints.groupby('route_id'): + summary = [] + for tp_id, info in points.groupby('time_point_id'): + inbound = info.loc[info.direction_id == 1] + outbound = info.loc[info.direction_id == 0] + + # This assumes a very linear route: any branches need to be added manually + # In addition, stop_order should be double checked, since this is just a guess + order_guess = inbound.time_point_order.max() + if pd.isnull(order_guess): + order_guess = 0 + else: + order_guess = int(order_guess) # none of that icky int64 stuff, json-serializable please + this_obj = { + "stop_name": station_names.get(tp_id.lower(), ""), + "branches": None, + "station": tp_id.lower(), + "order": order_guess, + "stops": { + "1": inbound.stop_id.tolist(), + "0": outbound.stop_id.tolist() + } + } + summary.append(this_obj) + + manifest[rte_id] = { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": sorted(summary, key = lambda x: x['order']) + } + + return manifest + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument('input', metavar='INPUT_CSV') + parser.add_argument('output', metavar='OUTPUT_JSON') + parser.add_argument('--routes', '-r', nargs="+", type=str) + parser.add_argument('--checkpoints', metavar="gtfs_checkpoints.txt") + + args = parser.parse_args() + input_csv = args.input + output_file = args.output + routes = args.routes + checkpoint_file = args.checkpoints + + data = load_data(input_csv, routes) + + manifest = create_manifest(data, checkpoint_file) + + with open(output_file, "w", encoding="utf-8") as fd: + json.dump(manifest, fd, ensure_ascii=False, indent=4) + fd.write("\n") + +if __name__ == "__main__": + main() \ No newline at end of file From 12f7fe3f75e371180976409599c379c79e0688a0 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 30 Sep 2021 14:05:52 -0400 Subject: [PATCH 017/152] move bus_constants to folder --- src/{constants_bus.js => bus_constants/1.json} | 2 +- src/stations.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/{constants_bus.js => bus_constants/1.json} (99%) diff --git a/src/constants_bus.js b/src/bus_constants/1.json similarity index 99% rename from src/constants_bus.js rename to src/bus_constants/1.json index 1b1eb6f4c..7e77c1bce 100644 --- a/src/constants_bus.js +++ b/src/bus_constants/1.json @@ -1,4 +1,4 @@ -export const bus_stations = { +{ "1": { "type": "bus", "direction": { diff --git a/src/stations.js b/src/stations.js index 06e95c783..d3413127b 100644 --- a/src/stations.js +++ b/src/stations.js @@ -1,7 +1,7 @@ import { stations as rt_stations } from './constants'; -import { bus_stations } from './constants_bus'; +import bus1 from './bus_constants/1.json'; -const stations = {...rt_stations, ...bus_stations}; +const stations = {...rt_stations, ...bus1}; const all_lines = () => { return Object.keys(stations); From 22df0dbcc78e9aec95286d7ed459adb6007bb7d7 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Fri, 1 Oct 2021 22:41:14 -0400 Subject: [PATCH 018/152] Update s3 to handle multiple stop_ids --- server/chalicelib/data_funcs.py | 4 ++-- server/chalicelib/s3.py | 14 ++++++++++++-- server/chalicelib/s3_historical.py | 14 +++++++------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/server/chalicelib/data_funcs.py b/server/chalicelib/data_funcs.py index 1ee678fef..9fea3cde1 100644 --- a/server/chalicelib/data_funcs.py +++ b/server/chalicelib/data_funcs.py @@ -97,7 +97,7 @@ def process_mbta_headways(stops, sdate, edate=None): def travel_times(sdate, from_stops, to_stops, edate=None): if edate is None: if use_S3(sdate, from_stops): - return s3_historical.travel_times(from_stops[0], to_stops[0], sdate, sdate) + return s3_historical.travel_times(from_stops, to_stops, sdate, sdate) else: return process_mbta_travel_times(from_stops, to_stops, sdate) @@ -105,7 +105,7 @@ def travel_times(sdate, from_stops, to_stops, edate=None): all_data = [] if s3_interval: start, end = s3_interval - all_data.extend(s3_historical.travel_times(from_stops[0], to_stops[0], start, end)) + all_data.extend(s3_historical.travel_times(from_stops, to_stops, start, end)) if api_interval: start, end = api_interval diff --git a/server/chalicelib/s3.py b/server/chalicelib/s3.py index 668cff6fa..aaa95d18f 100644 --- a/server/chalicelib/s3.py +++ b/server/chalicelib/s3.py @@ -25,6 +25,7 @@ def upload(key, bytes, compress=True): def download_one_event_file(date, stop_id): + """As advertised: single event file from s3""" year, month, day = date.year, date.month, date.day # Download events from S3 @@ -50,5 +51,14 @@ def download_one_event_file(date, stop_id): return rows_by_time -# signature: (date_iterable, stop_id) -download_event_range = parallel.make_parallel(download_one_event_file) +def download_single_day_events(date, stops): + """Will download events for single day, but can handle multiple stop_ids""" + result = [] + for stop_id in stops: + result += download_one_event_file(date, stop_id) + return result + + +# signature: (date_iterable, [stop_id]) +# Will download events for multiple stops, over a date range. +download_event_range = parallel.make_parallel(download_single_day_events) diff --git a/server/chalicelib/s3_historical.py b/server/chalicelib/s3_historical.py index 9130bdf9b..d19bd0347 100644 --- a/server/chalicelib/s3_historical.py +++ b/server/chalicelib/s3_historical.py @@ -37,8 +37,8 @@ def unique_everseen(iterable, key=None): yield element -def dwells(stop_id, sdate, edate): - rows_by_time = s3.download_event_range(parallel.date_range(sdate, edate), stop_id[0]) +def dwells(stop_ids, sdate, edate): + rows_by_time = s3.download_event_range(parallel.date_range(sdate, edate), stop_ids) dwells = [] for maybe_an_arrival, maybe_a_departure in pairwise(rows_by_time): @@ -63,8 +63,8 @@ def dwells(stop_id, sdate, edate): return dwells -def headways(stop_id, sdate, edate): - rows_by_time = s3.download_event_range(parallel.date_range(sdate, edate), stop_id[0]) +def headways(stop_ids, sdate, edate): + rows_by_time = s3.download_event_range(parallel.date_range(sdate, edate), stop_ids) only_departures = filter(lambda row: row['event_type'] in EVENT_DEPARTURE, rows_by_time) @@ -91,9 +91,9 @@ def headways(stop_id, sdate, edate): return headways -def travel_times(stop_a, stop_b, sdate, edate): - rows_by_time_a = s3.download_event_range(parallel.date_range(sdate, edate), stop_a) - rows_by_time_b = s3.download_event_range(parallel.date_range(sdate, edate), stop_b) +def travel_times(stops_a, stops_b, sdate, edate): + rows_by_time_a = s3.download_event_range(parallel.date_range(sdate, edate), stops_a) + rows_by_time_b = s3.download_event_range(parallel.date_range(sdate, edate), stops_b) departures = filter(lambda event: event["event_type"] in EVENT_DEPARTURE, rows_by_time_a) # we reverse arrivals so that if the same train arrives twice (this can happen), From a09f4a7d33ab9db234e800a25a012b5d00ff5f36 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 2 Oct 2021 10:52:34 -0400 Subject: [PATCH 019/152] Add Route 28 --- src/bus_constants/28.json | 166 ++++++++++++++++++++++++++++++++++++++ src/stations.js | 9 ++- 2 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 src/bus_constants/28.json diff --git a/src/bus_constants/28.json b/src/bus_constants/28.json new file mode 100644 index 000000000..f8dc00eb6 --- /dev/null +++ b/src/bus_constants/28.json @@ -0,0 +1,166 @@ +{ + "28": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Mattapan Station", + "branches": null, + "station": "matpn", + "order": 1, + "stops": { + "1": [ + "18511" + ], + "0": [ + "18511" + ] + } + }, + { + "stop_name": "Wellington Hill", + "branches": null, + "station": "wellh", + "order": 2, + "stops": { + "1": [ + "1728" + ], + "0": [ + "1714" + ] + } + }, + { + "stop_name": "Morton Street @ Blue Hill Avenue", + "branches": null, + "station": "mortn", + "order": 3, + "stops": { + "1": [ + "1731" + ], + "0": [ + "11712" + ] + } + }, + { + "stop_name": "Blue Hill Avenue @ Talbot Avenue", + "branches": null, + "station": "bltal", + "order": 4, + "stops": { + "1": [ + "1737" + ], + "0": [ + "419" + ] + } + }, + { + "stop_name": "Franklin Park", + "branches": null, + "station": "frnpk", + "order": 5, + "stops": { + "1": [ + "383" + ], + "0": [ + "415" + ] + } + }, + { + "stop_name": "Grove Hall", + "branches": null, + "station": "ghall", + "order": 6, + "stops": { + "1": [ + "386" + ], + "0": [ + "412" + ] + } + }, + { + "stop_name": "Warren Street @ Boston Latin Academy", + "branches": null, + "station": "latac", + "order": 7, + "stops": { + "1": [ + "390" + ], + "0": [ + "407" + ] + } + }, + { + "stop_name": "Warren Street @ Walnut Avenue", + "branches": null, + "station": "warwl", + "order": 8, + "stops": { + "1": [ + "396" + ], + "0": [ + "40001" + ] + } + }, + { + "stop_name": "Nubian Station", + "branches": null, + "station": "nubn", + "order": 9, + "stops": { + "1": [ + "64000" + ], + "0": [ + "64000" + ] + } + }, + { + "stop_name": "Roxbury Crossing Station (Bus)", + "branches": null, + "station": "roxbs", + "order": 10, + "stops": { + "1": [ + "21148" + ], + "0": [ + "11257" + ] + } + }, + { + "stop_name": "Ruggles", + "branches": null, + "station": "rugg", + "order": 11, + "stops": { + "1": [ + "17861" + ], + "0": [ + "17861", + "17862" + ] + } + } + ] + } +} diff --git a/src/stations.js b/src/stations.js index d3413127b..4db4e7b25 100644 --- a/src/stations.js +++ b/src/stations.js @@ -1,7 +1,12 @@ import { stations as rt_stations } from './constants'; -import bus1 from './bus_constants/1.json'; -const stations = {...rt_stations, ...bus1}; +import bus_1 from './bus_constants/1.json'; +import bus_28 from './bus_constants/28.json'; + +const stations = {...rt_stations, + ...bus_1, + ...bus_28, + }; const all_lines = () => { return Object.keys(stations); From 3523f5b448fde976a28eb567058400928363939a Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 2 Oct 2021 10:52:59 -0400 Subject: [PATCH 020/152] option: skip gzip for debugging --- server/bus/bus2train.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 3b1f8029a..3b008d269 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -71,7 +71,7 @@ def process_events(df): return df -def _write_file(events, outdir): +def _write_file(events, outdir, nozip=False): """ This is a helper that will write the events to disk. It will be called on each "groupby" object, grouping stop_id and service_date @@ -87,14 +87,14 @@ def _write_file(events, outdir): f"Day={service_date.day}", "events.csv.gz") fname.parent.mkdir(parents=True, exist_ok=True) - events.to_csv(fname, index=False, compression='gzip') + events.to_csv(fname, index=False, compression='gzip' if not nozip else None) -def to_disk(df, root): +def to_disk(df, root, nozip=False): """ For each service_date/stop_id group, we call the helper that will write it to disk. """ - df.groupby(['service_date', 'stop_id']).apply(lambda e: _write_file(e, root)) + df.groupby(['service_date', 'stop_id']).apply(lambda e: _write_file(e, root, nozip)) def main(): @@ -105,17 +105,19 @@ def main(): parser.add_argument('--routes', '-r', nargs="*", type=str, help="One note here: we should always be additive with our route set \ in case 2 lines share the same stop id: we need both in the result file.") + parser.add_argument('--nozip', '-nz', action='store_true', help="debug feature to skip gzipping") args = parser.parse_args() input_csv = args.input output_dir = args.output routes = args.routes + no_zip = args.nozip pathlib.Path(output_dir).mkdir(exist_ok=True) data = load_data(input_csv, routes) events = process_events(data) - to_disk(events, output_dir) + to_disk(events, output_dir, nozip=no_zip) if __name__ == "__main__": From 3c4f3fbd7b1e999a594f68d2327efc96e959c634 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 3 Oct 2021 16:40:19 -0400 Subject: [PATCH 021/152] Parse multiple stop params on server --- server/app.py | 37 ++++++++++++-------------------- server/chalicelib/aggregation.py | 12 +++++------ 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/server/app.py b/server/app.py index 76f3dbf29..7b14dea96 100644 --- a/server/app.py +++ b/server/app.py @@ -31,15 +31,6 @@ def parse_user_date(user_date): return date(year=year, month=month, day=day) -def parse_query_stop_args(query_params, expected_stop_param_names): - stops_dict = {} - for stop_param in expected_stop_param_names: - query_value = query_params.get(stop_param) - if query_value: - stops_dict[stop_param] = query_value - return stops_dict - - def mutlidict_to_dict(mutlidict): res_dict = {} for key in mutlidict.keys(): @@ -75,23 +66,23 @@ def healthcheck(): @app.route("/headways/{user_date}", cors=cors_config) def headways_route(user_date): date = parse_user_date(user_date) - stop = app.current_request.query_params["stop"] - return data_funcs.headways(date, [stop]) + stops = app.current_request.query_params.getlist("stop") + return data_funcs.headways(date, stops) @app.route("/dwells/{user_date}", cors=cors_config) def dwells_route(user_date): date = parse_user_date(user_date) - stop = app.current_request.query_params["stop"] - return data_funcs.dwells(date, [stop]) + stops = app.current_request.query_params.getlist("stop") + return data_funcs.dwells(date, stops) @app.route("/traveltimes/{user_date}", cors=cors_config) def traveltime_route(user_date): date = parse_user_date(user_date) - from_stop = app.current_request.query_params["from_stop"] - to_stop = app.current_request.query_params["to_stop"] - return data_funcs.travel_times(date, [from_stop], [to_stop]) + from_stops = app.current_request.query_params.getlist("from_stop") + to_stops = app.current_request.query_params.getlist("to_stop") + return data_funcs.travel_times(date, from_stops, to_stops) @app.route("/alerts/{user_date}", cors=cors_config) @@ -104,10 +95,10 @@ def alerts_route(user_date): def traveltime_aggregate_route(): sdate = parse_user_date(app.current_request.query_params["start_date"]) edate = parse_user_date(app.current_request.query_params["end_date"]) - from_stop = app.current_request.query_params["from_stop"] - to_stop = app.current_request.query_params["to_stop"] + from_stops = app.current_request.query_params.getlist("from_stop") + to_stops = app.current_request.query_params.getlist("to_stop") - response = aggregation.travel_times_over_time(sdate, edate, from_stop, to_stop) + response = aggregation.travel_times_over_time(sdate, edate, from_stops, to_stops) return json.dumps(response, indent=4, sort_keys=True, default=str) @@ -115,9 +106,9 @@ def traveltime_aggregate_route(): def headways_aggregate_route(): sdate = parse_user_date(app.current_request.query_params["start_date"]) edate = parse_user_date(app.current_request.query_params["end_date"]) - stop = app.current_request.query_params["stop"] + stops = app.current_request.query_params.getlist("stop") - response = aggregation.headways_over_time(sdate, edate, stop) + response = aggregation.headways_over_time(sdate, edate, stops) return json.dumps(response, indent=4, sort_keys=True, default=str) @@ -125,9 +116,9 @@ def headways_aggregate_route(): def dwells_aggregate_route(): sdate = parse_user_date(app.current_request.query_params["start_date"]) edate = parse_user_date(app.current_request.query_params["end_date"]) - stop = app.current_request.query_params["stop"] + stops = app.current_request.query_params.getlist("stop") - response = aggregation.dwells_over_time(sdate, edate, stop) + response = aggregation.dwells_over_time(sdate, edate, stops) return json.dumps(response, indent=4, sort_keys=True, default=str) diff --git a/server/chalicelib/aggregation.py b/server/chalicelib/aggregation.py index 5f43e6064..2e72a8191 100644 --- a/server/chalicelib/aggregation.py +++ b/server/chalicelib/aggregation.py @@ -40,8 +40,8 @@ def faster_describe(grouped): return stats.loc[stats['count'] > 4] -def travel_times_over_time(sdate, edate, from_stop, to_stop): - all_data = data_funcs.travel_times(sdate, [from_stop], [to_stop], edate) +def travel_times_over_time(sdate, edate, from_stops, to_stops): + all_data = data_funcs.travel_times(sdate, from_stops, to_stops, edate) if not all_data: return [] @@ -73,8 +73,8 @@ def travel_times_over_time(sdate, edate, from_stop, to_stop): return results.to_dict('records') -def headways_over_time(sdate, edate, stop): - all_data = data_funcs.headways(sdate, [stop], edate) +def headways_over_time(sdate, edate, stops): + all_data = data_funcs.headways(sdate, stops, edate) if not all_data: return [] @@ -106,8 +106,8 @@ def headways_over_time(sdate, edate, stop): return results.to_dict('records') -def dwells_over_time(sdate, edate, stop): - all_data = data_funcs.dwells(sdate, [stop], edate) +def dwells_over_time(sdate, edate, stops): + all_data = data_funcs.dwells(sdate, stops, edate) if not all_data: return [] From 2f563c7b941171ea22ec09d0e487bd61ec4ba96b Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Mon, 4 Oct 2021 20:43:05 -0400 Subject: [PATCH 022/152] important sort for merging files --- server/chalicelib/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/chalicelib/s3.py b/server/chalicelib/s3.py index aaa95d18f..30d946ed2 100644 --- a/server/chalicelib/s3.py +++ b/server/chalicelib/s3.py @@ -56,7 +56,7 @@ def download_single_day_events(date, stops): result = [] for stop_id in stops: result += download_one_event_file(date, stop_id) - return result + return sorted(result, key=lambda row: row["event_time"]) # signature: (date_iterable, [stop_id]) From 854563d6c8fa0a807ca60c0aeeeb2ba4d5d843ec Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 7 Oct 2021 21:47:46 -0400 Subject: [PATCH 023/152] add rte_id and dir_id to bus stop_ids; move to daily-bus-data --- server/bus/bus2train.py | 8 ++++---- server/bus/manifest.py | 4 ++-- server/chalicelib/data_funcs.py | 6 ++---- server/chalicelib/s3.py | 7 ++++++- src/bus_constants/1.json | 36 ++++++++++++++++----------------- 5 files changed, 32 insertions(+), 29 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 3b008d269..6d295f6bc 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -76,12 +76,12 @@ def _write_file(events, outdir, nozip=False): This is a helper that will write the events to disk. It will be called on each "groupby" object, grouping stop_id and service_date """ - service_date, stop_id = events.name + service_date, stop_id, direction_id, route_id = events.name fname = pathlib.Path(outdir, "Events", - "daily-data", - str(stop_id), + "daily-bus-data", + f"{route_id}-{direction_id}-{stop_id}", f"Year={service_date.year}", f"Month={service_date.month}", f"Day={service_date.day}", @@ -94,7 +94,7 @@ def to_disk(df, root, nozip=False): """ For each service_date/stop_id group, we call the helper that will write it to disk. """ - df.groupby(['service_date', 'stop_id']).apply(lambda e: _write_file(e, root, nozip)) + df.groupby(['service_date', 'stop_id', 'direction_id', 'route_id']).apply(lambda e: _write_file(e, root, nozip)) def main(): diff --git a/server/bus/manifest.py b/server/bus/manifest.py index d5da76719..84f1c069e 100644 --- a/server/bus/manifest.py +++ b/server/bus/manifest.py @@ -43,8 +43,8 @@ def create_manifest(df, checkpoint_file): "station": tp_id.lower(), "order": order_guess, "stops": { - "1": inbound.stop_id.tolist(), - "0": outbound.stop_id.tolist() + "1": [f"{rte_id}-1-{stop_id}" for stop_id in inbound.stop_id], + "0": [f"{rte_id}-0-{stop_id}" for stop_id in outbound.stop_id] } } summary.append(this_obj) diff --git a/server/chalicelib/data_funcs.py b/server/chalicelib/data_funcs.py index 9fea3cde1..b459d6c64 100644 --- a/server/chalicelib/data_funcs.py +++ b/server/chalicelib/data_funcs.py @@ -15,10 +15,8 @@ def stamp_to_dt(stamp): def use_S3(date, stops): archival = (date.today() - date).days >= 90 - stop_id = int(stops[0]) - bus = not stop_id in range(70000, 72000) or stop_id in [70618, 71391, 71855] - - return archival or bus + is_bus = '-' in stops[0] + return archival or is_bus def partition_S3_dates(start_date, end_date): CUTOFF = datetime.date.today() - datetime.timedelta(days=90) diff --git a/server/chalicelib/s3.py b/server/chalicelib/s3.py index 30d946ed2..42434406d 100644 --- a/server/chalicelib/s3.py +++ b/server/chalicelib/s3.py @@ -24,13 +24,18 @@ def upload(key, bytes, compress=True): s3.put_object(Bucket=BUCKET, Key=key, Body=bytes) +def is_bus(stop_id): + return '-' in stop_id + def download_one_event_file(date, stop_id): """As advertised: single event file from s3""" year, month, day = date.year, date.month, date.day + folder = 'daily-bus-data' if is_bus(stop_id) else 'daily-data' + key = f"Events/{folder}/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv.gz" + # Download events from S3 try: - key = f"Events/daily-data/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv.gz" decompressed = download(key, 'ascii') except ClientError as ex: diff --git a/src/bus_constants/1.json b/src/bus_constants/1.json index 7e77c1bce..cb935e7aa 100644 --- a/src/bus_constants/1.json +++ b/src/bus_constants/1.json @@ -13,10 +13,10 @@ "order": 1, "stops": { "1": [ - "110" + "1-1-110" ], "0": [ - "110" + "1-0-110" ] } }, @@ -27,10 +27,10 @@ "order": 2, "stops": { "1": [ - "67" + "1-1-67" ], "0": [ - "108" + "1-0-108" ] } }, @@ -41,10 +41,10 @@ "order": 3, "stops": { "1": [ - "72" + "1-1-72" ], "0": [ - "102" + "1-0-102" ] } }, @@ -55,10 +55,10 @@ "order": 4, "stops": { "1": [ - "75" + "1-1-75" ], "0": [ - "97" + "1-0-97" ] } }, @@ -69,10 +69,10 @@ "order": 5, "stops": { "1": [ - "79" + "1-1-79" ], "0": [ - "93" + "1-0-93" ] } }, @@ -83,10 +83,10 @@ "order": 6, "stops": { "1": [ - "187" + "1-1-187" ], "0": [ - "188" + "1-0-188" ] } }, @@ -97,10 +97,10 @@ "order": 7, "stops": { "1": [ - "59" + "1-1-59" ], "0": [ - "10590" + "1-0-10590" ] } }, @@ -111,10 +111,10 @@ "order": 8, "stops": { "1": [ - "62" + "1-1-62" ], "0": [ - "2" + "1-0-2" ] } }, @@ -125,10 +125,10 @@ "order": 9, "stops": { "1": [ - "64" + "1-1-64" ], "0": [ - "64" + "1-0-64" ] } } From b745ec2faac319af3425706203abf765bbfcd2a9 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 7 Oct 2021 21:58:06 -0400 Subject: [PATCH 024/152] update rte 28 --- src/bus_constants/28.json | 46 +++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/bus_constants/28.json b/src/bus_constants/28.json index f8dc00eb6..a4bff68bb 100644 --- a/src/bus_constants/28.json +++ b/src/bus_constants/28.json @@ -13,10 +13,10 @@ "order": 1, "stops": { "1": [ - "18511" + "28-1-18511" ], "0": [ - "18511" + "28-0-18511" ] } }, @@ -27,10 +27,10 @@ "order": 2, "stops": { "1": [ - "1728" + "28-1-1728" ], "0": [ - "1714" + "28-0-1714" ] } }, @@ -41,10 +41,10 @@ "order": 3, "stops": { "1": [ - "1731" + "28-1-1731" ], "0": [ - "11712" + "28-0-11712" ] } }, @@ -55,10 +55,10 @@ "order": 4, "stops": { "1": [ - "1737" + "28-1-1737" ], "0": [ - "419" + "28-0-419" ] } }, @@ -69,10 +69,10 @@ "order": 5, "stops": { "1": [ - "383" + "28-1-383" ], "0": [ - "415" + "28-0-415" ] } }, @@ -83,10 +83,10 @@ "order": 6, "stops": { "1": [ - "386" + "28-1-386" ], "0": [ - "412" + "28-0-412" ] } }, @@ -97,10 +97,10 @@ "order": 7, "stops": { "1": [ - "390" + "28-1-390" ], "0": [ - "407" + "28-0-407" ] } }, @@ -111,10 +111,10 @@ "order": 8, "stops": { "1": [ - "396" + "28-1-396" ], "0": [ - "40001" + "28-0-40001" ] } }, @@ -125,10 +125,10 @@ "order": 9, "stops": { "1": [ - "64000" + "28-1-64000" ], "0": [ - "64000" + "28-0-64000" ] } }, @@ -139,10 +139,10 @@ "order": 10, "stops": { "1": [ - "21148" + "28-1-21148" ], "0": [ - "11257" + "28-0-11257" ] } }, @@ -153,11 +153,11 @@ "order": 11, "stops": { "1": [ - "17861" + "28-1-17861" ], "0": [ - "17861", - "17862" + "28-0-17861", + "28-0-17862" ] } } From 6c685aa0354f620e963757543dbfccfaa3972f5d Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 7 Oct 2021 22:23:34 -0400 Subject: [PATCH 025/152] fix s3-bus logic for aggregation --- server/chalicelib/data_funcs.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/server/chalicelib/data_funcs.py b/server/chalicelib/data_funcs.py index b459d6c64..146b7a73e 100644 --- a/server/chalicelib/data_funcs.py +++ b/server/chalicelib/data_funcs.py @@ -12,19 +12,20 @@ def stamp_to_dt(stamp): dt = datetime.datetime.fromtimestamp(stamp, pytz.timezone("America/New_York")) return dt.strftime(DATE_FORMAT) +def is_bus(stops): + return '-' in stops[0] -def use_S3(date, stops): +def use_S3(date, bus=False): archival = (date.today() - date).days >= 90 - is_bus = '-' in stops[0] - return archival or is_bus + return archival or bus -def partition_S3_dates(start_date, end_date): +def partition_S3_dates(start_date, end_date, bus=False): CUTOFF = datetime.date.today() - datetime.timedelta(days=90) s3_dates = None api_dates = None - if end_date < CUTOFF: + if end_date < CUTOFF or bus: s3_dates = (start_date, end_date) elif CUTOFF <= start_date: api_dates = (start_date, end_date) @@ -37,12 +38,12 @@ def partition_S3_dates(start_date, end_date): def headways(sdate, stops, edate=None): if edate is None: - if use_S3(sdate, stops): + if use_S3(sdate, is_bus(stops)): return s3_historical.headways(stops, sdate, sdate) else: return process_mbta_headways(stops, sdate) - s3_interval, api_interval = partition_S3_dates(sdate, edate) + s3_interval, api_interval = partition_S3_dates(sdate, edate, is_bus(stops)) all_data = [] if s3_interval: start, end = s3_interval @@ -94,12 +95,12 @@ def process_mbta_headways(stops, sdate, edate=None): def travel_times(sdate, from_stops, to_stops, edate=None): if edate is None: - if use_S3(sdate, from_stops): + if use_S3(sdate, is_bus(from_stops)): return s3_historical.travel_times(from_stops, to_stops, sdate, sdate) else: return process_mbta_travel_times(from_stops, to_stops, sdate) - s3_interval, api_interval = partition_S3_dates(sdate, edate) + s3_interval, api_interval = partition_S3_dates(sdate, edate, is_bus(from_stops)) all_data = [] if s3_interval: start, end = s3_interval @@ -144,12 +145,12 @@ def process_mbta_travel_times(from_stops, to_stops, sdate, edate=None): def dwells(sdate, stops, edate=None): if edate is None: - if use_S3(sdate, stops): + if use_S3(sdate, is_bus(stops)): return s3_historical.dwells(stops, sdate, sdate) else: return process_mbta_dwells(stops, sdate) - s3_interval, api_interval = partition_S3_dates(sdate, edate) + s3_interval, api_interval = partition_S3_dates(sdate, edate, is_bus(stops)) all_data = [] if s3_interval: start, end = s3_interval From d199dfeb7e16dab89222c999b78ee311c4998626 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 11 Nov 2021 12:21:43 -0500 Subject: [PATCH 026/152] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 43f4b7877..41951b41b 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ yarn-error.log* .idea .vscode *~ +*.pyc From 64c01d88e223f35c082eff78a012bb926324d9e5 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Mon, 15 Nov 2021 11:10:07 -0500 Subject: [PATCH 027/152] oops, forgot to lint --- server/bus/bus2train.py | 10 +++++----- server/bus/manifest.py | 16 +++++++++------- server/chalicelib/data_funcs.py | 3 +++ server/chalicelib/s3.py | 5 +++-- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 6d295f6bc..70ceb5196 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -58,15 +58,15 @@ def process_events(df): df = df.rename(columns={'half_trip_id': 'trip_id', 'time_point_order': 'stop_sequence', 'actual': 'event_time'}) - df.drop(columns=['time_point_id','standard_type','scheduled','scheduled_headway','headway']) + df.drop(columns=['time_point_id', 'standard_type', 'scheduled', 'scheduled_headway', 'headway']) df['vehicle_id'] = "" df['vehicle_label'] = "" df['event_type'] = df.point_type.map({"Startpoint": ["DEP"], - "Midpoint": ["ARR", "DEP"], - "Endpoint": ["ARR"]}) + "Midpoint": ["ARR", "DEP"], + "Endpoint": ["ARR"]}) df = df.explode('event_type') - df = df[CSV_HEADER] # reorder + df = df[CSV_HEADER] # reorder return df @@ -114,7 +114,7 @@ def main(): no_zip = args.nozip pathlib.Path(output_dir).mkdir(exist_ok=True) - + data = load_data(input_csv, routes) events = process_events(data) to_disk(events, output_dir, nozip=no_zip) diff --git a/server/bus/manifest.py b/server/bus/manifest.py index 84f1c069e..588814929 100644 --- a/server/bus/manifest.py +++ b/server/bus/manifest.py @@ -13,14 +13,14 @@ def load_checkpoints(checkpoint_file): if checkpoint_file: return pd.read_csv(checkpoint_file, index_col="checkpoint_id").to_dict()["checkpoint_name"] else: - return {} + return {} def create_manifest(df, checkpoint_file): station_names = load_checkpoints(checkpoint_file) manifest = {} - + timepoints = df.groupby(['route_id', 'time_point_id', 'stop_id', 'direction_id'])['time_point_order'].agg(pd.Series.mode).reset_index() timepoints.stop_id = timepoints.stop_id.astype(str) @@ -36,7 +36,7 @@ def create_manifest(df, checkpoint_file): if pd.isnull(order_guess): order_guess = 0 else: - order_guess = int(order_guess) # none of that icky int64 stuff, json-serializable please + order_guess = int(order_guess) # none of that icky int64 stuff, json-serializable please this_obj = { "stop_name": station_names.get(tp_id.lower(), ""), "branches": None, @@ -55,11 +55,12 @@ def create_manifest(df, checkpoint_file): "0": "outbound", "1": "inbound" }, - "stations": sorted(summary, key = lambda x: x['order']) + "stations": sorted(summary, key=lambda x: x['order']) } return manifest + def main(): parser = argparse.ArgumentParser() @@ -75,12 +76,13 @@ def main(): checkpoint_file = args.checkpoints data = load_data(input_csv, routes) - + manifest = create_manifest(data, checkpoint_file) - + with open(output_file, "w", encoding="utf-8") as fd: json.dump(manifest, fd, ensure_ascii=False, indent=4) fd.write("\n") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/server/chalicelib/data_funcs.py b/server/chalicelib/data_funcs.py index c39a19fee..a5bffce9e 100644 --- a/server/chalicelib/data_funcs.py +++ b/server/chalicelib/data_funcs.py @@ -12,13 +12,16 @@ def stamp_to_dt(stamp): dt = datetime.datetime.fromtimestamp(stamp, pytz.timezone("America/New_York")) return dt.strftime(DATE_FORMAT) + def is_bus(stops): return '-' in stops[0] + def use_S3(date, bus=False): archival = (date.today() - date).days >= 90 return archival or bus + def partition_S3_dates(start_date, end_date, bus=False): CUTOFF = datetime.date.today() - datetime.timedelta(days=90) diff --git a/server/chalicelib/s3.py b/server/chalicelib/s3.py index 42434406d..513683fdf 100644 --- a/server/chalicelib/s3.py +++ b/server/chalicelib/s3.py @@ -14,7 +14,7 @@ def download(key, encoding="utf8"): obj = s3.get_object(Bucket=BUCKET, Key=key) s3_data = obj["Body"].read() # 32 should detect zlib vs gzip - decompressed = zlib.decompress(s3_data, zlib.MAX_WBITS|32).decode(encoding) + decompressed = zlib.decompress(s3_data, zlib.MAX_WBITS | 32).decode(encoding) return decompressed @@ -27,13 +27,14 @@ def upload(key, bytes, compress=True): def is_bus(stop_id): return '-' in stop_id + def download_one_event_file(date, stop_id): """As advertised: single event file from s3""" year, month, day = date.year, date.month, date.day folder = 'daily-bus-data' if is_bus(stop_id) else 'daily-data' key = f"Events/{folder}/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv.gz" - + # Download events from S3 try: decompressed = download(key, 'ascii') From 0a4a60305417d1189082286ed6928fc9c06c8aac Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Fri, 19 Nov 2021 19:44:51 -0500 Subject: [PATCH 028/152] improve manifest.py readability and usefulness --- server/Pipfile | 2 +- server/Pipfile.lock | 478 ++++++++++++++++++++++++----------------- server/bus/manifest.py | 78 +++---- 3 files changed, 324 insertions(+), 234 deletions(-) diff --git a/server/Pipfile b/server/Pipfile index 969fc4fe1..dacf12cad 100644 --- a/server/Pipfile +++ b/server/Pipfile @@ -14,7 +14,7 @@ chalice = "*" pytz = "*" boto3 = "*" flake8 = "*" -pandas = "*" +pandas = ">=1.3" numpy = "*" importlib-resources = "*" diff --git a/server/Pipfile.lock b/server/Pipfile.lock index 21fe26784..6c1e44ccd 100644 --- a/server/Pipfile.lock +++ b/server/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6ad90070056b08a822939e837920573e105598fe22e0978c9ede5cbc0278bb26" + "sha256": "bf13dc474de09dc13c7847c77ca32fe8fae7f23f0b9ac756c8ac86cdf14d8dc6" }, "pipfile-spec": 6, "requires": { @@ -18,69 +18,74 @@ "default": { "attrs": { "hashes": [ - "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", - "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1", + "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb" ], - "version": "==20.3.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.2.0" }, "blessed": { "hashes": [ - "sha256:8371d69ac55558e4b1591964873d6721136e9ea17a730aeb3add7d27761b134b", - "sha256:a9a774fc6eda05248735b0d86e866d640ca2fef26038878f7e4d23f7749a1e40" + "sha256:1f2d462631b2b6d2d4c3c65b54ef79ad87a6ca2dd55255df2f8d739fcc8a1ddb", + "sha256:4db0f94e5761aea330b528e84a250027ffe996b5a94bf03e502600c9a5ad7a61" ], - "version": "==1.17.6" + "markers": "python_version >= '2.7'", + "version": "==1.19.0" }, "boto3": { "hashes": [ - "sha256:ac10d832ad716281da6ca77cea824d723af479f8611087dee4b0489c48c32fd9", - "sha256:e2ef25afc36a301199bfbd662aef46dd11ed0db9baf96fce111db4043928065b" + "sha256:20a5109a37414a52c55d2048388f02cb7cf46fc0ca7be08b3bf81f4c5c053feb", + "sha256:e2b5ce2679424a6c2bfc2ee4bb42d9100c8c08b21eff8d74cff85a7243a76d7b" ], "index": "pypi", - "version": "==1.17.64" + "version": "==1.20.10" }, "botocore": { "hashes": [ - "sha256:7f54fa67b45cf767e1e4045741674cfdc47a3f424fe6f37570ae3ff1ca1e1e2a", - "sha256:d8992096d9c04e7be331924a59677e591cce6a3c6bd3a4c8fe26b00700d5255a" + "sha256:0adda9a4a95221027312eaaee0ec9fe2239fb2f285fced3ddca54b1310b864ee", + "sha256:11670d3ac14eed1122e0154a7e1563c2c270beef43996466f8d11fbf5cf31611" ], - "version": "==1.20.85" + "markers": "python_version >= '3.6'", + "version": "==1.23.10" }, "certifi": { "hashes": [ - "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", - "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" + "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", + "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569" ], - "version": "==2021.5.30" + "version": "==2021.10.8" }, "chalice": { "hashes": [ - "sha256:35a3ae531c29359bab98cdc3f09e2015856473a24c9774203a20f57386715e57", - "sha256:acfaaf0b2c7bf79bf0dcf93428e730f4f46e598ac1b0424c8d99008db816b87d" + "sha256:d36534c949c8554eab99678535673e8e7d1f327d6c9dd3d8241e1e08036970b6", + "sha256:e4979097272f3b558a0a539f8555725978d212f14d8841452694aa33b7110e11" ], "index": "pypi", - "version": "==1.22.4" + "version": "==1.26.2" }, - "chardet": { + "charset-normalizer": { "hashes": [ - "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", - "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + "sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0", + "sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b" ], - "version": "==4.0.0" + "markers": "python_version >= '3'", + "version": "==2.0.7" }, "click": { "hashes": [ - "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", - "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", + "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" ], - "version": "==7.1.2" + "markers": "python_version >= '3.6'", + "version": "==8.0.3" }, "flake8": { "hashes": [ - "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378", - "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a" + "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", + "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" ], "index": "pypi", - "version": "==3.9.1" + "version": "==4.0.1" }, "gunicorn": { "hashes": [ @@ -92,39 +97,41 @@ }, "idna": { "hashes": [ - "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", - "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "version": "==2.10" + "markers": "python_version >= '3'", + "version": "==3.3" }, "importlib-metadata": { "hashes": [ - "sha256:960d52ba7c21377c990412aca380bf3642d734c2eaab78a2c39319f67c6a5786", - "sha256:e592faad8de1bda9fe920cf41e15261e7131bcf266c30306eec00e8e225c1dd5" + "sha256:057e92c15bc8d9e8109738a48db0ccb31b4d9d5cfbee5a8670879a30be66304b", + "sha256:b7e52a1f8dec14a75ea73e0891f3060099ca1d8e6a462a4dff11c3e119ea1b31" ], "markers": "python_version < '3.8'", - "version": "==4.4.0" + "version": "==4.2.0" }, "importlib-resources": { "hashes": [ - "sha256:642586fc4740bd1cad7690f836b3321309402b20b332529f25617ff18e8e1370", - "sha256:ebab3efe74d83b04d6bf5cd9a17f0c5c93e60fb60f30c90f56265fce4682a469" + "sha256:33a95faed5fc19b4bc16b29a6eeae248a3fe69dd55d4d229d2b480e23eeaad45", + "sha256:d756e2f85dd4de2ba89be0b21dba2a3bbec2e871a42a3a16719258a11f87506b" ], "index": "pypi", - "version": "==5.1.2" + "version": "==5.4.0" }, "inquirer": { "hashes": [ - "sha256:d15e15de1ad5696f1967e7a23d8e2fce69d2e41a70b008948d676881ed94c3a5", - "sha256:e819188de0ca7985a99c282176c6f50fb08b0d33867fd1965d3f3e97d6c8f83f" + "sha256:08cdb7386ee01c76f91cd9813525f8723234f224dd3407019ac340ff89fdd731", + "sha256:237c14a68bcf0b2950899aa11bcc342de613e9389e4bf6fcd2ef97fcb3b1590a" ], - "version": "==2.7.0" + "version": "==2.8.0" }, "jmespath": { "hashes": [ "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.0" }, "json-api-doc": { @@ -151,126 +158,150 @@ }, "numpy": { "hashes": [ - "sha256:2428b109306075d89d21135bdd6b785f132a1f5a3260c371cee1fae427e12727", - "sha256:377751954da04d4a6950191b20539066b4e19e3b559d4695399c5e8e3e683bf6", - "sha256:4703b9e937df83f5b6b7447ca5912b5f5f297aba45f91dbbbc63ff9278c7aa98", - "sha256:471c0571d0895c68da309dacee4e95a0811d0a9f9f532a48dc1bea5f3b7ad2b7", - "sha256:61d5b4cf73622e4d0c6b83408a16631b670fc045afd6540679aa35591a17fe6d", - "sha256:6c915ee7dba1071554e70a3664a839fbc033e1d6528199d4621eeaaa5487ccd2", - "sha256:6e51e417d9ae2e7848314994e6fc3832c9d426abce9328cf7571eefceb43e6c9", - "sha256:719656636c48be22c23641859ff2419b27b6bdf844b36a2447cb39caceb00935", - "sha256:780ae5284cb770ade51d4b4a7dce4faa554eb1d88a56d0e8b9f35fca9b0270ff", - "sha256:878922bf5ad7550aa044aa9301d417e2d3ae50f0f577de92051d739ac6096cee", - "sha256:924dc3f83de20437de95a73516f36e09918e9c9c18d5eac520062c49191025fb", - "sha256:97ce8b8ace7d3b9288d88177e66ee75480fb79b9cf745e91ecfe65d91a856042", - "sha256:9c0fab855ae790ca74b27e55240fe4f2a36a364a3f1ebcfd1fb5ac4088f1cec3", - "sha256:9cab23439eb1ebfed1aaec9cd42b7dc50fc96d5cd3147da348d9161f0501ada5", - "sha256:a8e6859913ec8eeef3dbe9aed3bf475347642d1cdd6217c30f28dee8903528e6", - "sha256:aa046527c04688af680217fffac61eec2350ef3f3d7320c07fd33f5c6e7b4d5f", - "sha256:abc81829c4039e7e4c30f7897938fa5d4916a09c2c7eb9b244b7a35ddc9656f4", - "sha256:bad70051de2c50b1a6259a6df1daaafe8c480ca98132da98976d8591c412e737", - "sha256:c73a7975d77f15f7f68dacfb2bca3d3f479f158313642e8ea9058eea06637931", - "sha256:d15007f857d6995db15195217afdbddfcd203dfaa0ba6878a2f580eaf810ecd6", - "sha256:d76061ae5cab49b83a8cf3feacefc2053fac672728802ac137dd8c4123397677", - "sha256:e8e4fbbb7e7634f263c5b0150a629342cc19b47c5eba8d1cd4363ab3455ab576", - "sha256:e9459f40244bb02b2f14f6af0cd0732791d72232bbb0dc4bab57ef88e75f6935", - "sha256:edb1f041a9146dcf02cd7df7187db46ab524b9af2515f392f337c7cbbf5b52cd" + "sha256:0b78ecfa070460104934e2caf51694ccd00f37d5e5dbe76f021b1b0b0d221823", + "sha256:1247ef28387b7bb7f21caf2dbe4767f4f4175df44d30604d42ad9bd701ebb31f", + "sha256:1403b4e2181fc72664737d848b60e65150f272fe5a1c1cbc16145ed43884065a", + "sha256:170b2a0805c6891ca78c1d96ee72e4c3ed1ae0a992c75444b6ab20ff038ba2cd", + "sha256:2e4ed57f45f0aa38beca2a03b6532e70e548faf2debbeb3291cfc9b315d9be8f", + "sha256:32fe5b12061f6446adcbb32cf4060a14741f9c21e15aaee59a207b6ce6423469", + "sha256:34f3456f530ae8b44231c63082c8899fe9c983fd9b108c997c4b1c8c2d435333", + "sha256:4c9c23158b87ed0e70d9a50c67e5c0b3f75bcf2581a8e34668d4e9d7474d76c6", + "sha256:5d95668e727c75b3f5088ec7700e260f90ec83f488e4c0aaccb941148b2cd377", + "sha256:615d4e328af7204c13ae3d4df7615a13ff60a49cb0d9106fde07f541207883ca", + "sha256:69077388c5a4b997442b843dbdc3a85b420fb693ec8e33020bb24d647c164fa5", + "sha256:74b85a17528ca60cf98381a5e779fc0264b4a88b46025e6bcbe9621f46bb3e63", + "sha256:81225e58ef5fce7f1d80399575576fc5febec79a8a2742e8ef86d7b03beef49f", + "sha256:8890b3360f345e8360133bc078d2dacc2843b6ee6059b568781b15b97acbe39f", + "sha256:92aafa03da8658609f59f18722b88f0a73a249101169e28415b4fa148caf7e41", + "sha256:9864424631775b0c052f3bd98bc2712d131b3e2cd95d1c0c68b91709170890b0", + "sha256:9e6f5f50d1eff2f2f752b3089a118aee1ea0da63d56c44f3865681009b0af162", + "sha256:a3deb31bc84f2b42584b8c4001c85d1934dbfb4030827110bc36bfd11509b7bf", + "sha256:ad010846cdffe7ec27e3f933397f8a8d6c801a48634f419e3d075db27acf5880", + "sha256:b1e2312f5b8843a3e4e8224b2b48fe16119617b8fc0a54df8f50098721b5bed2", + "sha256:bc988afcea53e6156546e5b2885b7efab089570783d9d82caf1cfd323b0bb3dd", + "sha256:c449eb870616a7b62e097982c622d2577b3dbc800aaf8689254ec6e0197cbf1e", + "sha256:c74c699b122918a6c4611285cc2cad4a3aafdb135c22a16ec483340ef97d573c", + "sha256:c885bfc07f77e8fee3dc879152ba993732601f1f11de248d4f357f0ffea6a6d4", + "sha256:e3c3e990274444031482a31280bf48674441e0a5b55ddb168f3a6db3e0c38ec8", + "sha256:e4799be6a2d7d3c33699a6f77201836ac975b2e1b98c2a07f66a38f499cb50ce", + "sha256:e6c76a87633aa3fa16614b61ccedfae45b91df2767cf097aa9c933932a7ed1e0", + "sha256:e89717274b41ebd568cd7943fc9418eeb49b1785b66031bc8a7f6300463c5898", + "sha256:f5162ec777ba7138906c9c274353ece5603646c6965570d82905546579573f73", + "sha256:fde96af889262e85aa033f8ee1d3241e32bf36228318a61f1ace579df4e8170d" ], "index": "pypi", - "version": "==1.20.2" + "version": "==1.21.4" }, "pandas": { "hashes": [ - "sha256:167693a80abc8eb28051fbd184c1b7afd13ce2c727a5af47b048f1ea3afefff4", - "sha256:2111c25e69fa9365ba80bbf4f959400054b2771ac5d041ed19415a8b488dc70a", - "sha256:298f0553fd3ba8e002c4070a723a59cdb28eda579f3e243bc2ee397773f5398b", - "sha256:2b063d41803b6a19703b845609c0b700913593de067b552a8b24dd8eeb8c9895", - "sha256:2cb7e8f4f152f27dc93f30b5c7a98f6c748601ea65da359af734dd0cf3fa733f", - "sha256:52d2472acbb8a56819a87aafdb8b5b6d2b3386e15c95bde56b281882529a7ded", - "sha256:612add929bf3ba9d27b436cc8853f5acc337242d6b584203f207e364bb46cb12", - "sha256:649ecab692fade3cbfcf967ff936496b0cfba0af00a55dfaacd82bdda5cb2279", - "sha256:68d7baa80c74aaacbed597265ca2308f017859123231542ff8a5266d489e1858", - "sha256:8d4c74177c26aadcfb4fd1de6c1c43c2bf822b3e0fc7a9b409eeaf84b3e92aaa", - "sha256:971e2a414fce20cc5331fe791153513d076814d30a60cd7348466943e6e909e4", - "sha256:9db70ffa8b280bb4de83f9739d514cd0735825e79eef3a61d312420b9f16b758", - "sha256:b730add5267f873b3383c18cac4df2527ac4f0f0eed1c6cf37fcb437e25cf558", - "sha256:bd659c11a4578af740782288cac141a322057a2e36920016e0fc7b25c5a4b686", - "sha256:c601c6fdebc729df4438ec1f62275d6136a0dd14d332fc0e8ce3f7d2aadb4dd6", - "sha256:d0877407359811f7b853b548a614aacd7dea83b0c0c84620a9a643f180060950" + "sha256:003ba92db58b71a5f8add604a17a059f3068ef4e8c0c365b088468d0d64935fd", + "sha256:10e10a2527db79af6e830c3d5842a4d60383b162885270f8cffc15abca4ba4a9", + "sha256:22808afb8f96e2269dcc5b846decacb2f526dd0b47baebc63d913bf847317c8f", + "sha256:2d1dc09c0013d8faa7474574d61b575f9af6257ab95c93dcf33a14fd8d2c1bab", + "sha256:35c77609acd2e4d517da41bae0c11c70d31c87aae8dd1aabd2670906c6d2c143", + "sha256:372d72a3d8a5f2dbaf566a5fa5fa7f230842ac80f29a931fb4b071502cf86b9a", + "sha256:42493f8ae67918bf129869abea8204df899902287a7f5eaf596c8e54e0ac7ff4", + "sha256:4acc28364863127bca1029fb72228e6f473bb50c32e77155e80b410e2068eeac", + "sha256:5298a733e5bfbb761181fd4672c36d0c627320eb999c59c65156c6a90c7e1b4f", + "sha256:5ba0aac1397e1d7b654fccf263a4798a9e84ef749866060d19e577e927d66e1b", + "sha256:9707bdc1ea9639c886b4d3be6e2a45812c1ac0c2080f94c31b71c9fa35556f9b", + "sha256:a2aa18d3f0b7d538e21932f637fbfe8518d085238b429e4790a35e1e44a96ffc", + "sha256:a388960f979665b447f0847626e40f99af8cf191bce9dc571d716433130cb3a7", + "sha256:a51528192755f7429c5bcc9e80832c517340317c861318fea9cea081b57c9afd", + "sha256:b528e126c13816a4374e56b7b18bfe91f7a7f6576d1aadba5dee6a87a7f479ae", + "sha256:c1aa4de4919358c5ef119f6377bc5964b3a7023c23e845d9db7d9016fa0c5b1c", + "sha256:c2646458e1dce44df9f71a01dc65f7e8fa4307f29e5c0f2f92c97f47a5bf22f5", + "sha256:c2f44425594ae85e119459bb5abb0748d76ef01d9c08583a667e3339e134218e", + "sha256:d47750cf07dee6b55d8423471be70d627314277976ff2edd1381f02d52dbadf9", + "sha256:d99d2350adb7b6c3f7f8f0e5dfb7d34ff8dd4bc0a53e62c445b7e43e163fce63", + "sha256:dd324f8ee05925ee85de0ea3f0d66e1362e8c80799eb4eb04927d32335a3e44a", + "sha256:eaca36a80acaacb8183930e2e5ad7f71539a66805d6204ea88736570b2876a7b", + "sha256:f567e972dce3bbc3a8076e0b675273b4a9e8576ac629149cf8286ee13c259ae5", + "sha256:fe48e4925455c964db914b958f6e7032d285848b7538a5e1b19aeb26ffaea3ec" ], "index": "pypi", - "version": "==1.2.4" + "version": "==1.3.4" }, "pycodestyle": { "hashes": [ - "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068", - "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef" + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" ], - "version": "==2.7.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" }, "pyflakes": { "hashes": [ - "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3", - "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db" + "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", + "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" ], - "version": "==2.3.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.0" }, "python-dateutil": { "hashes": [ - "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", - "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "version": "==2.8.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" }, "python-editor": { "hashes": [ "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d", "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b", - "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8" + "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8", + "sha256:c3da2053dbab6b29c94e43c486ff67206eafbe7eb52dbec7390b5e2fb05aac77", + "sha256:ea87e17f6ec459e780e4221f295411462e0d0810858e055fc514684350a2f522" ], "version": "==1.0.4" }, "pytz": { "hashes": [ - "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da", - "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798" + "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c", + "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326" ], "index": "pypi", - "version": "==2021.1" + "version": "==2021.3" }, "pyyaml": { "hashes": [ - "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", - "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", - "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", - "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", - "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", - "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", - "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", - "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", - "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", - "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", - "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", - "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", - "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", - "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", - "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", - "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", - "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", - "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", - "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", - "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", - "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", - "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", - "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", - "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", - "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", - "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", - "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", - "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", - "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" - ], - "version": "==5.4.1" + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "markers": "python_version >= '3.6'", + "version": "==6.0" }, "readchar": { "hashes": [ @@ -281,42 +312,43 @@ }, "requests": { "hashes": [ - "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", - "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", + "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" ], "index": "pypi", - "version": "==2.25.1" + "version": "==2.26.0" }, "s3transfer": { "hashes": [ - "sha256:9b3752887a2880690ce628bc263d6d13a3864083aeacff4890c1c9839a5eb0bc", - "sha256:cb022f4b16551edebbb31a377d3f09600dbada7363d8c5db7976e7f47732e1b2" + "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c", + "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803" ], - "version": "==0.4.2" + "markers": "python_version >= '3.6'", + "version": "==0.5.0" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "typing-extensions": { "hashes": [ - "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", - "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", - "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" + "sha256:2cdf80e4e04866a9b3689a51869016d36db0814d84b8d8a568d22781d45d27ed", + "sha256:829704698b22e13ec9eaf959122315eabb370b0884400e9818334d8b677023d9" ], "markers": "python_version < '3.8'", - "version": "==3.10.0.0" + "version": "==4.0.0" }, "urllib3": { "hashes": [ - "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c", - "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098" + "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece", + "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844" ], - "index": "pypi", - "version": "==1.26.5" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.7" }, "wcwidth": { "hashes": [ @@ -327,34 +359,37 @@ }, "wheel": { "hashes": [ - "sha256:78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e", - "sha256:e11eefd162658ea59a60a0f6c7d493a7190ea4b9a85e335b33489d9f17e0245e" + "sha256:21014b2bd93c6d0034b6ba5d35e4eb284340e09d63c59aef6fc14b0f346146fd", + "sha256:e2ef7239991699e3355d54f8e968a21bb940a1dbf34a4d226741e64462516fad" ], - "version": "==0.36.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.37.0" }, "zipp": { "hashes": [ - "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76", - "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098" + "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832", + "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc" ], - "markers": "python_version < '3.8'", - "version": "==3.4.1" + "markers": "python_version < '3.10'", + "version": "==3.6.0" } }, "develop": { "astroid": { "hashes": [ - "sha256:3c9a2d84354185d13213ff2640ec03d39168dbcd13648abc84fb13ca3b2e2761", - "sha256:d66a600e1602736a0f24f725a511b0e50d12eb18f54b31ec276d2c26a0a62c6a" + "sha256:11f7356737b624c42e21e71fe85eea6875cb94c03c82ac76bd535a0ff10b0f25", + "sha256:abc423a1e85bc1553954a14f2053473d2b7f8baf32eae62a328be24f436b5107" ], - "version": "==2.5.7" + "markers": "python_version ~= '3.6'", + "version": "==2.8.5" }, "isort": { "hashes": [ - "sha256:0a943902919f65c5684ac4e0154b1ad4fac6dcaa5d9f3426b732f1c8b5419be6", - "sha256:2bb1680aad211e3c9944dbce1d4ba09a989f04e238296c87fe2139faa26d655d" + "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7", + "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951" ], - "version": "==5.8.0" + "markers": "python_version < '4.0' and python_full_version >= '3.6.1'", + "version": "==5.10.1" }, "lazy-object-proxy": { "hashes": [ @@ -381,6 +416,7 @@ "sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93", "sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==1.6.0" }, "mccabe": { @@ -390,71 +426,119 @@ ], "version": "==0.6.1" }, + "platformdirs": { + "hashes": [ + "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2", + "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d" + ], + "markers": "python_version >= '3.6'", + "version": "==2.4.0" + }, "pylint": { "hashes": [ - "sha256:586d8fa9b1891f4b725f587ef267abe2a1bad89d6b184520c7f07a253dd6e217", - "sha256:f7e2072654a6b6afdf5e2fb38147d3e2d2d43c89f648637baab63e026481279b" + "sha256:0f358e221c45cbd4dad2a1e4b883e75d28acdcccd29d40c76eb72b307269b126", + "sha256:2c9843fff1a88ca0ad98a256806c82c5a8f86086e7ccbdb93297d86c3f90c436" ], "index": "pypi", - "version": "==2.8.2" + "version": "==2.11.1" }, "toml": { "hashes": [ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "typed-ast": { "hashes": [ - "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace", - "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff", - "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266", - "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528", - "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6", - "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808", - "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4", - "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363", - "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341", - "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04", - "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41", - "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e", - "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3", - "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899", - "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805", - "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c", - "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c", - "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39", - "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a", - "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3", - "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7", - "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f", - "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075", - "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0", - "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40", - "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428", - "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927", - "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3", - "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f", - "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65" - ], - "markers": "implementation_name == 'cpython' and python_version < '3.8'", - "version": "==1.4.3" + "sha256:14fed8820114a389a2b7e91624db5f85f3f6682fda09fe0268a59aabd28fe5f5", + "sha256:155b74b078be842d2eb630dd30a280025eca0a5383c7d45853c27afee65f278f", + "sha256:224afecb8b39739f5c9562794a7c98325cb9d972712e1a98b6989a4720219541", + "sha256:361b9e5d27bd8e3ccb6ea6ad6c4f3c0be322a1a0f8177db6d56264fa0ae40410", + "sha256:37ba2ab65a0028b1a4f2b61a8fe77f12d242731977d274a03d68ebb751271508", + "sha256:49af5b8f6f03ed1eb89ee06c1d7c2e7c8e743d720c3746a5857609a1abc94c94", + "sha256:51040bf45aacefa44fa67fb9ebcd1f2bec73182b99a532c2394eea7dabd18e24", + "sha256:52ca2b2b524d770bed7a393371a38e91943f9160a190141e0df911586066ecda", + "sha256:618912cbc7e17b4aeba86ffe071698c6e2d292acbd6d1d5ec1ee724b8c4ae450", + "sha256:65c81abbabda7d760df7304d843cc9dbe7ef5d485504ca59a46ae2d1731d2428", + "sha256:7b310a207ee9fde3f46ba327989e6cba4195bc0c8c70a158456e7b10233e6bed", + "sha256:7e6731044f748340ef68dcadb5172a4b1f40847a2983fe3983b2a66445fbc8e6", + "sha256:806e0c7346b9b4af8c62d9a29053f484599921a4448c37fbbcbbf15c25138570", + "sha256:a67fd5914603e2165e075f1b12f5a8356bfb9557e8bfb74511108cfbab0f51ed", + "sha256:e4374a76e61399a173137e7984a1d7e356038cf844f24fd8aea46c8029a2f712", + "sha256:e8a9b9c87801cecaad3b4c2b8876387115d1a14caa602c1618cedbb0cb2a14e6", + "sha256:ea517c2bb11c5e4ba7a83a91482a2837041181d57d3ed0749a6c382a2b6b7086", + "sha256:ec184dfb5d3d11e82841dbb973e7092b75f306b625fad7b2e665b64c5d60ab3f", + "sha256:ff4ad88271aa7a55f19b6a161ed44e088c393846d954729549e3cde8257747bb" + ], + "markers": "python_version < '3.8' and implementation_name == 'cpython'", + "version": "==1.5.0" }, "typing-extensions": { "hashes": [ - "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", - "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", - "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" + "sha256:2cdf80e4e04866a9b3689a51869016d36db0814d84b8d8a568d22781d45d27ed", + "sha256:829704698b22e13ec9eaf959122315eabb370b0884400e9818334d8b677023d9" ], "markers": "python_version < '3.8'", - "version": "==3.10.0.0" + "version": "==4.0.0" }, "wrapt": { "hashes": [ - "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" - ], - "version": "==1.12.1" + "sha256:086218a72ec7d986a3eddb7707c8c4526d677c7b35e355875a0fe2918b059179", + "sha256:0877fe981fd76b183711d767500e6b3111378ed2043c145e21816ee589d91096", + "sha256:0a017a667d1f7411816e4bf214646d0ad5b1da2c1ea13dec6c162736ff25a374", + "sha256:0cb23d36ed03bf46b894cfec777eec754146d68429c30431c99ef28482b5c1df", + "sha256:1fea9cd438686e6682271d36f3481a9f3636195578bab9ca3382e2f5f01fc185", + "sha256:220a869982ea9023e163ba915077816ca439489de6d2c09089b219f4e11b6785", + "sha256:25b1b1d5df495d82be1c9d2fad408f7ce5ca8a38085e2da41bb63c914baadff7", + "sha256:2dded5496e8f1592ec27079b28b6ad2a1ef0b9296d270f77b8e4a3a796cf6909", + "sha256:2ebdde19cd3c8cdf8df3fc165bc7827334bc4e353465048b36f7deeae8ee0918", + "sha256:43e69ffe47e3609a6aec0fe723001c60c65305784d964f5007d5b4fb1bc6bf33", + "sha256:46f7f3af321a573fc0c3586612db4decb7eb37172af1bc6173d81f5b66c2e068", + "sha256:47f0a183743e7f71f29e4e21574ad3fa95676136f45b91afcf83f6a050914829", + "sha256:498e6217523111d07cd67e87a791f5e9ee769f9241fcf8a379696e25806965af", + "sha256:4b9c458732450ec42578b5642ac53e312092acf8c0bfce140ada5ca1ac556f79", + "sha256:51799ca950cfee9396a87f4a1240622ac38973b6df5ef7a41e7f0b98797099ce", + "sha256:5601f44a0f38fed36cc07db004f0eedeaadbdcec90e4e90509480e7e6060a5bc", + "sha256:5f223101f21cfd41deec8ce3889dc59f88a59b409db028c469c9b20cfeefbe36", + "sha256:610f5f83dd1e0ad40254c306f4764fcdc846641f120c3cf424ff57a19d5f7ade", + "sha256:6a03d9917aee887690aa3f1747ce634e610f6db6f6b332b35c2dd89412912bca", + "sha256:705e2af1f7be4707e49ced9153f8d72131090e52be9278b5dbb1498c749a1e32", + "sha256:766b32c762e07e26f50d8a3468e3b4228b3736c805018e4b0ec8cc01ecd88125", + "sha256:77416e6b17926d953b5c666a3cb718d5945df63ecf922af0ee576206d7033b5e", + "sha256:778fd096ee96890c10ce96187c76b3e99b2da44e08c9e24d5652f356873f6709", + "sha256:78dea98c81915bbf510eb6a3c9c24915e4660302937b9ae05a0947164248020f", + "sha256:7dd215e4e8514004c8d810a73e342c536547038fb130205ec4bba9f5de35d45b", + "sha256:7dde79d007cd6dfa65afe404766057c2409316135cb892be4b1c768e3f3a11cb", + "sha256:81bd7c90d28a4b2e1df135bfbd7c23aee3050078ca6441bead44c42483f9ebfb", + "sha256:85148f4225287b6a0665eef08a178c15097366d46b210574a658c1ff5b377489", + "sha256:865c0b50003616f05858b22174c40ffc27a38e67359fa1495605f96125f76640", + "sha256:87883690cae293541e08ba2da22cacaae0a092e0ed56bbba8d018cc486fbafbb", + "sha256:8aab36778fa9bba1a8f06a4919556f9f8c7b33102bd71b3ab307bb3fecb21851", + "sha256:8c73c1a2ec7c98d7eaded149f6d225a692caa1bd7b2401a14125446e9e90410d", + "sha256:936503cb0a6ed28dbfa87e8fcd0a56458822144e9d11a49ccee6d9a8adb2ac44", + "sha256:944b180f61f5e36c0634d3202ba8509b986b5fbaf57db3e94df11abee244ba13", + "sha256:96b81ae75591a795d8c90edc0bfaab44d3d41ffc1aae4d994c5aa21d9b8e19a2", + "sha256:981da26722bebb9247a0601e2922cedf8bb7a600e89c852d063313102de6f2cb", + "sha256:ae9de71eb60940e58207f8e71fe113c639da42adb02fb2bcbcaccc1ccecd092b", + "sha256:b73d4b78807bd299b38e4598b8e7bd34ed55d480160d2e7fdaabd9931afa65f9", + "sha256:d4a5f6146cfa5c7ba0134249665acd322a70d1ea61732723c7d3e8cc0fa80755", + "sha256:dd91006848eb55af2159375134d724032a2d1d13bcc6f81cd8d3ed9f2b8e846c", + "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a", + "sha256:e6906d6f48437dfd80464f7d7af1740eadc572b9f7a4301e7dd3d65db285cacf", + "sha256:e92d0d4fa68ea0c02d39f1e2f9cb5bc4b4a71e8c442207433d8db47ee79d7aa3", + "sha256:e94b7d9deaa4cc7bac9198a58a7240aaf87fe56c6277ee25fa5b3aa1edebd229", + "sha256:ea3e746e29d4000cd98d572f3ee2a6050a4f784bb536f4ac1f035987fc1ed83e", + "sha256:ec7e20258ecc5174029a0f391e1b948bf2906cd64c198a9b8b281b811cbc04de", + "sha256:ec9465dd69d5657b5d2fa6133b3e1e989ae27d29471a672416fd729b429eb554", + "sha256:f122ccd12fdc69628786d0c947bdd9cb2733be8f800d88b5a37c57f1f1d73c10", + "sha256:f99c0489258086308aad4ae57da9e8ecf9e1f3f30fa35d5e170b4d4896554d80", + "sha256:f9c51d9af9abb899bd34ace878fbec8bf357b3194a10c4e8e0a25512826ef056", + "sha256:fd76c47f20984b43d93de9a82011bb6e5f8325df6c9ed4d8310029a55fa361ea" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.13.3" } } } diff --git a/server/bus/manifest.py b/server/bus/manifest.py index 588814929..ab71abbba 100644 --- a/server/bus/manifest.py +++ b/server/bus/manifest.py @@ -11,52 +11,58 @@ def load_checkpoints(checkpoint_file): output: dict(checkpoint_id -> station_name) or {} """ if checkpoint_file: - return pd.read_csv(checkpoint_file, index_col="checkpoint_id").to_dict()["checkpoint_name"] + df = pd.read_csv(checkpoint_file, index_col="checkpoint_id") + df.index = df.index.str.lower() + return df.to_dict()["checkpoint_name"] else: return {} -def create_manifest(df, checkpoint_file): - station_names = load_checkpoints(checkpoint_file) +def emit_stop_obj(entry, branches = False): + tpt_id = entry.name + if entry['counts'].sum() < 20: + # this timepoint is very infrequently used + return None + + return { + "stop_name": entry.stop_name.iloc[0], + "branches": entry.route_id.unique().tolist() if branches else None, + "station": tpt_id, + "order": None, # TODO: determine the order guess + "stops": { + "1": entry.loc[entry.direction_id == 1].full_stop_id.tolist(), + "0": entry.loc[entry.direction_id == 0].full_stop_id.tolist(), + } + } + - manifest = {} - - timepoints = df.groupby(['route_id', 'time_point_id', 'stop_id', 'direction_id'])['time_point_order'].agg(pd.Series.mode).reset_index() - timepoints.stop_id = timepoints.stop_id.astype(str) - - for rte_id, points in timepoints.groupby('route_id'): - summary = [] - for tp_id, info in points.groupby('time_point_id'): - inbound = info.loc[info.direction_id == 1] - outbound = info.loc[info.direction_id == 0] - - # This assumes a very linear route: any branches need to be added manually - # In addition, stop_order should be double checked, since this is just a guess - order_guess = inbound.time_point_order.max() - if pd.isnull(order_guess): - order_guess = 0 - else: - order_guess = int(order_guess) # none of that icky int64 stuff, json-serializable please - this_obj = { - "stop_name": station_names.get(tp_id.lower(), ""), - "branches": None, - "station": tp_id.lower(), - "order": order_guess, - "stops": { - "1": [f"{rte_id}-1-{stop_id}" for stop_id in inbound.stop_id], - "0": [f"{rte_id}-0-{stop_id}" for stop_id in outbound.stop_id] - } - } - summary.append(this_obj) - - manifest[rte_id] = { +def create_manifest(df, routes, checkpoint_file): + output_route_name = '/'.join(routes) + + tpts = df[['route_id', 'stop_id', 'time_point_id', 'direction_id']].value_counts(dropna=False).rename("counts").reset_index() + + # use checkpoint file to map time_point_id to stop_name + station_names = load_checkpoints(checkpoint_file) + tpts['stop_name'] = tpts['time_point_id'].str.lower().map(station_names) + + # Create full stop id e.g. '66-0-64000' + tpts['full_stop_id'] = tpts[['route_id', 'direction_id', 'stop_id']].astype(str).agg('-'.join, axis=1) + + stop_objs = tpts.groupby('time_point_id', dropna=False).apply( + lambda x: emit_stop_obj(x, len(routes) > 1) + ).dropna() + # TODO: include order + + manifest = { + output_route_name: { "type": "bus", "direction": { "0": "outbound", "1": "inbound" }, - "stations": sorted(summary, key=lambda x: x['order']) + "stations": stop_objs.values.tolist() # TODO: sorted } + } return manifest @@ -77,7 +83,7 @@ def main(): data = load_data(input_csv, routes) - manifest = create_manifest(data, checkpoint_file) + manifest = create_manifest(data, routes, checkpoint_file) with open(output_file, "w", encoding="utf-8") as fd: json.dump(manifest, fd, ensure_ascii=False, indent=4) From 40997987f6f1ae5079a1e24f6e573a68a5b35884 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 20 Nov 2021 16:43:55 -0500 Subject: [PATCH 029/152] add order guessing back --- server/bus/manifest.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/server/bus/manifest.py b/server/bus/manifest.py index ab71abbba..9a98f88e6 100644 --- a/server/bus/manifest.py +++ b/server/bus/manifest.py @@ -11,9 +11,9 @@ def load_checkpoints(checkpoint_file): output: dict(checkpoint_id -> station_name) or {} """ if checkpoint_file: - df = pd.read_csv(checkpoint_file, index_col="checkpoint_id") - df.index = df.index.str.lower() - return df.to_dict()["checkpoint_name"] + chks = pd.read_csv(checkpoint_file, index_col="checkpoint_id", squeeze=True) + chks.index = chks.index.str.lower() + return chks.str.replace("(Bus)", "", regex=False).str.strip() else: return {} @@ -28,7 +28,7 @@ def emit_stop_obj(entry, branches = False): "stop_name": entry.stop_name.iloc[0], "branches": entry.route_id.unique().tolist() if branches else None, "station": tpt_id, - "order": None, # TODO: determine the order guess + "order": int(entry.order_guess.iloc[0]), "stops": { "1": entry.loc[entry.direction_id == 1].full_stop_id.tolist(), "0": entry.loc[entry.direction_id == 0].full_stop_id.tolist(), @@ -38,20 +38,26 @@ def emit_stop_obj(entry, branches = False): def create_manifest(df, routes, checkpoint_file): output_route_name = '/'.join(routes) + + df['time_point_id'] = df['time_point_id'].str.lower() + df['time_point_order'] = df['time_point_order'].fillna(0) tpts = df[['route_id', 'stop_id', 'time_point_id', 'direction_id']].value_counts(dropna=False).rename("counts").reset_index() # use checkpoint file to map time_point_id to stop_name station_names = load_checkpoints(checkpoint_file) - tpts['stop_name'] = tpts['time_point_id'].str.lower().map(station_names) + tpts['stop_name'] = tpts['time_point_id'].map(station_names) # Create full stop id e.g. '66-0-64000' tpts['full_stop_id'] = tpts[['route_id', 'direction_id', 'stop_id']].astype(str).agg('-'.join, axis=1) + # Must be arranged by inbound direction. Use most common (mode) as guess + orders = df.loc[df.direction_id == 1].groupby("time_point_id", dropna=False)['time_point_order'].agg(pd.Series.mode) + tpts['order_guess'] = tpts['time_point_id'].map(orders).fillna(0).astype(int) + stop_objs = tpts.groupby('time_point_id', dropna=False).apply( lambda x: emit_stop_obj(x, len(routes) > 1) ).dropna() - # TODO: include order manifest = { output_route_name: { @@ -60,7 +66,7 @@ def create_manifest(df, routes, checkpoint_file): "0": "outbound", "1": "inbound" }, - "stations": stop_objs.values.tolist() # TODO: sorted + "stations": sorted(stop_objs.values, key=lambda x: x['order']) } } From 21a41eff0ecdc540f1d1fd547f631af9fde0278f Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 20 Nov 2021 19:39:50 -0500 Subject: [PATCH 030/152] further manifest refinement --- server/bus/manifest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/bus/manifest.py b/server/bus/manifest.py index 9a98f88e6..fbe14c0e6 100644 --- a/server/bus/manifest.py +++ b/server/bus/manifest.py @@ -20,18 +20,18 @@ def load_checkpoints(checkpoint_file): def emit_stop_obj(entry, branches = False): tpt_id = entry.name - if entry['counts'].sum() < 20: + if entry['counts'].sum() < 100: # this timepoint is very infrequently used return None return { "stop_name": entry.stop_name.iloc[0], - "branches": entry.route_id.unique().tolist() if branches else None, + "branches": sorted(entry.route_id.unique().tolist()) if branches else None, "station": tpt_id, "order": int(entry.order_guess.iloc[0]), "stops": { - "1": entry.loc[entry.direction_id == 1].full_stop_id.tolist(), - "0": entry.loc[entry.direction_id == 0].full_stop_id.tolist(), + "1": sorted(entry.loc[entry.direction_id == 1].full_stop_id.tolist()), + "0": sorted(entry.loc[entry.direction_id == 0].full_stop_id.tolist()), } } From 8e6c2792e57e4d253c0361b2e4d0cfdc7ef08394 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 20 Nov 2021 19:42:03 -0500 Subject: [PATCH 031/152] =?UTF-8?q?Route=2066=20=E2=99=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bus_constants/66.json | 140 ++++++++++++++++++++++++++++++++++++++ src/stations.js | 2 + 2 files changed, 142 insertions(+) create mode 100644 src/bus_constants/66.json diff --git a/src/bus_constants/66.json b/src/bus_constants/66.json new file mode 100644 index 000000000..386edb4be --- /dev/null +++ b/src/bus_constants/66.json @@ -0,0 +1,140 @@ +{ + "66": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Harvard", + "branches": null, + "station": "harsq", + "order": 1, + "stops": { + "1": [ + "66-1-22549" + ], + "0": [ + "66-0-22549" + ] + } + }, + { + "stop_name": "North Harvard & Western", + "branches": null, + "station": "nhvws", + "order": 2, + "stops": { + "1": [ + "66-1-2553" + ], + "0": [ + "66-0-2561" + ] + } + }, + { + "stop_name": "Union Square", + "branches": null, + "station": "union", + "order": 3, + "stops": { + "1": [ + "66-1-926", + "66-1-925" + ], + "0": [ + "66-0-1111", + "66-0-966" + ] + } + }, + { + "stop_name": "Harvard Avenue & Commonwealth Avenue", + "branches": null, + "station": "hvdcm", + "order": 4, + "stops": { + "1": [ + "66-1-1302" + ], + "0": [ + "66-0-1378" + ] + } + }, + { + "stop_name": "Coolidge Corner", + "branches": null, + "station": "cool", + "order": 5, + "stops": { + "1": [ + "66-1-1308" + ], + "0": [ + "66-0-1372" + ] + } + }, + { + "stop_name": "Brookline Village", + "branches": null, + "station": "brkvl", + "order": 6, + "stops": { + "1": [ + "66-1-1555" + ], + "0": [ + "66-0-1526" + ] + } + }, + { + "stop_name": "Brigham Circle", + "branches": null, + "station": "brghm", + "order": 7, + "stops": { + "1": [ + "66-1-1317" + ], + "0": [ + "66-0-1362" + ] + } + }, + { + "stop_name": "Roxbury Crossing Station", + "branches": null, + "station": "roxbs", + "order": 8, + "stops": { + "1": [ + "66-1-1323" + ], + "0": [ + "66-0-1357", + "66-0-21148" + ] + } + }, + { + "stop_name": "Nubian Station", + "branches": null, + "station": "nubn", + "order": 9, + "stops": { + "1": [ + "66-1-64000" + ], + "0": [ + "66-0-64000" + ] + } + } + ] + } +} diff --git a/src/stations.js b/src/stations.js index 4db4e7b25..6131cdd20 100644 --- a/src/stations.js +++ b/src/stations.js @@ -2,10 +2,12 @@ import { stations as rt_stations } from './constants'; import bus_1 from './bus_constants/1.json'; import bus_28 from './bus_constants/28.json'; +import bus_66 from './bus_constants/66.json'; const stations = {...rt_stations, ...bus_1, ...bus_28, + ...bus_66 }; const all_lines = () => { From 6452df5969b1bebf7ee3c709852f276dd0b57d24 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 06:50:08 -0500 Subject: [PATCH 032/152] no gzip mtime plz --- server/bus/bus2train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 70ceb5196..3ed5b4067 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -87,7 +87,8 @@ def _write_file(events, outdir, nozip=False): f"Day={service_date.day}", "events.csv.gz") fname.parent.mkdir(parents=True, exist_ok=True) - events.to_csv(fname, index=False, compression='gzip' if not nozip else None) + # set mtime to 0 in gzip header for determinism (so we can re-gen old routes, and rsync to s3 will ignore) + events.to_csv(fname, index=False, compression={'method': 'gzip', 'mtime': 0} if not nozip else None) def to_disk(df, root, nozip=False): From c79f0534b5eb6006073067d02aede955c90f5632 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 06:50:35 -0500 Subject: [PATCH 033/152] some util scripts --- server/bus/gen_bus_data.sh | 5 +++++ server/bus/gen_manifests.sh | 10 ++++++++++ 2 files changed, 15 insertions(+) create mode 100755 server/bus/gen_bus_data.sh create mode 100755 server/bus/gen_manifests.sh diff --git a/server/bus/gen_bus_data.sh b/server/bus/gen_bus_data.sh new file mode 100755 index 000000000..957240fac --- /dev/null +++ b/server/bus/gen_bus_data.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +for f in $(find data/input/MBTA_Bus_Arrival_Departure_Times_2021/ -name *.csv); do + pipenv run python bus2train.py $f data/actual -r 57 # 1 28 66 114 116 117 +done diff --git a/server/bus/gen_manifests.sh b/server/bus/gen_manifests.sh new file mode 100755 index 000000000..64bc90e9a --- /dev/null +++ b/server/bus/gen_manifests.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +route="57" + +mkdir data/output/manifests/$route + +for f in $(find data/input/MBTA_Bus_Arrival_Departure_Times_2021/ -name *.csv); do + month=$(echo $f | rev | cut -d- -f1 | rev | cut -d. -f1) + pipenv run python manifest.py $f "data/output/manifests/$route/$route_$month.json" --checkpoints "data/MBTA_GTFS/checkpoints.txt" -r $route +done \ No newline at end of file From 34fb045153602c4aa248af3232ac0347d7f0d799 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 06:53:13 -0500 Subject: [PATCH 034/152] add routes 57, 114/116/117 --- src/bus_constants/114-116-117.json | 203 +++++++++++++++++++++++++++++ src/bus_constants/57.json | 126 ++++++++++++++++++ src/stations.js | 6 +- 3 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 src/bus_constants/114-116-117.json create mode 100644 src/bus_constants/57.json diff --git a/src/bus_constants/114-116-117.json b/src/bus_constants/114-116-117.json new file mode 100644 index 000000000..e68e2cc07 --- /dev/null +++ b/src/bus_constants/114-116-117.json @@ -0,0 +1,203 @@ +{ + "114/116/117": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Wonderland Station West", + "branches": [ + "116", + "117" + ], + "station": "wondw", + "order": 1, + "stops": { + "1": [ + "116-1-15795", + "117-1-15795" + ], + "0": [ + "116-0-15795", + "117-0-15795" + ] + } + }, + { + "stop_name": "[117] Bell Circle", + "branches": [ + "117" + ], + "station": "bell", + "order": 2, + "stops": { + "1": [ + "117-1-4733" + ], + "0": [ + "117-0-4717" + ] + } + }, + { + "stop_name": "[117] Revere City Hall", + "_comment": "only the 117 is logged here", + "branches": [ + "117" + ], + "station": "rcith", + "order": 3, + "stops": { + "1": [ + "117-1-5714" + ], + "0": [ + "117-0-5761" + ] + } + }, + { + "stop_name": "[116] Broadway & Central (Revere)", + "_comment": "only the 116 is logged here, plus first and last 117 trips each day. They can be honorary 116s", + "branches": [ + "116" + ], + "station": "bwcen", + "order": 10, + "stops": { + "1": [ + "116-1-5713", + "117-1-5713" + ], + "0": [ + "116-0-5763", + "117-0-5790" + ] + } + }, + { + "stop_name": "Broadway & Webster", + "branches": [ + "116", + "117" + ], + "station": "webst", + "order": 11, + "stops": { + "1": [ + "116-1-5720", + "117-1-5720" + ], + "0": [ + "116-0-5755", + "117-0-5755" + ] + } + }, + { + "stop_name": "Market Basket", + "branches": [ + "114" + ], + "station": "mysml", + "order": 21, + "stops": { + "1": [ + "114-1-5045" + ], + "0": [ + "114-0-5045" + ] + } + }, + { + "stop_name": "Bellingham Square (Chelsea)", + "branches": [ + "114", + "116", + "117" + ], + "station": "belsq", + "order": 100, + "stops": { + "1": [ + "114-1-5605", + "116-1-5605", + "116-1-56170", + "117-1-5605" + ], + "0": [ + "114-0-5615", + "116-0-5615", + "116-0-56170", + "117-0-5615" + ] + } + }, + { + "stop_name": "Central Square (East Boston)", + "branches": [ + "114", + "116", + "117" + ], + "station": "cntle", + "order": 101, + "stops": { + "1": [ + "114-1-5736", + "116-1-5736", + "117-1-5736" + ], + "0": [ + "114-0-5743", + "116-0-5743", + "117-0-5743" + ] + } + }, + { + "stop_name": "Maverick Square (East Boston)", + "branches": [ + "114", + "116", + "117" + ], + "station": "mavck", + "order": 102, + "stops": { + "1": [ + "114-1-5740", + "116-1-5740", + "117-1-5740" + ], + "0": [ + "114-0-5740", + "116-0-5740", + "117-0-5740" + ] + } + }, + { + "stop_name": "Haymarket", + "branches": [ + "117" + ], + "station": "hayms", + "order": 200, + "stops": { + "1": [ + "117-1-8309", + "117-1-8310" + ], + "0": [ + "117-0-8309", + "117-0-8310" + ] + } + } + ] + } +} diff --git a/src/bus_constants/57.json b/src/bus_constants/57.json new file mode 100644 index 000000000..98a509ac4 --- /dev/null +++ b/src/bus_constants/57.json @@ -0,0 +1,126 @@ +{ + "57": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Watertown Yard", + "branches": null, + "station": "wtryd", + "order": 1, + "stops": { + "1": [ + "57-1-900" + ], + "0": [ + "57-0-900" + ] + } + }, + { + "stop_name": "Newton Corner", + "branches": null, + "station": "nwcor", + "order": 2, + "stops": { + "1": [ + "57-1-903" + ], + "0": [ + "57-0-987" + ] + } + }, + { + "stop_name": "Oak Square", + "branches": null, + "station": "oaksq", + "order": 3, + "stops": { + "1": [ + "57-1-912", + "57-1-913" + ], + "0": [ + "57-0-9780", + "57-0-979" + ] + } + }, + { + "stop_name": "Brighton Center", + "branches": null, + "station": "brctr", + "order": 4, + "stops": { + "1": [ + "57-1-918" + ], + "0": [ + "57-0-973" + ] + } + }, + { + "stop_name": "Union Square", + "branches": null, + "station": "union", + "order": 5, + "stops": { + "1": [ + "57-1-925", + "57-1-926" + ], + "0": [ + "57-0-966" + ] + } + }, + { + "stop_name": "Brighton & Commonwealth Avenue", + "branches": null, + "station": "brcom", + "order": 6, + "stops": { + "1": [ + "57-1-931" + ], + "0": [ + "57-0-959" + ] + } + }, + { + "stop_name": "Commonwealth Avenue @ BU Bridge", + "branches": null, + "station": "bubrg", + "order": 7, + "stops": { + "1": [ + "57-1-937" + ], + "0": [ + "57-0-954" + ] + } + }, + { + "stop_name": "Kenmore (Busway)", + "branches": null, + "station": "kenbs", + "order": 8, + "stops": { + "1": [ + "57-1-899" + ], + "0": [ + "57-0-899" + ] + } + } + ] + } +} diff --git a/src/stations.js b/src/stations.js index 6131cdd20..bfa3b45a5 100644 --- a/src/stations.js +++ b/src/stations.js @@ -2,12 +2,16 @@ import { stations as rt_stations } from './constants'; import bus_1 from './bus_constants/1.json'; import bus_28 from './bus_constants/28.json'; +import bus_57 from './bus_constants/57.json'; import bus_66 from './bus_constants/66.json'; +import bus_114_116_117 from './bus_constants/114-116-117.json' const stations = {...rt_stations, ...bus_1, ...bus_28, - ...bus_66 + ...bus_57, + ...bus_66, + ...bus_114_116_117 }; const all_lines = () => { From 76d65afc22986a1b9ff2643621bcd8c9ba067c67 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 07:01:51 -0500 Subject: [PATCH 035/152] multiple modes fix --- server/bus/manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/bus/manifest.py b/server/bus/manifest.py index fbe14c0e6..22cfd0233 100644 --- a/server/bus/manifest.py +++ b/server/bus/manifest.py @@ -51,8 +51,8 @@ def create_manifest(df, routes, checkpoint_file): # Create full stop id e.g. '66-0-64000' tpts['full_stop_id'] = tpts[['route_id', 'direction_id', 'stop_id']].astype(str).agg('-'.join, axis=1) - # Must be arranged by inbound direction. Use most common (mode) as guess - orders = df.loc[df.direction_id == 1].groupby("time_point_id", dropna=False)['time_point_order'].agg(pd.Series.mode) + # Must be arranged by inbound direction. Use most common (mode) as guess (w/ max in case 2 modes) + orders = df.loc[df.direction_id == 1].groupby("time_point_id", dropna=False)['time_point_order'].agg(lambda x: x.mode().max()) tpts['order_guess'] = tpts['time_point_id'].map(orders).fillna(0).astype(int) stop_objs = tpts.groupby('time_point_id', dropna=False).apply( From 515d0e3d679af5f3ff4445b9ae58b9e54e623c35 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 07:51:06 -0500 Subject: [PATCH 036/152] add route 23 --- src/bus_constants/23.json | 152 ++++++++++++++++++++++++++++++++++++++ src/stations.js | 2 + 2 files changed, 154 insertions(+) create mode 100644 src/bus_constants/23.json diff --git a/src/bus_constants/23.json b/src/bus_constants/23.json new file mode 100644 index 000000000..9b4554106 --- /dev/null +++ b/src/bus_constants/23.json @@ -0,0 +1,152 @@ +{ + "23": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Ashmont Terminal", + "branches": null, + "station": "ashmt", + "order": 1, + "stops": { + "1": [ + "23-1-334" + ], + "0": [ + "23-0-334" + ] + } + }, + { + "stop_name": "Codman Square", + "branches": null, + "station": "codmn", + "order": 2, + "stops": { + "1": [ + "23-1-371" + ], + "0": [ + "23-0-426" + ] + } + }, + { + "stop_name": "Four Corners (Dorchester)", + "branches": null, + "station": "fourc", + "order": 3, + "stops": { + "1": [ + "23-1-463" + ], + "0": [ + "23-0-478" + ] + } + }, + { + "stop_name": "Washington Street & Columbia Road", + "branches": null, + "station": "wacol", + "order": 4, + "stops": { + "1": [ + "23-1-473" + ], + "0": [ + "23-0-468" + ] + } + }, + { + "stop_name": "Grove Hall", + "branches": null, + "station": "ghall", + "order": 5, + "stops": { + "1": [ + "23-1-386" + ], + "0": [ + "23-0-412" + ] + } + }, + { + "stop_name": "Warren Street @ Boston Latin Academy", + "branches": null, + "station": "latac", + "order": 6, + "stops": { + "1": [ + "23-1-390" + ], + "0": [ + "23-0-13321", + "23-0-407" + ] + } + }, + { + "stop_name": "Warren Street @ Walnut Avenue", + "branches": null, + "station": "warwl", + "order": 7, + "stops": { + "1": [ + "23-1-396" + ], + "0": [ + "23-0-40001" + ] + } + }, + { + "stop_name": "Nubian Station", + "branches": null, + "station": "nubn", + "order": 8, + "stops": { + "1": [ + "23-1-64000" + ], + "0": [ + "23-0-64000" + ] + } + }, + { + "stop_name": "Roxbury Crossing Station", + "branches": null, + "station": "roxbs", + "order": 9, + "stops": { + "1": [ + "23-1-21148" + ], + "0": [ + "23-0-11257" + ] + } + }, + { + "stop_name": "Ruggles", + "branches": null, + "station": "rugg", + "order": 10, + "stops": { + "1": [ + "23-1-17861" + ], + "0": [ + "23-0-17862" + ] + } + } + ] + } +} diff --git a/src/stations.js b/src/stations.js index bfa3b45a5..23da2c63a 100644 --- a/src/stations.js +++ b/src/stations.js @@ -1,6 +1,7 @@ import { stations as rt_stations } from './constants'; import bus_1 from './bus_constants/1.json'; +import bus_23 from './bus_constants/23.json'; import bus_28 from './bus_constants/28.json'; import bus_57 from './bus_constants/57.json'; import bus_66 from './bus_constants/66.json'; @@ -8,6 +9,7 @@ import bus_114_116_117 from './bus_constants/114-116-117.json' const stations = {...rt_stations, ...bus_1, + ...bus_23, ...bus_28, ...bus_57, ...bus_66, From 79dfa1c6aafa43c9eaee625af8dfc7ab2a497d22 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 08:54:07 -0500 Subject: [PATCH 037/152] add route 111 --- src/bus_constants/111.json | 155 +++++++++++++++++++++++++++++++++++++ src/stations.js | 2 + 2 files changed, 157 insertions(+) create mode 100644 src/bus_constants/111.json diff --git a/src/bus_constants/111.json b/src/bus_constants/111.json new file mode 100644 index 000000000..afdb141ff --- /dev/null +++ b/src/bus_constants/111.json @@ -0,0 +1,155 @@ +{ + "111": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Woodlawn", + "branches": null, + "station": "woodc", + "order": 1, + "stops": { + "1": [ + "111-1-5547" + ], + "0": [ + "111-0-5547" + ] + } + }, + { + "stop_name": "Garfield Avenue & Clyde", + "branches": null, + "station": "garfi", + "order": 2, + "stops": { + "1": [ + "111-1-5592" + ], + "0": [ + "111-0-5629" + ] + } + }, + { + "stop_name": "Sagamore & Washington", + "branches": null, + "station": "smore", + "order": 3, + "stops": { + "1": [ + "111-1-5595" + ], + "0": [ + "111-0-5626" + ] + } + }, + { + "stop_name": "Washington & Cary", + "branches": null, + "station": "wacry", + "order": 4, + "stops": { + "1": [ + "111-1-5601" + ], + "0": [ + "111-0-5601", + "111-0-5620" + ] + } + }, + { + "stop_name": "Bellingham Square (Chelsea)", + "branches": null, + "station": "belsq", + "order": 5, + "stops": { + "1": [ + "111-1-5605" + ], + "0": [ + "111-0-5615" + ] + } + }, + { + "stop_name": "[INB] Third & Chestnut | [OB] Broadway & Beacon", + "_comment": "The route splits with differing timepoints. Hopefully this is clear yet concise", + "branches": null, + "station": "thche|bwybc", + "order": 6, + "stops": { + "1": [ + "111-1-5607" + ], + "0": [ + "111-0-5611" + ] + } + }, + { + "stop_name": "Tobin Bridge", + "branches": null, + "station": "tobin", + "order": 7, + "stops": { + "1": [ + "111-1-12001" + ], + "0": [ + "111-0-12002" + ] + } + }, + { + "stop_name": "Rutherford Avenue & Ramp", + "branches": null, + "station": "ruthf", + "order": 8, + "stops": { + "1": [ + "111-1-12003" + ], + "0": [ + "111-0-12004" + ] + } + }, + { + "stop_name": "Causeway Street & Commercial Street", + "branches": null, + "station": "cacom", + "order": 9, + "stops": { + "1": [ + "111-1-2829" + ], + "0": [ + "111-0-2832" + ] + } + }, + { + "stop_name": "Haymarket", + "branches": null, + "station": "hayms", + "order": 10, + "stops": { + "1": [ + "111-1-8309", + "111-1-8310" + ], + "0": [ + "111-0-8309", + "111-0-8310" + ] + } + } + ] + } +} diff --git a/src/stations.js b/src/stations.js index 23da2c63a..9a42131a8 100644 --- a/src/stations.js +++ b/src/stations.js @@ -5,6 +5,7 @@ import bus_23 from './bus_constants/23.json'; import bus_28 from './bus_constants/28.json'; import bus_57 from './bus_constants/57.json'; import bus_66 from './bus_constants/66.json'; +import bus_111 from './bus_constants/111.json'; import bus_114_116_117 from './bus_constants/114-116-117.json' const stations = {...rt_stations, @@ -13,6 +14,7 @@ const stations = {...rt_stations, ...bus_28, ...bus_57, ...bus_66, + ...bus_111, ...bus_114_116_117 }; From 6150e843197228ed25c450c8a8858d518293f981 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 08:57:27 -0500 Subject: [PATCH 038/152] add some util scripts for bus gen --- server/bus/gen_bus_data.sh | 2 +- server/bus/gen_manifests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/bus/gen_bus_data.sh b/server/bus/gen_bus_data.sh index 957240fac..caac11c81 100755 --- a/server/bus/gen_bus_data.sh +++ b/server/bus/gen_bus_data.sh @@ -1,5 +1,5 @@ #!/bin/bash for f in $(find data/input/MBTA_Bus_Arrival_Departure_Times_2021/ -name *.csv); do - pipenv run python bus2train.py $f data/actual -r 57 # 1 28 66 114 116 117 + pipenv run python bus2train.py $f data/actual -r 111 # 23 # 57 # 1 28 66 114 116 117 done diff --git a/server/bus/gen_manifests.sh b/server/bus/gen_manifests.sh index 64bc90e9a..53524bd9e 100755 --- a/server/bus/gen_manifests.sh +++ b/server/bus/gen_manifests.sh @@ -1,6 +1,6 @@ #!/bin/bash -route="57" +route="111" mkdir data/output/manifests/$route From 1e7b0403013bd1d53bf0800e58ed8bcdba95bb41 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 09:37:09 -0500 Subject: [PATCH 039/152] don't show stop moves in alerts --- src/AlertFilter.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/AlertFilter.js b/src/AlertFilter.js index 9d104db91..f3fc1e0d4 100644 --- a/src/AlertFilter.js +++ b/src/AlertFilter.js @@ -6,8 +6,18 @@ const known = [ /notice/, ]; +const anti = [ + / stop .* move /i // "The stop X will permanently move to Y" +] + const findMatch = (alert) => { const text = alert.text; + for (const exp of anti) { + const match = text.match(exp); + if (match != null) { + return null; + } + } for (const exp of known) { const match = text.match(exp); if (match != null) { From 14512746a8eaaf9dd9a1e4de08c0a5f07d314201 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 15:51:26 -0500 Subject: [PATCH 040/152] fix benchmark display bugs --- src/App.js | 4 ++-- src/line.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/App.js b/src/App.js index 46a588c72..6d82667d1 100644 --- a/src/App.js +++ b/src/App.js @@ -452,7 +452,7 @@ class App extends React.Component { seriesName={"travel time"} xField={'dep_dt'} yField={'travel_time_sec'} - benchmarkField={'benchmark_travel_time_sec'} + benchmarkField={(this.state.traveltimes.some(e => e.benchmark_travel_time_sec > 0)) ? 'benchmark_travel_time_sec' : null} location={this.locationDescription(true)} isLoading={this.getIsLoadingDataset('traveltimes')} date={this.state.configuration.date_start} @@ -463,7 +463,7 @@ class App extends React.Component { seriesName={"headway"} xField={"current_dep_dt"} yField={'headway_time_sec'} - benchmarkField={'benchmark_headway_time_sec'} + benchmarkField={(this.state.traveltimes.some(e => e.benchmark_headway_time_sec > 0)) ? 'benchmark_headway_time_sec' : null} location={this.locationDescription(false)} isLoading={this.getIsLoadingDataset('headways')} date={this.state.configuration.date_start} diff --git a/src/line.js b/src/line.js index 132f65900..8bb796e9d 100644 --- a/src/line.js +++ b/src/line.js @@ -135,7 +135,7 @@ class SingleDayLine extends React.Component { // TODO: tooltip is under title words callbacks: { afterBody: (tooltipItems) => { - return departure_from_normal_string(tooltipItems[0].value, tooltipItems[1].value); + return departure_from_normal_string(tooltipItems[0].value, tooltipItems[1]?.value); } } }, From 05a1f1e937420a8c069c0eac560c390358da51a0 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 21 Nov 2021 19:20:59 -0500 Subject: [PATCH 041/152] add shuttle alerts --- src/AlertFilter.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/AlertFilter.js b/src/AlertFilter.js index f3fc1e0d4..64b7c28e1 100644 --- a/src/AlertFilter.js +++ b/src/AlertFilter.js @@ -4,6 +4,7 @@ const known = [ /[A-Za-z]+ speeds/, // "Reduced speeds" /delay/, /notice/, + /shuttle/, // might want to worry about this one... ]; const anti = [ From b56eb0cf35513ed540df33224b834dde3b1642a2 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Tue, 23 Nov 2021 14:47:04 -0500 Subject: [PATCH 042/152] refactor renderCharts into chartset functions --- src/App.js | 97 +++++++++--------------------------------------- src/ChartSet.js | 98 +++++++++++++++++++++++++++++++++++++++++++++++++ src/Title.js | 16 ++++---- src/line.js | 4 +- 4 files changed, 125 insertions(+), 90 deletions(-) create mode 100644 src/ChartSet.js diff --git a/src/App.js b/src/App.js index 46a588c72..1aff9e322 100644 --- a/src/App.js +++ b/src/App.js @@ -1,9 +1,9 @@ import React from 'react'; import ReactGA from 'react-ga'; -import { SingleDayLine, AggregateLine } from './line'; +import { SingleDaySet, AggregateSet } from './ChartSet'; import StationConfiguration from './StationConfiguration'; import { withRouter } from 'react-router-dom'; -import { lookup_station_by_id, station_direction, get_stop_ids_for_stations } from './stations'; +import { lookup_station_by_id, get_stop_ids_for_stations } from './stations'; import { recognize } from './AlertFilter'; import AlertBar from './AlertBar'; import ProgressBar from './ui/ProgressBar'; @@ -333,20 +333,6 @@ class App extends React.Component { } } - locationDescription(bothStops) { - const { from, to, line } = this.state.configuration; - - if (from && to) { - return { - bothStops: bothStops, - to: to.stop_name, - from: from.stop_name, - direction: station_direction(from, to, line), - line: line, - }; - } - return {}; - } chartTimeframe() { // Set alert-bar interval to be 5:30am today to 1am tomorrow. @@ -414,72 +400,23 @@ class App extends React.Component { } renderCharts() { + const propsToPass = { + traveltimes: this.state.traveltimes, + headways: this.state.headways, + dwells: this.state.dwells, + isLoadingTraveltimes: this.getIsLoadingDataset('traveltimes'), + isLoadingHeadways: this.getIsLoadingDataset('headways'), + isLoadingDwells: this.getIsLoadingDataset('dwells'), + startDate: this.state.configuration.date_start, + endDate: this.state.configuration.date_end, + from: this.state.configuration.from, + to: this.state.configuration.to, + line: this.state.configuration.line + } if (this.isAggregation()) { - return
- - - -
+ return } else { - return
- - - -
+ return } } diff --git a/src/ChartSet.js b/src/ChartSet.js new file mode 100644 index 000000000..5c2b8c3a1 --- /dev/null +++ b/src/ChartSet.js @@ -0,0 +1,98 @@ +import React from 'react'; +import { SingleDayLine, AggregateLine } from './line'; +import { station_direction } from './stations'; + + +function getLocationDescription(from, to, line) { + if (from && to) { + return { + to: to.stop_name, + from: from.stop_name, + direction: station_direction(from, to, line), + line: line, + }; + } + return {}; +} + +function AggregateSet(props) { + const locationDescription = getLocationDescription(props.from, props.to, props.line) + return( +
+ + + +
+ ) +} + +function SingleDaySet(props) { + const locationDescription = getLocationDescription(props.from, props.to, props.line) + return
+ e.benchmark_travel_time_sec > 0)) ? 'benchmark_travel_time_sec' : null} + location={locationDescription} + titleBothStops={true} + isLoading={props.isLoadingTraveltimes} + date={props.startDate} + /> + e.benchmark_headway_time_sec > 0)) ? 'benchmark_headway_time_sec' : null} + location={locationDescription} + titleBothStops={false} + isLoading={props.isLoadingHeadways} + date={props.startDate} + /> + +
+} + +export { SingleDaySet, AggregateSet } \ No newline at end of file diff --git a/src/Title.js b/src/Title.js index e7bd9e071..c31785646 100644 --- a/src/Title.js +++ b/src/Title.js @@ -3,14 +3,14 @@ import { colorsForLine } from './constants.js'; const getLineColor = (lineName) => colorsForLine[lineName] || 'black'; const titleColor = 'gray'; -const parse_location_description = (location) => { +const parse_location_description = (location, bothStops) => { let result = []; const lineColor = getLineColor(location['line']); result.push([location['from'], lineColor]); - if (location['bothStops']) { + if (bothStops) { result.push([' to ', titleColor]); result.push([location['to'], lineColor]); } else { @@ -19,7 +19,7 @@ const parse_location_description = (location) => { return result; }; -const drawTitle = (title, location, chart) => { +const drawTitle = (title, location, bothStops, chart) => { let ctx = chart.chart.ctx; ctx.save(); @@ -31,9 +31,9 @@ const drawTitle = (title, location, chart) => { let position; const titleWidth = ctx.measureText(title).width; - const locationWidth = parse_location_description(location) - .map(x => ctx.measureText(x[0]).width) - .reduce((a,b) => a + b, 0); + const locationWidth = parse_location_description(location, bothStops) + .map(x => ctx.measureText(x[0]).width) + .reduce((a,b) => a + b, 0); if ((leftMargin + titleWidth + minGap + locationWidth) > chart.chart.width) { // small screen: centered title stacks vertically @@ -44,7 +44,7 @@ const drawTitle = (title, location, chart) => { ctx.textAlign = 'left'; position = chart.chart.width/2 - locationWidth/2; - for (const [word, color] of parse_location_description(location)) { + for (const [word, color] of parse_location_description(location, bothStops)) { ctx.fillStyle = color; ctx.fillText(word, position, vpos_row2); position += ctx.measureText(word).width; @@ -60,7 +60,7 @@ const drawTitle = (title, location, chart) => { // location components are aligned right ctx.textAlign = 'right'; position = chart.chart.width; - for (const [word, color] of parse_location_description(location).reverse()) { + for (const [word, color] of parse_location_description(location, bothStops).reverse()) { ctx.fillStyle = color; ctx.fillText(word, position, vpos_row2); position -= ctx.measureText(word).width; diff --git a/src/line.js b/src/line.js index 132f65900..e9def690c 100644 --- a/src/line.js +++ b/src/line.js @@ -173,7 +173,7 @@ class SingleDayLine extends React.Component { } }} plugins={[{ - afterDraw: (chart) => drawTitle(this.props.title, this.props.location, chart) + afterDraw: (chart) => drawTitle(this.props.title, this.props.location, this.props.titleBothStops, chart) }]} /> @@ -258,7 +258,7 @@ class AggregateLine extends React.Component { } }} plugins={[{ - afterDraw: (chart) => drawTitle(this.props.title, this.props.location, chart) + afterDraw: (chart) => drawTitle(this.props.title, this.props.location, this.props.titleBothStops, chart) }]} /> From da44c8cc7d7e14295c8d44e5e863b8a42a5bdf8c Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Tue, 23 Nov 2021 14:53:06 -0500 Subject: [PATCH 043/152] fix benchmark tooltip bug --- src/line.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/line.js b/src/line.js index e9def690c..c574fe1a2 100644 --- a/src/line.js +++ b/src/line.js @@ -135,7 +135,7 @@ class SingleDayLine extends React.Component { // TODO: tooltip is under title words callbacks: { afterBody: (tooltipItems) => { - return departure_from_normal_string(tooltipItems[0].value, tooltipItems[1].value); + return departure_from_normal_string(tooltipItems[0].value, tooltipItems[1]?.value); } } }, From 73b04fe05adddc68caccfb62a43bfa1a682cc885 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 24 Nov 2021 07:20:45 -0500 Subject: [PATCH 044/152] new Aggregation chart with Radio --- src/AggregateDaily.js | 77 +++++++++++++++++++++++++++++++++++++++++++ src/ChartSet.js | 30 +++++++++++------ src/line.js | 48 +++++++++++++++++++++++---- 3 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 src/AggregateDaily.js diff --git a/src/AggregateDaily.js b/src/AggregateDaily.js new file mode 100644 index 000000000..4dcad5932 --- /dev/null +++ b/src/AggregateDaily.js @@ -0,0 +1,77 @@ +import React from 'react'; +import { useState } from 'react'; +import classNames from 'classnames'; +import { AggregateOverTime } from './line'; + +const RadioForm = props => { + const { options, onChange, defaultValue, className } = props; + const [ checked, setChecked ] = useState(defaultValue); + + const handleChange = (evt) => { + const value = evt.target.value; + setChecked(value); + onChange(evt.target.value); + } + + return ( +
+ {options.map((opt, index) => + + )} +
+ ) +} + +class AggregateDaily extends React.Component { + /* + Props: + data + location + isLoading + + */ + constructor(props) { + super(props); + this.onChangeValue = this.onChangeValue.bind(this); + + this.state = {title: 'Aggregate by day (peak)'} //.peak + } + + options = [{value: "peak", label: "Peak"}, + {value: "offpeak", label: "Off-Peak"}] + + onChangeValue(value) { + // TODO: set data/field to be peak or offpeak + if (value === "peak") { + this.setState({title: "peaking"}) + } else { + this.setState({title: "not-peaking"}) + } + } + + render() { + return( + + + + ) + } +} + +export { AggregateDaily } \ No newline at end of file diff --git a/src/ChartSet.js b/src/ChartSet.js index 5c2b8c3a1..7ff98c138 100644 --- a/src/ChartSet.js +++ b/src/ChartSet.js @@ -1,5 +1,6 @@ import React from 'react'; -import { SingleDayLine, AggregateLine } from './line'; +import { AggregateDaily } from './AggregateDaily'; +import { SingleDayLine, AggregateOverTime } from './line'; import { station_direction } from './stations'; @@ -15,11 +16,11 @@ function getLocationDescription(from, to, line) { return {}; } -function AggregateSet(props) { - const locationDescription = getLocationDescription(props.from, props.to, props.line) +const AggregateSet = (props) => { + const locationDescription = getLocationDescription(props.from, props.to, props.line); return(
- - - +
) } -function SingleDaySet(props) { - const locationDescription = getLocationDescription(props.from, props.to, props.line) +const SingleDaySet = (props) => { + const locationDescription = getLocationDescription(props.from, props.to, props.line); + const anyTravelBenchmarks = props.traveltimes.some(e => e.benchmark_travel_time_sec > 0); + const anyHeadwayBenchmarks = props.headways.some(e => e.benchmark_headway_time_sec > 0); return
e.benchmark_travel_time_sec > 0)) ? 'benchmark_travel_time_sec' : null} + benchmarkField={anyTravelBenchmarks ? 'benchmark_travel_time_sec' : null} location={locationDescription} titleBothStops={true} isLoading={props.isLoadingTraveltimes} @@ -74,7 +84,7 @@ function SingleDaySet(props) { seriesName={"headway"} xField={"current_dep_dt"} yField={'headway_time_sec'} - benchmarkField={(props.headways.some(e => e.benchmark_headway_time_sec > 0)) ? 'benchmark_headway_time_sec' : null} + benchmarkField={anyHeadwayBenchmarks ? 'benchmark_headway_time_sec' : null} location={locationDescription} titleBothStops={false} isLoading={props.isLoadingHeadways} diff --git a/src/line.js b/src/line.js index c574fe1a2..3f2c91ac4 100644 --- a/src/line.js +++ b/src/line.js @@ -189,6 +189,11 @@ class AggregateLine extends React.Component { render() { /* Props: + xField + timeUnit + timeFormat + xMin + xMax title data seriesName @@ -198,7 +203,7 @@ class AggregateLine extends React.Component { endDate */ const { isLoading } = this.props; - let labels = this.props.data.map(item => item['service_date']); + let labels = this.props.data.map(item => item[this.props.xField]); return (
@@ -245,14 +250,14 @@ class AggregateLine extends React.Component { xAxes: [{ type: 'time', time: { - unit: 'day', + unit: this.props.timeUnit, unitStepSize: 1, - tooltipFormat: "ddd MMM D YYYY" + tooltipFormat: this.props.timeFormat }, ticks: { // force graph to show startDate to endDate, even if missing data - min: new Date(`${this.props.startDate}T00:00:00`), - max: new Date(`${this.props.endDate}T00:00:00`), + min: this.props.xMin, //new Date(`${this.props.startDate}T00:00:00`), + max: this.props.xMax, //new Date(`${this.props.endDate}T00:00:00`), } }] } @@ -262,10 +267,39 @@ class AggregateLine extends React.Component { }]} />
- +
+ + {this.props.children} +
); } } -export { SingleDayLine, AggregateLine }; +const AggregateOverTime = (props) => { + return( + + ) +} + +const AggregateDaily = (props) => { + return( + + ) +} + +export { SingleDayLine, AggregateOverTime, AggregateDaily }; From 370754329b4cdfe60b43de8e471f967c3406a837 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 24 Nov 2021 08:14:31 -0500 Subject: [PATCH 045/152] Adjust css for chart control --- src/AggregateDaily.js | 4 ++-- src/App.css | 27 +++++++++++++++++++++++---- src/StationConfiguration.js | 6 ++---- src/Title.js | 5 +++-- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/AggregateDaily.js b/src/AggregateDaily.js index 4dcad5932..31d798dc2 100644 --- a/src/AggregateDaily.js +++ b/src/AggregateDaily.js @@ -14,7 +14,7 @@ const RadioForm = props => { } return ( -
+
{options.map((opt, index) =>
- {this.props.benchmarkField && } +
+ {this.props.benchmarkField && } +
); - // TODO: hide legend when benchmarks are 0s } } @@ -256,8 +257,8 @@ class AggregateLine extends React.Component { }, ticks: { // force graph to show startDate to endDate, even if missing data - min: this.props.xMin, //new Date(`${this.props.startDate}T00:00:00`), - max: this.props.xMax, //new Date(`${this.props.endDate}T00:00:00`), + min: this.props.xMin, + max: this.props.xMax, } }] } From 7a1e5be9ee779cf85036febcb7ed4cccc9cb7cdf Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 24 Nov 2021 21:44:33 -0500 Subject: [PATCH 047/152] Add backend support for daily aggregation --- server/app.py | 9 +++++ server/chalicelib/aggregation.py | 57 +++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/server/app.py b/server/app.py index 76f3dbf29..428722ab9 100644 --- a/server/app.py +++ b/server/app.py @@ -110,6 +110,15 @@ def traveltime_aggregate_route(): response = aggregation.travel_times_over_time(sdate, edate, from_stop, to_stop) return json.dumps(response, indent=4, sort_keys=True, default=str) +@app.route("/aggregate/traveltimes2", cors=cors_config) +def traveltime_aggregate_route(): + sdate = parse_user_date(app.current_request.query_params["start_date"]) + edate = parse_user_date(app.current_request.query_params["end_date"]) + from_stop = app.current_request.query_params["from_stop"] + to_stop = app.current_request.query_params["to_stop"] + + response = aggregation.travel_times_all(sdate, edate, from_stop, to_stop) + return json.dumps(response, indent=4, sort_keys=True, default=str) @app.route("/aggregate/headways", cors=cors_config) def headways_aggregate_route(): diff --git a/server/chalicelib/aggregation.py b/server/chalicelib/aggregation.py index 5f43e6064..166999179 100644 --- a/server/chalicelib/aggregation.py +++ b/server/chalicelib/aggregation.py @@ -15,6 +15,7 @@ def train_peak_status(df): # Peak Hours: non-holiday weekdays 6:30-9am; 3:30-6:30pm is_peak_day = (~df['holiday']) & (df['weekday'] < 5) + df['is_peak_day'] = is_peak_day conditions = [is_peak_day & (df['dep_time'].between(datetime.time(6, 30), datetime.time(9, 0))), is_peak_day & (df['dep_time'].between(datetime.time(15, 30), datetime.time(18, 30)))] choices = ['am_peak', 'pm_peak'] @@ -40,10 +41,17 @@ def faster_describe(grouped): return stats.loc[stats['count'] > 4] -def travel_times_over_time(sdate, edate, from_stop, to_stop): +#################### +### TRAVEL TIMES ### +#################### +# `aggregate_traveltime_data` will fetch and clean the data +# There are `calc_travel_times_over_time` and `calc_travel_times_daily` will use the data to aggregate in various ways +# `travel_times_all` will return all calculated aggregates +# `travel_times_over_time` is legacy and returns just the over-time calculation +def aggregate_traveltime_data(sdate, edate, from_stop, to_stop): all_data = data_funcs.travel_times(sdate, [from_stop], [to_stop], edate) if not all_data: - return [] + return None # convert to pandas df = pd.DataFrame.from_records(all_data) @@ -56,6 +64,32 @@ def travel_times_over_time(sdate, edate, from_stop, to_stop): df['weekday'] = service_date.dt.dayofweek df = train_peak_status(df) + return df + + +def calc_travel_times_daily(df): + # convert time of day to a consistent datetime relative to epoch + timedeltas = pd.to_timedelta(df['dep_time'].astype(str)) + timedeltas.loc[timedeltas < SERVICE_HR_OFFSET] += datetime.timedelta(days=1) + df['dep_time_from_epoch'] = timedeltas + datetime.datetime(1970, 1, 1) + + workday = df.loc[df.is_peak_day] + weekend = df.loc[~df.is_peak_day] + + # resample: groupby on 'dep_time_from_epoch' in 30 minute chunks. + workday_stats = faster_describe(workday.resample('30T', on='dep_time_from_epoch')['travel_time_sec']).reset_index() + weekend_stats = faster_describe(weekend.resample('30T', on='dep_time_from_epoch')['travel_time_sec']).reset_index() + + workday_stats['dep_time_from_epoch'] = workday_stats['dep_time_from_epoch'].dt.strftime("%Y-%m-%dT%H:%M:%S") + weekend_stats['dep_time_from_epoch'] = weekend_stats['dep_time_from_epoch'].dt.strftime("%Y-%m-%dT%H:%M:%S") + + return { + 'workdays': workday_stats.to_dict('records'), + 'weekends': weekend_stats.to_dict('records') + } + + +def calc_travel_times_over_time(df): # get summary stats summary_stats = faster_describe(df.groupby('service_date')['travel_time_sec']) summary_stats['peak'] = 'all' @@ -67,12 +101,27 @@ def travel_times_over_time(sdate, edate, from_stop, to_stop): # combine summary stats summary_stats_final = summary_stats.append(summary_stats_peak) - # filter peak status results = summary_stats_final.loc[summary_stats_final['peak'] == 'all'] - # convert to dictionary return results.to_dict('records') +def travel_times_all(sdate, edate, from_stop, to_stop): + df = aggregate_traveltime_data(sdate, edate, from_stop, to_stop) + if df is None: + return {'overtime': [], 'daily': []} + daily = calc_travel_times_daily(df) + overtime = calc_travel_times_over_time(df) + + return { + 'overtime': overtime, + 'daily': daily + } + + +def travel_times_over_time(sdate, edate, from_stop, to_stop): + return travel_times_all(sdate, edate, from_stop, to_stop)['overtime'] + + def headways_over_time(sdate, edate, stop): all_data = data_funcs.headways(sdate, [stop], edate) if not all_data: From c1cc30cd201bd28d3658b7916e24a28387b44a66 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 24 Nov 2021 21:47:28 -0500 Subject: [PATCH 048/152] Hook chart frontend into new backend --- src/AggregateDaily.js | 31 ++++++++++++++++++++----------- src/App.js | 8 +++++++- src/ChartSet.js | 7 ++++--- src/line.js | 6 +++--- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/AggregateDaily.js b/src/AggregateDaily.js index 31d798dc2..80d4f8e08 100644 --- a/src/AggregateDaily.js +++ b/src/AggregateDaily.js @@ -1,7 +1,7 @@ import React from 'react'; import { useState } from 'react'; import classNames from 'classnames'; -import { AggregateOverTime } from './line'; +import { AggregateDailyLine } from './line'; const RadioForm = props => { const { options, onChange, defaultValue, className } = props; @@ -40,26 +40,35 @@ class AggregateDaily extends React.Component { super(props); this.onChangeValue = this.onChangeValue.bind(this); - this.state = {title: 'Aggregate by day (peak)'} //.peak + this.state = { + title: `${props.title} (Work days)`, + data: props.data.workdays || [] + } } - options = [{value: "peak", label: "Peak"}, - {value: "offpeak", label: "Off-Peak"}] + options = [{value: "weekday", label: "Work days"}, + {value: "offday", label: "Weekends/Holidays"}] onChangeValue(value) { - // TODO: set data/field to be peak or offpeak - if (value === "peak") { - this.setState({title: "peaking"}) + if (value === "weekday") { + this.setState({ + title: `${this.props.title} (Work days)`, + data: this.props.data.workdays || [] + }) } else { - this.setState({title: "not-peaking"}) + this.setState({ + title: `${this.props.title} (Weekends)`, + data: this.props.data.weekends || [] + }) } } render() { + // TODO: this isn't updating when new data loads return( - - + ) } } diff --git a/src/App.js b/src/App.js index 1aff9e322..f52525c38 100644 --- a/src/App.js +++ b/src/App.js @@ -201,7 +201,13 @@ class App extends React.Component { if (this.state.configuration.date_end) { options["start_date"] = this.state.configuration.date_start; options["end_date"] = this.state.configuration.date_end; - url = new URL(`${APP_DATA_BASE_PATH}/aggregate/${name}`, window.location.origin); + + var method = name; + if (name === "traveltimes") { + method = "traveltimes2" + } + + url = new URL(`${APP_DATA_BASE_PATH}/aggregate/${method}`, window.location.origin); } else { url = new URL(`${APP_DATA_BASE_PATH}/${name}/${this.state.configuration.date_start}`, window.location.origin); diff --git a/src/ChartSet.js b/src/ChartSet.js index 7ff98c138..52a0aa789 100644 --- a/src/ChartSet.js +++ b/src/ChartSet.js @@ -22,7 +22,7 @@ const AggregateSet = (props) => {
{ endDate={props.endDate} /> + />
) } diff --git a/src/line.js b/src/line.js index 7a37c9eba..4e76bfafd 100644 --- a/src/line.js +++ b/src/line.js @@ -290,11 +290,11 @@ const AggregateOverTime = (props) => { ) } -const AggregateDaily = (props) => { +const AggregateDailyLine = (props) => { return( { ) } -export { SingleDayLine, AggregateOverTime, AggregateDaily }; +export { SingleDayLine, AggregateOverTime, AggregateDailyLine }; From 0920b5ca33eb0a59060e5136977b6ca42b8966a6 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 24 Nov 2021 22:15:28 -0500 Subject: [PATCH 049/152] lint backend --- server/app.py | 4 +++- server/chalicelib/aggregation.py | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/server/app.py b/server/app.py index 428722ab9..98f2d9f6c 100644 --- a/server/app.py +++ b/server/app.py @@ -110,8 +110,9 @@ def traveltime_aggregate_route(): response = aggregation.travel_times_over_time(sdate, edate, from_stop, to_stop) return json.dumps(response, indent=4, sort_keys=True, default=str) + @app.route("/aggregate/traveltimes2", cors=cors_config) -def traveltime_aggregate_route(): +def traveltime_aggregate_route_2(): sdate = parse_user_date(app.current_request.query_params["start_date"]) edate = parse_user_date(app.current_request.query_params["end_date"]) from_stop = app.current_request.query_params["from_stop"] @@ -120,6 +121,7 @@ def traveltime_aggregate_route(): response = aggregation.travel_times_all(sdate, edate, from_stop, to_stop) return json.dumps(response, indent=4, sort_keys=True, default=str) + @app.route("/aggregate/headways", cors=cors_config) def headways_aggregate_route(): sdate = parse_user_date(app.current_request.query_params["start_date"]) diff --git a/server/chalicelib/aggregation.py b/server/chalicelib/aggregation.py index 166999179..50798c075 100644 --- a/server/chalicelib/aggregation.py +++ b/server/chalicelib/aggregation.py @@ -42,12 +42,13 @@ def faster_describe(grouped): #################### -### TRAVEL TIMES ### +# TRAVEL TIMES #################### # `aggregate_traveltime_data` will fetch and clean the data # There are `calc_travel_times_over_time` and `calc_travel_times_daily` will use the data to aggregate in various ways # `travel_times_all` will return all calculated aggregates # `travel_times_over_time` is legacy and returns just the over-time calculation + def aggregate_traveltime_data(sdate, edate, from_stop, to_stop): all_data = data_funcs.travel_times(sdate, [from_stop], [to_stop], edate) if not all_data: @@ -122,6 +123,9 @@ def travel_times_over_time(sdate, edate, from_stop, to_stop): return travel_times_all(sdate, edate, from_stop, to_stop)['overtime'] +#################### +# HEADWAYS +#################### def headways_over_time(sdate, edate, stop): all_data = data_funcs.headways(sdate, [stop], edate) if not all_data: @@ -155,6 +159,9 @@ def headways_over_time(sdate, edate, stop): return results.to_dict('records') +#################### +# DWELLS +#################### def dwells_over_time(sdate, edate, stop): all_data = data_funcs.dwells(sdate, [stop], edate) if not all_data: From 7670d8bd58b9390c5902626c71b31e9821317fa5 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 28 Nov 2021 00:49:50 -0500 Subject: [PATCH 050/152] fix data update issue --- src/AggregateDaily.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/AggregateDaily.js b/src/AggregateDaily.js index 80d4f8e08..15aa1c9f0 100644 --- a/src/AggregateDaily.js +++ b/src/AggregateDaily.js @@ -42,33 +42,32 @@ class AggregateDaily extends React.Component { this.state = { title: `${props.title} (Work days)`, - data: props.data.workdays || [] + selected: "workdays" } } - options = [{value: "weekday", label: "Work days"}, - {value: "offday", label: "Weekends/Holidays"}] + options = [{value: "workdays", label: "Work days"}, + {value: "weekends", label: "Weekends/Holidays"}] onChangeValue(value) { - if (value === "weekday") { + if (value === "workdays") { this.setState({ title: `${this.props.title} (Work days)`, - data: this.props.data.workdays || [] + selected: value }) } else { this.setState({ title: `${this.props.title} (Weekends)`, - data: this.props.data.weekends || [] + selected: value }) } } render() { - // TODO: this isn't updating when new data loads return( ) From be27108f341f7ca8042ab5271fe132f5196b6c20 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Tue, 30 Nov 2021 13:10:23 -0500 Subject: [PATCH 051/152] apply new peak/offpeak selection to traveltimes overtime also --- server/chalicelib/aggregation.py | 54 ++++++------- src/AggregateCharts.js | 135 +++++++++++++++++++++++++++++++ src/AggregateDaily.js | 85 ------------------- src/ChartSet.js | 18 +++-- src/line.js | 17 ++-- 5 files changed, 178 insertions(+), 131 deletions(-) create mode 100644 src/AggregateCharts.js delete mode 100644 src/AggregateDaily.js diff --git a/server/chalicelib/aggregation.py b/server/chalicelib/aggregation.py index 50798c075..a18174ab3 100644 --- a/server/chalicelib/aggregation.py +++ b/server/chalicelib/aggregation.py @@ -35,19 +35,18 @@ def faster_describe(grouped): q3.name = '75%' # TODO: we can take this out if we filter for 'median' in the front end stats.rename(columns={'median': '50%'}, inplace=True) - stats = pd.concat([stats, q1, q3, std], axis=1) + stats = pd.concat([stats, q1, q3, std], axis=1).reset_index() # This will filter out some probable outliers. return stats.loc[stats['count'] > 4] - #################### # TRAVEL TIMES #################### # `aggregate_traveltime_data` will fetch and clean the data # There are `calc_travel_times_over_time` and `calc_travel_times_daily` will use the data to aggregate in various ways # `travel_times_all` will return all calculated aggregates -# `travel_times_over_time` is legacy and returns just the over-time calculation +# `travel_times_over_time` is legacy and returns just the over-time aggregation w/ peak == all def aggregate_traveltime_data(sdate, edate, from_stop, to_stop): all_data = data_funcs.travel_times(sdate, [from_stop], [to_stop], edate) @@ -74,53 +73,48 @@ def calc_travel_times_daily(df): timedeltas.loc[timedeltas < SERVICE_HR_OFFSET] += datetime.timedelta(days=1) df['dep_time_from_epoch'] = timedeltas + datetime.datetime(1970, 1, 1) - workday = df.loc[df.is_peak_day] - weekend = df.loc[~df.is_peak_day] - - # resample: groupby on 'dep_time_from_epoch' in 30 minute chunks. - workday_stats = faster_describe(workday.resample('30T', on='dep_time_from_epoch')['travel_time_sec']).reset_index() - weekend_stats = faster_describe(weekend.resample('30T', on='dep_time_from_epoch')['travel_time_sec']).reset_index() + stats = faster_describe(df.groupby('is_peak_day').resample('30T', on='dep_time_from_epoch')['travel_time_sec']) + stats['dep_time_from_epoch'] = stats['dep_time_from_epoch'].dt.strftime("%Y-%m-%dT%H:%M:%S") - workday_stats['dep_time_from_epoch'] = workday_stats['dep_time_from_epoch'].dt.strftime("%Y-%m-%dT%H:%M:%S") - weekend_stats['dep_time_from_epoch'] = weekend_stats['dep_time_from_epoch'].dt.strftime("%Y-%m-%dT%H:%M:%S") - - return { - 'workdays': workday_stats.to_dict('records'), - 'weekends': weekend_stats.to_dict('records') - } + return stats def calc_travel_times_over_time(df): # get summary stats summary_stats = faster_describe(df.groupby('service_date')['travel_time_sec']) summary_stats['peak'] = 'all' - # reset_index to turn into dataframe - summary_stats = summary_stats.reset_index() + # summary_stats for peak / off-peak trains - summary_stats_peak = faster_describe(df.groupby(['service_date', 'peak'])['travel_time_sec']).reset_index() + summary_stats_peak = faster_describe(df.groupby(['service_date', 'peak'])['travel_time_sec']) # combine summary stats summary_stats_final = summary_stats.append(summary_stats_peak) - results = summary_stats_final.loc[summary_stats_final['peak'] == 'all'] - return results.to_dict('records') + return summary_stats_final def travel_times_all(sdate, edate, from_stop, to_stop): df = aggregate_traveltime_data(sdate, edate, from_stop, to_stop) if df is None: - return {'overtime': [], 'daily': []} + return { + 'daily': [], + 'overtime': [] + } daily = calc_travel_times_daily(df) overtime = calc_travel_times_over_time(df) return { - 'overtime': overtime, - 'daily': daily + 'daily': daily.to_dict('records'), + 'overtime': overtime.to_dict('records') } def travel_times_over_time(sdate, edate, from_stop, to_stop): - return travel_times_all(sdate, edate, from_stop, to_stop)['overtime'] + df = aggregate_traveltime_data(sdate, edate, from_stop, to_stop) + if df is None: + return [] + overtime = calc_travel_times_over_time(df) + return overtime.loc[overtime['peak'] == 'all'].to_dict('records') #################### @@ -145,10 +139,9 @@ def headways_over_time(sdate, edate, stop): # get summary stats summary_stats = faster_describe(df.groupby('service_date')['headway_time_sec']) summary_stats['peak'] = 'all' - # reset_index to turn into dataframe - summary_stats = summary_stats.reset_index() + # summary_stats for peak / off-peak trains - summary_stats_peak = faster_describe(df.groupby(['service_date', 'peak'])['headway_time_sec']).reset_index() + summary_stats_peak = faster_describe(df.groupby(['service_date', 'peak'])['headway_time_sec']) # combine summary stats summary_stats_final = summary_stats.append(summary_stats_peak) @@ -181,10 +174,9 @@ def dwells_over_time(sdate, edate, stop): # get summary stats summary_stats = faster_describe(df.groupby('service_date')['dwell_time_sec']) summary_stats['peak'] = 'all' - # reset_index to turn into dataframe - summary_stats = summary_stats.reset_index() + # summary_stats for peak / off-peak trains - summary_stats_peak = faster_describe(df.groupby(['service_date', 'peak'])['dwell_time_sec']).reset_index() + summary_stats_peak = faster_describe(df.groupby(['service_date', 'peak'])['dwell_time_sec']) # combine summary stats summary_stats_final = summary_stats.append(summary_stats_peak) diff --git a/src/AggregateCharts.js b/src/AggregateCharts.js new file mode 100644 index 000000000..20f1e9e35 --- /dev/null +++ b/src/AggregateCharts.js @@ -0,0 +1,135 @@ +import React from 'react'; +import { useState } from 'react'; +import classNames from 'classnames'; +import { AggregateDailyLine, AggregateOverTimeLine } from './line'; + +const RadioForm = props => { + const { options, onChange, defaultValue, className } = props; + const [ checked, setChecked ] = useState(defaultValue); + + const handleChange = (evt) => { + const value = evt.target.value; + setChecked(value); + onChange(evt.target.value); + } + + return ( +
+ {options.map((opt, index) => + + )} +
+ ) +} + +class AggregateDaily extends React.Component { + /* + Props: + data + location + isLoading + + */ + constructor(props) { + super(props); + this.onChangeValue = this.onChangeValue.bind(this); + + this.state = { + title: `${props.title} (Work days)`, + selected: "workdays" + } + } + + options = [ + {value: "workdays", label: "Work days", titleSuffix: "(Work days)"}, + {value: "weekends", label: "Weekends/Holidays", titleSuffix: "(Weekends)"} + ] + + onChangeValue(value) { + let titleSuffix = this.options.find(x => x.value === value).titleSuffix + this.setState({ + title: `${this.props.title} ${titleSuffix}`, + selected: value + }) + } + + render() { + return( + x.is_peak_day === (this.state.selected === "workdays"))} + seriesName={this.props.seriesName} + location={this.props.location} + titleBothStops={this.props.titleBothStops} + isLoading={this.props.isLoading} + > + + + ) + } +} + +class AggregateOverTime extends React.Component { + /* + Props: + data + location + isLoading + + */ + constructor(props) { + super(props); + this.onChangeValue = this.onChangeValue.bind(this); + + this.state = { + title: `${props.title}`, + selected: "all" + } + } + + options = [ + {value: "all", label: "All times", titleSuffix: ""}, + {value: "am_peak", label: "AM Peak", titleSuffix: "(AM Peak only)"}, + {value: "pm_peak", label: "PM Peak", titleSuffix: "(PM Peak only)"}, + {value: "off_peak", label: "Off-peak", titleSuffix: "(Off-peak only)"} + ] + + onChangeValue(value) { + let titleSuffix = this.options.find(x => x.value === value).titleSuffix || "" + this.setState({ + title: `${this.props.title} ${titleSuffix}`, + selected: value + }) + } + + render() { + return( + x.peak === this.state.selected)} + seriesName={this.props.seriesName} + location={this.props.location} + titleBothStops={this.props.titleBothStops} + isLoading={this.props.isLoading} + startDate={this.props.startDate} + endDate={this.props.endDate} + > + + + ) + } +} + +export { AggregateDaily, AggregateOverTime } \ No newline at end of file diff --git a/src/AggregateDaily.js b/src/AggregateDaily.js deleted file mode 100644 index 15aa1c9f0..000000000 --- a/src/AggregateDaily.js +++ /dev/null @@ -1,85 +0,0 @@ -import React from 'react'; -import { useState } from 'react'; -import classNames from 'classnames'; -import { AggregateDailyLine } from './line'; - -const RadioForm = props => { - const { options, onChange, defaultValue, className } = props; - const [ checked, setChecked ] = useState(defaultValue); - - const handleChange = (evt) => { - const value = evt.target.value; - setChecked(value); - onChange(evt.target.value); - } - - return ( -
- {options.map((opt, index) => - - )} -
- ) -} - -class AggregateDaily extends React.Component { - /* - Props: - data - location - isLoading - - */ - constructor(props) { - super(props); - this.onChangeValue = this.onChangeValue.bind(this); - - this.state = { - title: `${props.title} (Work days)`, - selected: "workdays" - } - } - - options = [{value: "workdays", label: "Work days"}, - {value: "weekends", label: "Weekends/Holidays"}] - - onChangeValue(value) { - if (value === "workdays") { - this.setState({ - title: `${this.props.title} (Work days)`, - selected: value - }) - } else { - this.setState({ - title: `${this.props.title} (Weekends)`, - selected: value - }) - } - } - - render() { - return( - - - - ) - } -} - -export { AggregateDaily } \ No newline at end of file diff --git a/src/ChartSet.js b/src/ChartSet.js index 52a0aa789..89a021e72 100644 --- a/src/ChartSet.js +++ b/src/ChartSet.js @@ -1,6 +1,6 @@ import React from 'react'; -import { AggregateDaily } from './AggregateDaily'; -import { SingleDayLine, AggregateOverTime } from './line'; +import { AggregateDaily, AggregateOverTime } from './AggregateCharts'; +import { SingleDayLine, AggregateOverTimeLine } from './line'; import { station_direction } from './stations'; @@ -20,6 +20,10 @@ const AggregateSet = (props) => { const locationDescription = getLocationDescription(props.from, props.to, props.line); return(
+ {/** + * Perhaps we want AggregateOverTimeLine still for rail, and only have the peak/offpeak for bus + * In which case, data={props.traveltimes.overtime.filter(x => x.peak === 'all')} + */} { startDate={props.startDate} endDate={props.endDate} /> - { startDate={props.startDate} endDate={props.endDate} /> - { endDate={props.endDate} />
) diff --git a/src/line.js b/src/line.js index 4e76bfafd..b3affe6c2 100644 --- a/src/line.js +++ b/src/line.js @@ -226,7 +226,7 @@ class AggregateLine extends React.Component { { label: "25th percentile", fill: 1, - backgroundColor: "rgba(191,200,214,0.5)", + backgroundColor: this.props.fillColor, lineTension: 0.4, pointRadius: 0, data: this.props.data.map(item => (item["25%"] / 60).toFixed(2)) @@ -234,7 +234,7 @@ class AggregateLine extends React.Component { { label: "75th percentile", fill: 1, - backgroundColor: "rgba(191,200,214,0.5)", + backgroundColor: this.props.fillColor, lineTension: 0.4, pointRadius: 0, data: this.props.data.map(item => (item["75%"] / 60).toFixed(2)) @@ -277,15 +277,16 @@ class AggregateLine extends React.Component { } } -const AggregateOverTime = (props) => { +const AggregateOverTimeLine = (props) => { return( ) } @@ -296,11 +297,11 @@ const AggregateDailyLine = (props) => { {...props} xField={'dep_time_from_epoch'} timeUnit={'hour'} - timeFormat={'LTS'} // maybe change this w/o seconds? - // TODO: xMin, xMax - // also: maybe change fill color to be distinct? + timeFormat={'LT'} // momentjs format: locale time + fillColor={"rgba(136,174,230,0.5)"} + // xMin, xMax? /> ) } -export { SingleDayLine, AggregateOverTime, AggregateDailyLine }; +export { SingleDayLine, AggregateOverTimeLine, AggregateDailyLine }; From eddb6fa93aae8e2c160a7450429025558534ee5e Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Tue, 30 Nov 2021 13:19:18 -0500 Subject: [PATCH 052/152] forgot to lint again --- server/chalicelib/aggregation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/chalicelib/aggregation.py b/server/chalicelib/aggregation.py index a18174ab3..f4523ab09 100644 --- a/server/chalicelib/aggregation.py +++ b/server/chalicelib/aggregation.py @@ -40,6 +40,7 @@ def faster_describe(grouped): # This will filter out some probable outliers. return stats.loc[stats['count'] > 4] + #################### # TRAVEL TIMES #################### From 9dafd236cdbc17cacefb30fb68f3f5759149551b Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 1 Dec 2021 01:42:59 -0500 Subject: [PATCH 053/152] review feedback (mostly renaming) --- server/chalicelib/aggregation.py | 24 +++++++-------- src/App.js | 4 +-- src/{ChartSet.js => ChartSets.js} | 18 +++++------ ...AggregateCharts.js => SelectableCharts.js} | 30 +++++++++---------- src/line.js | 8 ++--- 5 files changed, 42 insertions(+), 42 deletions(-) rename src/{ChartSet.js => ChartSets.js} (89%) rename src/{AggregateCharts.js => SelectableCharts.js} (79%) diff --git a/server/chalicelib/aggregation.py b/server/chalicelib/aggregation.py index f4523ab09..7e9e81ab2 100644 --- a/server/chalicelib/aggregation.py +++ b/server/chalicelib/aggregation.py @@ -45,9 +45,9 @@ def faster_describe(grouped): # TRAVEL TIMES #################### # `aggregate_traveltime_data` will fetch and clean the data -# There are `calc_travel_times_over_time` and `calc_travel_times_daily` will use the data to aggregate in various ways +# There are `calc_travel_times_by_date` and `calc_travel_times_by_time` will use the data to aggregate in various ways # `travel_times_all` will return all calculated aggregates -# `travel_times_over_time` is legacy and returns just the over-time aggregation w/ peak == all +# `travel_times_over_time` is legacy and returns just the by_date aggregation w/ peak == all def aggregate_traveltime_data(sdate, edate, from_stop, to_stop): all_data = data_funcs.travel_times(sdate, [from_stop], [to_stop], edate) @@ -68,7 +68,7 @@ def aggregate_traveltime_data(sdate, edate, from_stop, to_stop): return df -def calc_travel_times_daily(df): +def calc_travel_times_by_time(df): # convert time of day to a consistent datetime relative to epoch timedeltas = pd.to_timedelta(df['dep_time'].astype(str)) timedeltas.loc[timedeltas < SERVICE_HR_OFFSET] += datetime.timedelta(days=1) @@ -80,7 +80,7 @@ def calc_travel_times_daily(df): return stats -def calc_travel_times_over_time(df): +def calc_travel_times_by_date(df): # get summary stats summary_stats = faster_describe(df.groupby('service_date')['travel_time_sec']) summary_stats['peak'] = 'all' @@ -98,15 +98,15 @@ def travel_times_all(sdate, edate, from_stop, to_stop): df = aggregate_traveltime_data(sdate, edate, from_stop, to_stop) if df is None: return { - 'daily': [], - 'overtime': [] + 'by_date': [], + 'by_time': [] } - daily = calc_travel_times_daily(df) - overtime = calc_travel_times_over_time(df) + by_date = calc_travel_times_by_date(df) + by_time = calc_travel_times_by_time(df) return { - 'daily': daily.to_dict('records'), - 'overtime': overtime.to_dict('records') + 'by_date': by_date.to_dict('records'), + 'by_time': by_time.to_dict('records') } @@ -114,8 +114,8 @@ def travel_times_over_time(sdate, edate, from_stop, to_stop): df = aggregate_traveltime_data(sdate, edate, from_stop, to_stop) if df is None: return [] - overtime = calc_travel_times_over_time(df) - return overtime.loc[overtime['peak'] == 'all'].to_dict('records') + stats = calc_travel_times_by_date(df) + return stats.loc[stats['peak'] == 'all'].to_dict('records') #################### diff --git a/src/App.js b/src/App.js index f52525c38..d879b06ef 100644 --- a/src/App.js +++ b/src/App.js @@ -1,6 +1,6 @@ import React from 'react'; import ReactGA from 'react-ga'; -import { SingleDaySet, AggregateSet } from './ChartSet'; +import { SingleDaySet, AggregateSet } from './ChartSets'; import StationConfiguration from './StationConfiguration'; import { withRouter } from 'react-router-dom'; import { lookup_station_by_id, get_stop_ids_for_stations } from './stations'; @@ -202,7 +202,7 @@ class App extends React.Component { options["start_date"] = this.state.configuration.date_start; options["end_date"] = this.state.configuration.date_end; - var method = name; + let method = name; if (name === "traveltimes") { method = "traveltimes2" } diff --git a/src/ChartSet.js b/src/ChartSets.js similarity index 89% rename from src/ChartSet.js rename to src/ChartSets.js index 89a021e72..7c20a88c2 100644 --- a/src/ChartSet.js +++ b/src/ChartSets.js @@ -1,6 +1,6 @@ import React from 'react'; -import { AggregateDaily, AggregateOverTime } from './AggregateCharts'; -import { SingleDayLine, AggregateOverTimeLine } from './line'; +import { AggregateByDateSelectable, AggregateByTimeSelectable } from './SelectableCharts'; +import { SingleDayLine, AggregateByDate } from './line'; import { station_direction } from './stations'; @@ -24,9 +24,9 @@ const AggregateSet = (props) => { * Perhaps we want AggregateOverTimeLine still for rail, and only have the peak/offpeak for bus * In which case, data={props.traveltimes.overtime.filter(x => x.peak === 'all')} */} - { startDate={props.startDate} endDate={props.endDate} /> - { startDate={props.startDate} endDate={props.endDate} /> - { startDate={props.startDate} endDate={props.endDate} /> - {
} -export { SingleDaySet, AggregateSet } \ No newline at end of file +export { SingleDaySet, AggregateSet } diff --git a/src/AggregateCharts.js b/src/SelectableCharts.js similarity index 79% rename from src/AggregateCharts.js rename to src/SelectableCharts.js index 20f1e9e35..90cd4c10a 100644 --- a/src/AggregateCharts.js +++ b/src/SelectableCharts.js @@ -1,7 +1,7 @@ import React from 'react'; import { useState } from 'react'; import classNames from 'classnames'; -import { AggregateDailyLine, AggregateOverTimeLine } from './line'; +import { AggregateByTime, AggregateByDate } from './line'; const RadioForm = props => { const { options, onChange, defaultValue, className } = props; @@ -28,7 +28,7 @@ const RadioForm = props => { ) } -class AggregateDaily extends React.Component { +class AggregateByTimeSelectable extends React.Component { /* Props: data @@ -41,14 +41,14 @@ class AggregateDaily extends React.Component { this.onChangeValue = this.onChangeValue.bind(this); this.state = { - title: `${props.title} (Work days)`, - selected: "workdays" + title: `${props.title} (Weekday)`, + selected: "weekday" } } options = [ - {value: "workdays", label: "Work days", titleSuffix: "(Work days)"}, - {value: "weekends", label: "Weekends/Holidays", titleSuffix: "(Weekends)"} + {value: "weekday", label: "Weekday", titleSuffix: "(Weekday)"}, + {value: "weekend", label: "Weekend/Holiday", titleSuffix: "(Weekend/Holiday)"} ] onChangeValue(value) { @@ -61,9 +61,9 @@ class AggregateDaily extends React.Component { render() { return( - x.is_peak_day === (this.state.selected === "workdays"))} + data={this.props.data.filter(x => x.is_peak_day === (this.state.selected === "weekday"))} seriesName={this.props.seriesName} location={this.props.location} titleBothStops={this.props.titleBothStops} @@ -73,12 +73,12 @@ class AggregateDaily extends React.Component { options={this.options} defaultValue={this.state.selected} /> - + ) } } -class AggregateOverTime extends React.Component { +class AggregateByDateSelectable extends React.Component { /* Props: data @@ -91,7 +91,7 @@ class AggregateOverTime extends React.Component { this.onChangeValue = this.onChangeValue.bind(this); this.state = { - title: `${props.title}`, + title: props.title, selected: "all" } } @@ -104,7 +104,7 @@ class AggregateOverTime extends React.Component { ] onChangeValue(value) { - let titleSuffix = this.options.find(x => x.value === value).titleSuffix || "" + const titleSuffix = this.options.find(x => x.value === value).titleSuffix || "" this.setState({ title: `${this.props.title} ${titleSuffix}`, selected: value @@ -113,7 +113,7 @@ class AggregateOverTime extends React.Component { render() { return( - x.peak === this.state.selected)} seriesName={this.props.seriesName} @@ -127,9 +127,9 @@ class AggregateOverTime extends React.Component { options={this.options} defaultValue={this.state.selected} /> - + ) } } -export { AggregateDaily, AggregateOverTime } \ No newline at end of file +export { AggregateByDateSelectable, AggregateByTimeSelectable } diff --git a/src/line.js b/src/line.js index b3affe6c2..7c1d0703f 100644 --- a/src/line.js +++ b/src/line.js @@ -277,13 +277,13 @@ class AggregateLine extends React.Component { } } -const AggregateOverTimeLine = (props) => { +const AggregateByDate = (props) => { return( { ) } -const AggregateDailyLine = (props) => { +const AggregateByTime = (props) => { return( { ) } -export { SingleDayLine, AggregateOverTimeLine, AggregateDailyLine }; +export { SingleDayLine, AggregateByDate, AggregateByTime }; From 806eef417585bd35fdbe0a5f3a7f16979ed83616 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 1 Dec 2021 02:26:46 -0500 Subject: [PATCH 054/152] hacky isArray to fix leaving agg mode bug --- src/ChartSets.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ChartSets.js b/src/ChartSets.js index 7c20a88c2..9807947b7 100644 --- a/src/ChartSets.js +++ b/src/ChartSets.js @@ -68,12 +68,14 @@ const AggregateSet = (props) => { const SingleDaySet = (props) => { const locationDescription = getLocationDescription(props.from, props.to, props.line); - const anyTravelBenchmarks = props.traveltimes.some(e => e.benchmark_travel_time_sec > 0); + // props.traveltimes might be an Object leftover from aggregate mode + // TODO: fix this by clearing existing data in updateConfiguration + const anyTravelBenchmarks = Array.isArray(props.traveltimes) && props.traveltimes.some(e => e.benchmark_travel_time_sec > 0); const anyHeadwayBenchmarks = props.headways.some(e => e.benchmark_headway_time_sec > 0); return
Date: Thu, 2 Dec 2021 22:29:45 -0500 Subject: [PATCH 055/152] tidying up (more review feedback) --- src/App.js | 6 +-- src/ChartSets.js | 92 ++++++++++++++++++++++++----------------- src/SelectableCharts.js | 51 ++++++++++------------- src/line.js | 2 +- 4 files changed, 78 insertions(+), 73 deletions(-) diff --git a/src/App.js b/src/App.js index d879b06ef..7adb6b46d 100644 --- a/src/App.js +++ b/src/App.js @@ -202,11 +202,7 @@ class App extends React.Component { options["start_date"] = this.state.configuration.date_start; options["end_date"] = this.state.configuration.date_end; - let method = name; - if (name === "traveltimes") { - method = "traveltimes2" - } - + const method = (name === "traveltimes") ? "traveltimes2" : name; url = new URL(`${APP_DATA_BASE_PATH}/aggregate/${method}`, window.location.origin); } else { diff --git a/src/ChartSets.js b/src/ChartSets.js index 7c20a88c2..0e99af1b2 100644 --- a/src/ChartSets.js +++ b/src/ChartSets.js @@ -3,6 +3,26 @@ import { AggregateByDateSelectable, AggregateByTimeSelectable } from './Selectab import { SingleDayLine, AggregateByDate } from './line'; import { station_direction } from './stations'; +const dataFields = { + traveltimes: { + seriesName: "travel time", + xField: "dep_dt", + yField: "travel_time_sec", + benchmarkField: "benchmark_travel_time_sec" + }, + headways: { + seriesName: "headway", + xField: "current_dep_dt", + yField: "headway_time_sec", + benchmarkField: "benchmark_headway_time_sec" + }, + dwells: { + seriesName: "dwell time", + xField: "arr_dt", + yField: "dwell_time_sec", + benchmarkField: null + } +} function getLocationDescription(from, to, line) { if (from && to) { @@ -70,44 +90,40 @@ const SingleDaySet = (props) => { const locationDescription = getLocationDescription(props.from, props.to, props.line); const anyTravelBenchmarks = props.traveltimes.some(e => e.benchmark_travel_time_sec > 0); const anyHeadwayBenchmarks = props.headways.some(e => e.benchmark_headway_time_sec > 0); - return
- - - -
+ return( +
+ + + +
+ ) } export { SingleDaySet, AggregateSet } diff --git a/src/SelectableCharts.js b/src/SelectableCharts.js index 90cd4c10a..a9233e963 100644 --- a/src/SelectableCharts.js +++ b/src/SelectableCharts.js @@ -1,17 +1,9 @@ import React from 'react'; -import { useState } from 'react'; import classNames from 'classnames'; import { AggregateByTime, AggregateByDate } from './line'; -const RadioForm = props => { - const { options, onChange, defaultValue, className } = props; - const [ checked, setChecked ] = useState(defaultValue); - - const handleChange = (evt) => { - const value = evt.target.value; - setChecked(value); - onChange(evt.target.value); - } +const RadioForm = (props) => { + const { options, onChange, checked, className } = props; return (
@@ -19,7 +11,7 @@ const RadioForm = props => { @@ -28,6 +20,11 @@ const RadioForm = props => { ) } +const dayOptions = [ + {value: "weekday", label: "Weekday", titleSuffix: "(Weekday)"}, + {value: "weekend", label: "Weekend/Holiday", titleSuffix: "(Weekend/Holiday)"} +] + class AggregateByTimeSelectable extends React.Component { /* Props: @@ -46,13 +43,8 @@ class AggregateByTimeSelectable extends React.Component { } } - options = [ - {value: "weekday", label: "Weekday", titleSuffix: "(Weekday)"}, - {value: "weekend", label: "Weekend/Holiday", titleSuffix: "(Weekend/Holiday)"} - ] - onChangeValue(value) { - let titleSuffix = this.options.find(x => x.value === value).titleSuffix + let titleSuffix = dayOptions.find(x => x.value === value).titleSuffix this.setState({ title: `${this.props.title} ${titleSuffix}`, selected: value @@ -70,14 +62,22 @@ class AggregateByTimeSelectable extends React.Component { isLoading={this.props.isLoading} > ) } } + +const peakOptions = [ + {value: "all", label: "All times", titleSuffix: ""}, + {value: "am_peak", label: "AM Peak", titleSuffix: "(AM Peak only)"}, + {value: "pm_peak", label: "PM Peak", titleSuffix: "(PM Peak only)"}, + {value: "off_peak", label: "Off-peak", titleSuffix: "(Off-peak only)"} +] + class AggregateByDateSelectable extends React.Component { /* Props: @@ -96,15 +96,8 @@ class AggregateByDateSelectable extends React.Component { } } - options = [ - {value: "all", label: "All times", titleSuffix: ""}, - {value: "am_peak", label: "AM Peak", titleSuffix: "(AM Peak only)"}, - {value: "pm_peak", label: "PM Peak", titleSuffix: "(PM Peak only)"}, - {value: "off_peak", label: "Off-peak", titleSuffix: "(Off-peak only)"} - ] - onChangeValue(value) { - const titleSuffix = this.options.find(x => x.value === value).titleSuffix || "" + const titleSuffix = peakOptions.find(x => x.value === value).titleSuffix || "" this.setState({ title: `${this.props.title} ${titleSuffix}`, selected: value @@ -124,8 +117,8 @@ class AggregateByDateSelectable extends React.Component { endDate={this.props.endDate} > ) diff --git a/src/line.js b/src/line.js index 7c1d0703f..03db35ed8 100644 --- a/src/line.js +++ b/src/line.js @@ -125,7 +125,7 @@ class SingleDayLine extends React.Component { }, { label: `Benchmark MBTA ${this.props.seriesName}`, - data: this.props.data.map(item => (item[this.props.benchmarkField] / 60).toFixed(2)), + data: this.props.useBenchmarks && this.props.data.map(item => (item[this.props.benchmarkField] / 60).toFixed(2)), pointRadius: 0 } ] From 5f415cd2ba84ededf3e59d47fc48f0db960c03d8 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 2 Dec 2021 23:09:26 -0500 Subject: [PATCH 056/152] lint --- server/bus/manifest.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/bus/manifest.py b/server/bus/manifest.py index 22cfd0233..c5aa7eefb 100644 --- a/server/bus/manifest.py +++ b/server/bus/manifest.py @@ -18,12 +18,12 @@ def load_checkpoints(checkpoint_file): return {} -def emit_stop_obj(entry, branches = False): +def emit_stop_obj(entry, branches=False): tpt_id = entry.name if entry['counts'].sum() < 100: # this timepoint is very infrequently used return None - + return { "stop_name": entry.stop_name.iloc[0], "branches": sorted(entry.route_id.unique().tolist()) if branches else None, @@ -41,7 +41,7 @@ def create_manifest(df, routes, checkpoint_file): df['time_point_id'] = df['time_point_id'].str.lower() df['time_point_order'] = df['time_point_order'].fillna(0) - + tpts = df[['route_id', 'stop_id', 'time_point_id', 'direction_id']].value_counts(dropna=False).rename("counts").reset_index() # use checkpoint file to map time_point_id to stop_name @@ -50,14 +50,14 @@ def create_manifest(df, routes, checkpoint_file): # Create full stop id e.g. '66-0-64000' tpts['full_stop_id'] = tpts[['route_id', 'direction_id', 'stop_id']].astype(str).agg('-'.join, axis=1) - + # Must be arranged by inbound direction. Use most common (mode) as guess (w/ max in case 2 modes) orders = df.loc[df.direction_id == 1].groupby("time_point_id", dropna=False)['time_point_order'].agg(lambda x: x.mode().max()) tpts['order_guess'] = tpts['time_point_id'].map(orders).fillna(0).astype(int) stop_objs = tpts.groupby('time_point_id', dropna=False).apply( - lambda x: emit_stop_obj(x, len(routes) > 1) - ).dropna() + lambda x: emit_stop_obj(x, len(routes) > 1) + ).dropna() manifest = { output_route_name: { From 923a8850a25496df592b78ac88c3ced3157b45ef Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 1 Dec 2021 03:31:56 -0500 Subject: [PATCH 057/152] add bare-bones toggle UI --- src/StationConfiguration.js | 6 +++++ src/ui/toggle.css | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 src/ui/toggle.css diff --git a/src/StationConfiguration.js b/src/StationConfiguration.js index 88d674104..80995e645 100644 --- a/src/StationConfiguration.js +++ b/src/StationConfiguration.js @@ -2,6 +2,7 @@ import React from 'react'; import classNames from 'classnames'; import flatpickr from 'flatpickr'; import 'flatpickr/dist/themes/light.css'; +import './ui/toggle.css'; import Select from './Select'; import { all_lines, options_station } from './stations'; @@ -156,6 +157,11 @@ export default class StationConfiguration extends React.Component { return (
+ +
- - +
+ Subway + + Bus +
- + + Bus
diff --git a/src/index.js b/src/index.js index a92aac265..a0b2b8dcf 100644 --- a/src/index.js +++ b/src/index.js @@ -7,5 +7,5 @@ import { BrowserRouter, Route } from 'react-router-dom'; ReactDOM.render( - + , document.getElementById('root')); From eebee3e7efa5057e6328ab1ed54f8a4aac830a64 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Fri, 3 Dec 2021 18:47:37 -0500 Subject: [PATCH 060/152] toggle updates line selector --- src/StationConfiguration.js | 25 ++++++++++++++++++------- src/stations.js | 10 ++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/StationConfiguration.js b/src/StationConfiguration.js index c253a3b34..15fc26301 100644 --- a/src/StationConfiguration.js +++ b/src/StationConfiguration.js @@ -5,18 +5,29 @@ import 'flatpickr/dist/themes/light.css'; import './ui/toggle.css'; import Select from './Select'; -import { all_lines, options_station } from './stations'; +import { bus_lines, subway_lines, options_station } from './stations'; const ua = window.navigator.userAgent; const iOSDevice = !!ua.match(/iPad/i) || !!ua.match(/iPhone/i); const useFlatPickr = !iOSDevice; -const options_lines = all_lines().map((line) => { - return { - value: line, - label: line.charAt(0) + line.slice(1) + ' Line', +const options_lines = (is_bus) => { + if (is_bus) { + return bus_lines().map((line) => { + return { + value: line, + label: line.includes("/") ? `Routes ${line}` : `Route ${line}` + } + }); + } else { + return subway_lines().map((line) => { + return { + value: line, + label: `${line} Line` + } + }); } -}); +}; const options_station_ui = (line) => { return options_station(line).map((station) => { @@ -127,7 +138,7 @@ export default class StationConfiguration extends React.Component { optionsForField(type) { if (type === "line") { - return options_lines; + return options_lines(this.props.current.bus_mode); } if (type === "from") { const toStation = this.decode("to"); diff --git a/src/stations.js b/src/stations.js index 9a42131a8..e571cf614 100644 --- a/src/stations.js +++ b/src/stations.js @@ -22,6 +22,14 @@ const all_lines = () => { return Object.keys(stations); }; +const bus_lines = () => { + return all_lines().filter((line) => stations[line].type === "bus"); +} + +const subway_lines = () => { + return all_lines().filter((line) => stations[line].type != "bus") +} + const lookup_station_by_id = (line, id) => { if (line === "" || line === undefined || id === "" || id === undefined) { return undefined; @@ -57,6 +65,8 @@ const get_stop_ids_for_stations = (from, to) => { export { all_lines, + bus_lines, + subway_lines, options_station, station_direction, lookup_station_by_id, From 4c5800c81f4ed5b904444e0c50131f28e2b517ec Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Fri, 3 Dec 2021 22:36:55 -0500 Subject: [PATCH 061/152] Begin adding bus color --- src/App.css | 5 +++++ src/App.js | 2 +- src/StationConfiguration.js | 8 +++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/App.css b/src/App.css index 2cb39f3f6..fd9be3f97 100644 --- a/src/App.css +++ b/src/App.css @@ -4,6 +4,7 @@ --orange-line: #e66f00; --blue-line: #0e3d8c; --green-line: #159765; + --bus: #ffc72c; --global-shadow: 1px 1px 2px rgba(0, 0, 0, 0.1); --incident-color: #c02828; --incident-tooltip-color: #eaeaea; @@ -143,6 +144,10 @@ background-color: var(--green-line); } +.station-configuration-wrapper.Bus { + background-color: var(--bus); +} + .station-configuration { color: white; padding: 10px; diff --git a/src/App.js b/src/App.js index 529bbdddc..56104bd3c 100644 --- a/src/App.js +++ b/src/App.js @@ -163,7 +163,7 @@ class App extends React.Component { }); } - if (config_change.line && config_change.line !== this.state.configuration.line) { + if ("line" in config_change && config_change.line !== this.state.configuration.line) { update.configuration.from = null; update.configuration.to = null; update.headways = []; diff --git a/src/StationConfiguration.js b/src/StationConfiguration.js index 15fc26301..10b4b762c 100644 --- a/src/StationConfiguration.js +++ b/src/StationConfiguration.js @@ -95,8 +95,9 @@ export default class StationConfiguration extends React.Component { handleBusToggle() { this.props.onConfigurationChange({ - bus_mode: !this.props.current.bus_mode - }); + bus_mode: !this.props.current.bus_mode, + line: null + }, false); } handleSelectDate(field_name) { @@ -173,7 +174,8 @@ export default class StationConfiguration extends React.Component { render() { const currentLine = this.decode("line"); return ( -
+
Subway From b36e9ae7413e7b94776f23e8de098945b2804878 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 4 Dec 2021 00:19:15 -0500 Subject: [PATCH 062/152] add new alert filter --- src/AlertFilter.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/AlertFilter.js b/src/AlertFilter.js index 64b7c28e1..1d72df42a 100644 --- a/src/AlertFilter.js +++ b/src/AlertFilter.js @@ -7,8 +7,11 @@ const known = [ /shuttle/, // might want to worry about this one... ]; +// TODO: audit this. Like, list all the alerts +// to see what filters are actually accurate const anti = [ - / stop .* move /i // "The stop X will permanently move to Y" + / stop .* move /i, // "The stop X will permanently move to Y" + /temporary stop/ ] const findMatch = (alert) => { From 29a724772663fc7a17dcac021c22cab7db3c8363 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 4 Dec 2021 00:20:55 -0500 Subject: [PATCH 063/152] black on yellow for bus config --- src/App.css | 23 ++++++++++++++++++----- src/stations.js | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/App.css b/src/App.css index fd9be3f97..bccaa8e29 100644 --- a/src/App.css +++ b/src/App.css @@ -8,7 +8,8 @@ --global-shadow: 1px 1px 2px rgba(0, 0, 0, 0.1); --incident-color: #c02828; --incident-tooltip-color: #eaeaea; - --swap: url("data:image/svg+xml,"); + --swap-white: url("data:image/svg+xml,"); + --swap-black: url("data:image/svg+xml,"); } .App { @@ -126,6 +127,7 @@ .station-configuration-wrapper { background-color: var(--transitmatters-red); box-shadow: var(--global-shadow); + color: white; } .station-configuration-wrapper.Red { @@ -146,10 +148,10 @@ .station-configuration-wrapper.Bus { background-color: var(--bus); + color: black; } .station-configuration { - color: white; padding: 10px; } @@ -201,10 +203,17 @@ button:disabled, .station-configuration select, .station-configuration input, .station-configuration button { - color: white; + color: inherit; border-color: rgba(255, 255, 255, 0.6); } +.station-configuration-wrapper.Bus select, +.station-configuration-wrapper.Bus input, +.station-configuration-wrapper.Bus button { + color: inherit; + border-color: rgba(0, 0, 0, 0.4) +} + .station-configuration select, .station-configuration input { box-shadow: var(--global-shadow); @@ -221,7 +230,7 @@ button:disabled, } .station-configuration input::placeholder { - color: white; + color: inherit; } .station-configuration button:not(:disabled):hover { @@ -235,7 +244,7 @@ button:disabled, } .station-configuration .swap-stations-button .swap-icon { - background-image: var(--swap); + background-image: var(--swap-white); background-repeat: no-repeat; background-size: 100%; background-position: center; @@ -243,6 +252,10 @@ button:disabled, width: 24px; } +.station-configuration-wrapper.Bus .station-configuration .swap-stations-button .swap-icon { + background-image: var(--swap-black); +} + .station-configuration .swap-stations-button .swap-label { margin-left: 6px; } diff --git a/src/stations.js b/src/stations.js index e571cf614..094c96908 100644 --- a/src/stations.js +++ b/src/stations.js @@ -27,7 +27,7 @@ const bus_lines = () => { } const subway_lines = () => { - return all_lines().filter((line) => stations[line].type != "bus") + return all_lines().filter((line) => stations[line].type !== "bus") } const lookup_station_by_id = (line, id) => { From 9bcc2788a0085159ccf455c1154e26caa2792911 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 4 Dec 2021 00:21:32 -0500 Subject: [PATCH 064/152] switch to T approved colors --- src/App.css | 8 ++++---- src/constants.js | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/App.css b/src/App.css index bccaa8e29..cc28f4fac 100644 --- a/src/App.css +++ b/src/App.css @@ -1,9 +1,9 @@ :root { --transitmatters-red: #d31a2b; - --red-line: #d13434; - --orange-line: #e66f00; - --blue-line: #0e3d8c; - --green-line: #159765; + --red-line: #da291c; + --orange-line: #ed8b00; + --blue-line: #003da5; + --green-line: #00843d; --bus: #ffc72c; --global-shadow: 1px 1px 2px rgba(0, 0, 0, 0.1); --incident-color: #c02828; diff --git a/src/constants.js b/src/constants.js index 99b114111..e94c40e3a 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,8 +1,8 @@ export const colorsForLine = { - Red: '#d13434', - Orange: '#e66f00', - Blue: '#0e3d8c', - Green: '#159765', + Red: '#da291c', + Orange: '#ed8b00', + Blue: '#003da5', + Green: '#00834d', bus: '#ffc72c', }; From 5039fff2b72922c5fc7ee66f9d548fe8782518e0 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 5 Dec 2021 21:18:29 -0500 Subject: [PATCH 065/152] single -> double quotes --- server/bus/bus2train.py | 32 ++++++++++++++++---------------- server/bus/gen_manifests.sh | 2 +- server/bus/manifest.py | 32 ++++++++++++++++---------------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 3ed5b4067..9e30a1689 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -25,12 +25,12 @@ def load_data(input_csv, routes): """ # thinking about doing this in pandas to have all the info at once - df = pd.read_csv(input_csv, parse_dates=['service_date', 'scheduled', 'actual']) + df = pd.read_csv(input_csv, parse_dates=["service_date", "scheduled", "actual"]) # We need to keep both "Headway" AND "Schedule": both can have timepoint data. df = df.loc[df.actual.notnull()] - df.route_id = df.route_id.str.lstrip('0') + df.route_id = df.route_id.str.lstrip("0") if routes: df = df.loc[df.route_id.isin(routes)] @@ -55,17 +55,17 @@ def process_events(df): CSV_HEADER = ["service_date", "route_id", "trip_id", "direction_id", "stop_id", "stop_sequence", "vehicle_id", "vehicle_label", "event_type", "event_time"] - df = df.rename(columns={'half_trip_id': 'trip_id', - 'time_point_order': 'stop_sequence', - 'actual': 'event_time'}) - df.drop(columns=['time_point_id', 'standard_type', 'scheduled', 'scheduled_headway', 'headway']) - df['vehicle_id'] = "" - df['vehicle_label'] = "" + df = df.rename(columns={"half_trip_id": "trip_id", + "time_point_order": "stop_sequence", + "actual": "event_time"}) + df.drop(columns=["time_point_id", "standard_type", "scheduled", "scheduled_headway", "headway"]) + df["vehicle_id"] = "" + df["vehicle_label"] = "" - df['event_type'] = df.point_type.map({"Startpoint": ["DEP"], + df["event_type"] = df.point_type.map({"Startpoint": ["DEP"], "Midpoint": ["ARR", "DEP"], "Endpoint": ["ARR"]}) - df = df.explode('event_type') + df = df.explode("event_type") df = df[CSV_HEADER] # reorder return df @@ -88,25 +88,25 @@ def _write_file(events, outdir, nozip=False): "events.csv.gz") fname.parent.mkdir(parents=True, exist_ok=True) # set mtime to 0 in gzip header for determinism (so we can re-gen old routes, and rsync to s3 will ignore) - events.to_csv(fname, index=False, compression={'method': 'gzip', 'mtime': 0} if not nozip else None) + events.to_csv(fname, index=False, compression={"method": "gzip", "mtime": 0} if not nozip else None) def to_disk(df, root, nozip=False): """ For each service_date/stop_id group, we call the helper that will write it to disk. """ - df.groupby(['service_date', 'stop_id', 'direction_id', 'route_id']).apply(lambda e: _write_file(e, root, nozip)) + df.groupby(["service_date", "stop_id", "direction_id", "route_id"]).apply(lambda e: _write_file(e, root, nozip)) def main(): parser = argparse.ArgumentParser() - parser.add_argument('input', metavar='INPUT_CSV') - parser.add_argument('output', metavar='OUTPUT_DIR') - parser.add_argument('--routes', '-r', nargs="*", type=str, + parser.add_argument("input", metavar="INPUT_CSV") + parser.add_argument("output", metavar="OUTPUT_DIR") + parser.add_argument("--routes", "-r", nargs="*", type=str, help="One note here: we should always be additive with our route set \ in case 2 lines share the same stop id: we need both in the result file.") - parser.add_argument('--nozip', '-nz', action='store_true', help="debug feature to skip gzipping") + parser.add_argument("--nozip", "-nz", action="store_true", help="debug feature to skip gzipping") args = parser.parse_args() input_csv = args.input diff --git a/server/bus/gen_manifests.sh b/server/bus/gen_manifests.sh index 53524bd9e..39d408857 100755 --- a/server/bus/gen_manifests.sh +++ b/server/bus/gen_manifests.sh @@ -7,4 +7,4 @@ mkdir data/output/manifests/$route for f in $(find data/input/MBTA_Bus_Arrival_Departure_Times_2021/ -name *.csv); do month=$(echo $f | rev | cut -d- -f1 | rev | cut -d. -f1) pipenv run python manifest.py $f "data/output/manifests/$route/$route_$month.json" --checkpoints "data/MBTA_GTFS/checkpoints.txt" -r $route -done \ No newline at end of file +done diff --git a/server/bus/manifest.py b/server/bus/manifest.py index c5aa7eefb..15e913c6f 100644 --- a/server/bus/manifest.py +++ b/server/bus/manifest.py @@ -20,7 +20,7 @@ def load_checkpoints(checkpoint_file): def emit_stop_obj(entry, branches=False): tpt_id = entry.name - if entry['counts'].sum() < 100: + if entry["counts"].sum() < 100: # this timepoint is very infrequently used return None @@ -37,25 +37,25 @@ def emit_stop_obj(entry, branches=False): def create_manifest(df, routes, checkpoint_file): - output_route_name = '/'.join(routes) + output_route_name = "/".join(routes) - df['time_point_id'] = df['time_point_id'].str.lower() - df['time_point_order'] = df['time_point_order'].fillna(0) + df["time_point_id"] = df["time_point_id"].str.lower() + df["time_point_order"] = df["time_point_order"].fillna(0) - tpts = df[['route_id', 'stop_id', 'time_point_id', 'direction_id']].value_counts(dropna=False).rename("counts").reset_index() + tpts = df[["route_id", "stop_id", "time_point_id", "direction_id"]].value_counts(dropna=False).rename("counts").reset_index() # use checkpoint file to map time_point_id to stop_name station_names = load_checkpoints(checkpoint_file) - tpts['stop_name'] = tpts['time_point_id'].map(station_names) + tpts["stop_name"] = tpts["time_point_id"].map(station_names) - # Create full stop id e.g. '66-0-64000' - tpts['full_stop_id'] = tpts[['route_id', 'direction_id', 'stop_id']].astype(str).agg('-'.join, axis=1) + # Create full stop id e.g. "66-0-64000" + tpts["full_stop_id"] = tpts[["route_id", "direction_id", "stop_id"]].astype(str).agg("-".join, axis=1) # Must be arranged by inbound direction. Use most common (mode) as guess (w/ max in case 2 modes) - orders = df.loc[df.direction_id == 1].groupby("time_point_id", dropna=False)['time_point_order'].agg(lambda x: x.mode().max()) - tpts['order_guess'] = tpts['time_point_id'].map(orders).fillna(0).astype(int) + orders = df.loc[df.direction_id == 1].groupby("time_point_id", dropna=False)["time_point_order"].agg(lambda x: x.mode().max()) + tpts["order_guess"] = tpts["time_point_id"].map(orders).fillna(0).astype(int) - stop_objs = tpts.groupby('time_point_id', dropna=False).apply( + stop_objs = tpts.groupby("time_point_id", dropna=False).apply( lambda x: emit_stop_obj(x, len(routes) > 1) ).dropna() @@ -66,7 +66,7 @@ def create_manifest(df, routes, checkpoint_file): "0": "outbound", "1": "inbound" }, - "stations": sorted(stop_objs.values, key=lambda x: x['order']) + "stations": sorted(stop_objs.values, key=lambda x: x["order"]) } } @@ -76,10 +76,10 @@ def create_manifest(df, routes, checkpoint_file): def main(): parser = argparse.ArgumentParser() - parser.add_argument('input', metavar='INPUT_CSV') - parser.add_argument('output', metavar='OUTPUT_JSON') - parser.add_argument('--routes', '-r', nargs="+", type=str) - parser.add_argument('--checkpoints', metavar="gtfs_checkpoints.txt") + parser.add_argument("input", metavar="INPUT_CSV") + parser.add_argument("output", metavar="OUTPUT_JSON") + parser.add_argument("--routes", "-r", nargs="+", type=str) + parser.add_argument("--checkpoints", metavar="gtfs_checkpoints.txt") args = parser.parse_args() input_csv = args.input From 6b74efdf8c487ea54b365762b2a2e7dee9d186f5 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 5 Dec 2021 21:44:33 -0500 Subject: [PATCH 066/152] use for loop for readability --- server/bus/bus2train.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 9e30a1689..5bb5685f2 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -71,31 +71,26 @@ def process_events(df): return df -def _write_file(events, outdir, nozip=False): +def to_disk(df, outdir, nozip=False): """ - This is a helper that will write the events to disk. - It will be called on each "groupby" object, grouping stop_id and service_date + For each service_date/stop_id/direction/route group, we write the events to disk. """ - service_date, stop_id, direction_id, route_id = events.name - - fname = pathlib.Path(outdir, - "Events", - "daily-bus-data", - f"{route_id}-{direction_id}-{stop_id}", - f"Year={service_date.year}", - f"Month={service_date.month}", - f"Day={service_date.day}", - "events.csv.gz") - fname.parent.mkdir(parents=True, exist_ok=True) - # set mtime to 0 in gzip header for determinism (so we can re-gen old routes, and rsync to s3 will ignore) - events.to_csv(fname, index=False, compression={"method": "gzip", "mtime": 0} if not nozip else None) - - -def to_disk(df, root, nozip=False): - """ - For each service_date/stop_id group, we call the helper that will write it to disk. - """ - df.groupby(["service_date", "stop_id", "direction_id", "route_id"]).apply(lambda e: _write_file(e, root, nozip)) + grouped = df.groupby(["service_date", "stop_id", "direction_id", "route_id"]) + + for name, events in grouped: + service_date, stop_id, direction_id, route_id = name + + fname = pathlib.Path(outdir, + "Events", + "daily-bus-data", + f"{route_id}-{direction_id}-{stop_id}", + f"Year={service_date.year}", + f"Month={service_date.month}", + f"Day={service_date.day}", + "events.csv.gz") + fname.parent.mkdir(parents=True, exist_ok=True) + # set mtime to 0 in gzip header for determinism (so we can re-gen old routes, and rsync to s3 will ignore) + events.to_csv(fname, index=False, compression={"method": "gzip", "mtime": 0} if not nozip else None) def main(): From 65b838ab28d4e0675a245525268d00b6b2ac127b Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 5 Dec 2021 21:57:28 -0500 Subject: [PATCH 067/152] more readable alert filtering --- src/AlertFilter.js | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/AlertFilter.js b/src/AlertFilter.js index 64b7c28e1..f6b8509f7 100644 --- a/src/AlertFilter.js +++ b/src/AlertFilter.js @@ -4,7 +4,7 @@ const known = [ /[A-Za-z]+ speeds/, // "Reduced speeds" /delay/, /notice/, - /shuttle/, // might want to worry about this one... + /[Ss]huttle/, // might want to worry about this one... ]; const anti = [ @@ -13,19 +13,11 @@ const anti = [ const findMatch = (alert) => { const text = alert.text; - for (const exp of anti) { - const match = text.match(exp); - if (match != null) { - return null; - } - } - for (const exp of known) { - const match = text.match(exp); - if (match != null) { - return match; - } + + if (anti.some((exp) => text.match(exp))) { + return false; } - return null; + return known.some((exp) => text.match(exp)) } const recognize = (alert) => { From 85ffbeeef43d33271c1b5e47b5a17a97a7f799c6 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 5 Dec 2021 22:13:23 -0500 Subject: [PATCH 068/152] differentiate bus and train chartsets --- src/App.js | 1 + src/ChartSets.js | 60 +++++++++++++++++++++++++----------------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/App.js b/src/App.js index 56104bd3c..c21407da7 100644 --- a/src/App.js +++ b/src/App.js @@ -407,6 +407,7 @@ class App extends React.Component { renderCharts() { const propsToPass = { + bus_mode: this.state.configuration.bus_mode, traveltimes: this.state.traveltimes, headways: this.state.headways, dwells: this.state.dwells, diff --git a/src/ChartSets.js b/src/ChartSets.js index c5f102f62..1f14674f7 100644 --- a/src/ChartSets.js +++ b/src/ChartSets.js @@ -1,5 +1,5 @@ import React from 'react'; -import { AggregateByDateSelectable, AggregateByTimeSelectable } from './SelectableCharts'; +import { AggregateByTimeSelectable } from './SelectableCharts'; import { SingleDayLine, AggregateByDate } from './line'; import { station_direction } from './stations'; @@ -38,15 +38,12 @@ function getLocationDescription(from, to, line) { const AggregateSet = (props) => { const locationDescription = getLocationDescription(props.from, props.to, props.line); + const headwayTitle = props.bus_mode ? "Time between buses (headways)" : "Time between trains (headways)"; return(
- {/** - * Perhaps we want AggregateOverTimeLine still for rail, and only have the peak/offpeak for bus - * In which case, data={props.traveltimes.overtime.filter(x => x.peak === 'all')} - */} - x.peak === 'all') || []} seriesName={"Median travel time"} location={locationDescription} titleBothStops={true} @@ -55,7 +52,7 @@ const AggregateSet = (props) => { endDate={props.endDate} /> { startDate={props.startDate} endDate={props.endDate} /> - + {!props.bus_mode && + + } { // TODO: fix this by clearing existing data in updateConfiguration const anyTravelBenchmarks = Array.isArray(props.traveltimes) && props.traveltimes.some(e => e.benchmark_travel_time_sec > 0); const anyHeadwayBenchmarks = props.headways.some(e => e.benchmark_headway_time_sec > 0); + const headwayTitle = props.bus_mode ? "Time between buses (headways)" : "Time between trains (headways)"; return(
{ /> { isLoading={props.isLoadingHeadways} date={props.startDate} /> - + {!props.bus_mode && + + }
) } From 3481cf784651c293a2711444c49a8490b6a553b9 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 5 Dec 2021 23:13:04 -0500 Subject: [PATCH 069/152] mobile css fix for toggle --- src/App.css | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/App.css b/src/App.css index cc28f4fac..e76c2e3c3 100644 --- a/src/App.css +++ b/src/App.css @@ -391,29 +391,37 @@ button:disabled, .station-configuration { display: grid; - grid-template-columns: 1fr 40px; + grid-template-columns: .5fr 1fr 40px; grid-template-areas: - 'line line' - 'date date' - 'stations swap'; + 'toggle line line' + 'date date date' + 'stations stations swap'; column-gap: 10px; row-gap: 10px; } .station-configuration .from-to-label, - .station-configuration .date-label, - .station-configuration .switch-label { + .station-configuration .date-label { width: 30px; text-align: right; margin-right: 10px; flex-shrink: 0; } + .station-configuration .switch-label { + margin-left: 10px; + margin-right: 10px; + } + .station-configuration .end-date-label { margin-right: 0; text-align: center; } + .station-configuration .option-mode { + grid-area: toggle; + } + .station-configuration > .option-line { grid-area: line; } From 91807c3452b8ef9b63a992061e5aabd411c553ae Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 8 Dec 2021 02:06:10 -0500 Subject: [PATCH 070/152] Add outline to toggle switch --- src/ui/toggle.css | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/ui/toggle.css b/src/ui/toggle.css index ab6044fa7..d1f501d18 100644 --- a/src/ui/toggle.css +++ b/src/ui/toggle.css @@ -20,11 +20,20 @@ right: 0; bottom: 0; border-radius: 10px; + outline: 1px solid; background-color: #fff; -webkit-transition: .4s; transition: .4s; } +.station-configuration .slider { + outline-color: rgba(0, 0, 0, 0) +} + +.station-configuration-wrapper.Bus .slider { + outline-color: rgba(0, 0, 0, 0.4); +} + .slider:before { position: absolute; content: ""; From 787075e60b8ded970624d571ec730fa71c8410dc Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 8 Dec 2021 02:10:54 -0500 Subject: [PATCH 071/152] use "decode" --- src/StationConfiguration.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/StationConfiguration.js b/src/StationConfiguration.js index 10b4b762c..f5bcbbe41 100644 --- a/src/StationConfiguration.js +++ b/src/StationConfiguration.js @@ -95,7 +95,7 @@ export default class StationConfiguration extends React.Component { handleBusToggle() { this.props.onConfigurationChange({ - bus_mode: !this.props.current.bus_mode, + bus_mode: !this.decode("bus_mode"), line: null }, false); } @@ -139,7 +139,7 @@ export default class StationConfiguration extends React.Component { optionsForField(type) { if (type === "line") { - return options_lines(this.props.current.bus_mode); + return options_lines(this.decode("bus_mode")); } if (type === "from") { const toStation = this.decode("to"); @@ -175,12 +175,13 @@ export default class StationConfiguration extends React.Component { const currentLine = this.decode("line"); return (
+ this.decode("bus_mode") ? "Bus" : currentLine)}>
+
Subway Bus @@ -191,7 +192,7 @@ export default class StationConfiguration extends React.Component { value={this.decode("line")} options={this.optionsForField("line")} onChange={this.handleSelectOption("line")} - defaultLabel="Select a line..." + defaultLabel={this.decode("bus_mode") ? "Select a route..." : "Select a line..."} />
From 02b1836780e693d67ab1298c403d1f4acad05126 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 8 Dec 2021 02:16:39 -0500 Subject: [PATCH 072/152] add shadow to slider --- src/App.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/App.css b/src/App.css index e76c2e3c3..2e8cc5325 100644 --- a/src/App.css +++ b/src/App.css @@ -221,6 +221,10 @@ button:disabled, padding-left: 10px; } +.station-configuration .slider { + box-shadow: var(--global-shadow); +} + .station-configuration button { display: flex; align-items: center; From 3374bfe49317be3d7c62c8843ded2954cc320eb2 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 8 Dec 2021 03:55:02 -0500 Subject: [PATCH 073/152] switch to react-flatpickr --- package.json | 2 +- src/StationConfiguration.js | 74 ++++++++++++++----------------------- 2 files changed, 28 insertions(+), 48 deletions(-) diff --git a/package.json b/package.json index 00bf87be4..ffefa9d8d 100644 --- a/package.json +++ b/package.json @@ -7,11 +7,11 @@ "chart.js": "^2.9.4", "classnames": "^2.2.6", "concurrently": "^5.2.0", - "flatpickr": "4.5.7", "lodash.merge": "^4.6.2", "react": "^16.13.1", "react-chartjs-2": "^2.11.1", "react-dom": "^16.13.1", + "react-flatpickr": "^3.10.7", "react-ga": "^3.1.2", "react-router-dom": "^5.1.2", "react-scripts": "^3.4.1", diff --git a/src/StationConfiguration.js b/src/StationConfiguration.js index f5bcbbe41..db99003a0 100644 --- a/src/StationConfiguration.js +++ b/src/StationConfiguration.js @@ -1,7 +1,7 @@ import React from 'react'; import classNames from 'classnames'; -import flatpickr from 'flatpickr'; -import 'flatpickr/dist/themes/light.css'; +import Flatpickr from "react-flatpickr"; +import 'react-flatpickr/node_modules/flatpickr/dist/themes/light.css'; import './ui/toggle.css'; import Select from './Select'; @@ -11,6 +11,16 @@ const ua = window.navigator.userAgent; const iOSDevice = !!ua.match(/iPad/i) || !!ua.match(/iPhone/i); const useFlatPickr = !iOSDevice; +const dateRange = { + minDate: "2016-01-01", + maxDate: "today" +}; + +const busDateRange = { + minDate: "2021-01-01", + maxDate: "2021-09-30" +}; + const options_lines = (is_bus) => { if (is_bus) { return bus_lines().map((line) => { @@ -42,13 +52,10 @@ const options_station_ui = (line) => { export default class StationConfiguration extends React.Component { constructor(props) { super(props); - this.picker_start = React.createRef(); - this.picker_end = React.createRef(); this.handleSelectDate = this.handleSelectDate.bind(this); this.handleSelectRawDate = this.handleSelectRawDate.bind(this); this.handleSwapStations = this.handleSwapStations.bind(this); this.clearMoreOptions = this.clearMoreOptions.bind(this); - this.setupPickers = this.setupPickers.bind(this); this.handleBusToggle = this.handleBusToggle.bind(this); this.state = { @@ -56,34 +63,8 @@ export default class StationConfiguration extends React.Component { }; } - setupPickers() { - if (useFlatPickr) { - // Only initialize once, even after rerenders - if(!this.picker_start.current._flatpickr) { - flatpickr(this.picker_start.current, { - onChange: this.handleSelectDate("date_start"), - maxDate: 'today', - minDate: "2016-01-15" - }); - } - // Only initialize once, even after rerenders - if (this.state.show_date_end_picker && !this.picker_end.current._flatpickr) { - flatpickr(this.picker_end.current, { - onChange: this.handleSelectDate("date_end"), - maxDate: 'today', - minDate: "2016-01-15" - }); - } - } - } - - componentDidMount() { - this.setupPickers(); - } componentDidUpdate(prevProps) { - this.setupPickers(); - // If the date_end prop shows up because a config preset set it, // then show the end date picker. if(this.props.current.date_end !== prevProps.current.date_end) { @@ -96,7 +77,9 @@ export default class StationConfiguration extends React.Component { handleBusToggle() { this.props.onConfigurationChange({ bus_mode: !this.decode("bus_mode"), - line: null + line: null, + date_start: null, + date_end: null }, false); } @@ -163,9 +146,6 @@ export default class StationConfiguration extends React.Component { this.setState({ show_date_end_picker: false, }); - if(this.picker_end.current._flatpickr) { - this.picker_end.current._flatpickr.destroy(); - } this.props.onConfigurationChange({ date_end: null, }); @@ -226,12 +206,12 @@ export default class StationConfiguration extends React.Component {
Date - {!!this.state.show_date_end_picker && <> to -
Date - {!!this.state.show_date_end_picker && <> to -
@@ -179,7 +179,7 @@ export default class StationConfiguration extends React.Component {
Date to Date: Tue, 14 Dec 2021 19:33:07 -0500 Subject: [PATCH 090/152] Add prior-year stops for existing routes. --- src/bus_constants/111.json | 19 ++++++++++++++++++- src/bus_constants/114-116-117.json | 1 + src/bus_constants/28.json | 5 +++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/bus_constants/111.json b/src/bus_constants/111.json index 9ccc2b31e..c833e9531 100644 --- a/src/bus_constants/111.json +++ b/src/bus_constants/111.json @@ -16,7 +16,8 @@ "111-1-5547" ], "0": [ - "111-0-5547" + "111-0-5547", + "111-0-5636" ] } }, @@ -133,6 +134,22 @@ "111-0-2832" ] } + }, + { + "stop_name": "Haymarket", + "branches": null, + "station": "hayms", + "order": 10, + "stops": { + "1": [ + "111-1-8309", + "111-1-8310" + ], + "0": [ + "111-0-8309", + "111-0-8310" + ] + } } ] } diff --git a/src/bus_constants/114-116-117.json b/src/bus_constants/114-116-117.json index e68e2cc07..97ec0407a 100644 --- a/src/bus_constants/114-116-117.json +++ b/src/bus_constants/114-116-117.json @@ -125,6 +125,7 @@ "1": [ "114-1-5605", "116-1-5605", + "116-1-5615", "116-1-56170", "117-1-5605" ], diff --git a/src/bus_constants/28.json b/src/bus_constants/28.json index 7c78a129a..e792b4831 100644 --- a/src/bus_constants/28.json +++ b/src/bus_constants/28.json @@ -100,7 +100,8 @@ "28-1-390" ], "0": [ - "28-0-407" + "28-0-407", + "28-0-13321" ] } }, @@ -133,7 +134,7 @@ } }, { - "stop_name": "Roxbury Crossing Station", + "stop_name": "Roxbury Crossing Station (Bus)", "branches": null, "station": "roxbs", "order": 10, From 2ef18398bbd07f37e56a23683e75c99ac3fea828 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 20:22:39 -0500 Subject: [PATCH 091/152] Adding more bus lines to the potential list for bus dashboard. --- src/bus_constants/15.json | 139 +++++++++++++++++++++++++++++++++++ src/bus_constants/22.json | 141 +++++++++++++++++++++++++++++++++++ src/bus_constants/32.json | 97 ++++++++++++++++++++++++ src/bus_constants/39.json | 150 ++++++++++++++++++++++++++++++++++++++ src/bus_constants/71.json | 125 +++++++++++++++++++++++++++++++ src/bus_constants/73.json | 111 ++++++++++++++++++++++++++++ src/bus_constants/77.json | 142 ++++++++++++++++++++++++++++++++++++ src/stations.js | 7 ++ 8 files changed, 912 insertions(+) create mode 100644 src/bus_constants/15.json create mode 100644 src/bus_constants/22.json create mode 100644 src/bus_constants/32.json create mode 100644 src/bus_constants/39.json create mode 100644 src/bus_constants/71.json create mode 100644 src/bus_constants/73.json create mode 100644 src/bus_constants/77.json diff --git a/src/bus_constants/15.json b/src/bus_constants/15.json new file mode 100644 index 000000000..bc1b0079f --- /dev/null +++ b/src/bus_constants/15.json @@ -0,0 +1,139 @@ +{ + "15": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Fields Corner", + "branches": null, + "station": "fldcr", + "order": 1, + "stops": { + "1": [ + "15-1-322", + "15-1-323" + ], + "0": [ + "15-0-323" + ] + } + }, + { + "stop_name": "Saint Peter's Square (Dorchester)", + "branches": null, + "station": "peter", + "order": 1, + "stops": { + "1": [ + "15-1-15100" + ], + "0": [ + "15-0-15100" + ] + } + }, + { + "stop_name": "Geneva Avenue @ Bowdoin Street", + "branches": null, + "station": "genbo", + "order": 2, + "stops": { + "1": [ + "15-1-1468" + ], + "0": [ + "15-0-1515" + ] + } + }, + { + "stop_name": "Kane Square", + "branches": null, + "station": "kane", + "order": 3, + "stops": { + "1": [ + "15-1-1475" + ], + "0": [ + "15-0-1508" + ] + } + }, + { + "stop_name": "Uphams Corner", + "branches": null, + "station": "upham", + "order": 4, + "stops": { + "1": [ + "15-1-1480" + ], + "0": [ + "15-0-1503" + ] + } + }, + { + "stop_name": "Blue Hill Avenue @ Dudley Street", + "branches": null, + "station": "bhdud", + "order": 5, + "stops": { + "1": [ + "15-1-1486" + ], + "0": [ + "15-0-1497" + ] + } + }, + { + "stop_name": "Nubian Station", + "branches": null, + "station": "nubn", + "order": 6, + "stops": { + "1": [ + "15-1-64000", + "15-1-64" + ], + "0": [ + "15-0-64000" + ] + } + }, + { + "stop_name": "Roxbury Crossing Station", + "branches": null, + "station": "roxbs", + "order": 7, + "stops": { + "1": [ + "15-1-21148" + ], + "0": [ + "15-0-11257" + ] + } + }, + { + "stop_name": "Ruggles", + "branches": null, + "station": "rugg", + "order": 8, + "stops": { + "1": [ + "15-1-17861" + ], + "0": [ + "15-0-17863" + ] + } + } + ] + } +} diff --git a/src/bus_constants/22.json b/src/bus_constants/22.json new file mode 100644 index 000000000..d559da09e --- /dev/null +++ b/src/bus_constants/22.json @@ -0,0 +1,141 @@ +{ + "22": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Ashmont Terminal", + "branches": null, + "station": "ashmt", + "order": 1, + "stops": { + "1": [ + "22-1-334" + ], + "0": [ + "22-0-334" + ] + } + }, + { + "stop_name": "Codman Square", + "branches": null, + "station": "codmn", + "order": 2, + "stops": { + "1": [ + "22-1-371" + ], + "0": [ + "22-0-426" + ] + } + }, + { + "stop_name": "Blue Hill Avenue @ Talbot Avenue", + "branches": null, + "station": "bltal", + "order": 3, + "stops": { + "1": [ + "22-1-378" + ], + "0": [ + "22-0-419" + ] + } + }, + { + "stop_name": "Franklin Park", + "branches": null, + "station": "frnpk", + "order": 4, + "stops": { + "1": [ + "22-1-383" + ], + "0": [ + "22-0-17391", + "22-0-415" + ] + } + }, + { + "stop_name": "Seaver & Humboldt", + "branches": null, + "station": "svrhm", + "order": 5, + "stops": { + "1": [ + "22-1-1741" + ], + "0": [ + "22-0-17411" + ] + } + }, + { + "stop_name": "Egleston Square", + "branches": null, + "station": "egles", + "order": 6, + "stops": { + "1": [ + "22-1-1188" + ], + "0": [ + "22-0-10413", + "22-0-1267" + ] + } + }, + { + "stop_name": "Jackson Square Station", + "branches": null, + "station": "jasst", + "order": 7, + "stops": { + "1": [ + "22-1-11531" + ], + "0": [ + "22-0-11531" + ] + } + }, + { + "stop_name": "Roxbury Crossing Station", + "branches": null, + "station": "roxbs", + "order": 8, + "stops": { + "1": [ + "22-1-1222" + ], + "0": [ + "22-0-1258", + "22-0-21148" + ] + } + }, + { + "stop_name": "Ruggles", + "branches": null, + "station": "rugg", + "order": 9, + "stops": { + "1": [ + "22-1-17861", + "22-1-17863" + ], + "0": [ + "22-0-17862" + ] + } + } + ] + } +} diff --git a/src/bus_constants/32.json b/src/bus_constants/32.json new file mode 100644 index 000000000..c70c125bc --- /dev/null +++ b/src/bus_constants/32.json @@ -0,0 +1,97 @@ +{ + "32": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Wolcott Square", + "branches": null, + "station": "wolsq", + "order": 1, + "stops": { + "1": [ + "32-1-42819" + ], + "0": [ + "32-0-42819" + ] + } + }, + { + "stop_name": "Cleary Square", + "branches": null, + "station": "clsq", + "order": 2, + "stops": { + "1": [ + "32-1-36466" + ], + "0": [ + "32-0-2819", + "32-0-6509" + ] + } + }, + { + "stop_name": "Greenwood & Hyde Park", + "branches": null, + "station": "green", + "order": 3, + "stops": { + "1": [ + "32-1-6471" + ], + "0": [ + "32-0-6504" + ] + } + }, + { + "stop_name": "Hyde Park Avenue @ American Legion Highway", + "branches": null, + "station": "hpaml", + "order": 4, + "stops": { + "1": [ + "32-1-6474" + ], + "0": [ + "32-0-6500" + ] + } + }, + { + "stop_name": "Hyde Park Avenue @ Cummins Highway", + "branches": null, + "station": "hpave", + "order": 5, + "stops": { + "1": [ + "32-1-6478" + ], + "0": [ + "32-0-6496" + ] + } + }, + { + "stop_name": "Forest Hills Busway", + "branches": null, + "station": "fhill", + "order": 6, + "stops": { + "1": [ + "32-1-10642", + "32-1-875" + ], + "0": [ + "32-0-875" + ] + } + } + ] + } +} diff --git a/src/bus_constants/39.json b/src/bus_constants/39.json new file mode 100644 index 000000000..01b6afdbc --- /dev/null +++ b/src/bus_constants/39.json @@ -0,0 +1,150 @@ +{ + "39": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Forest Hills Busway", + "branches": null, + "station": "fhill", + "order": 1, + "stops": { + "1": [ + "39-1-10642" + ], + "0": [ + "39-0-10642" + ] + } + }, + { + "stop_name": "Jamaica Plain Center (Monument)", + "branches": null, + "station": "jpctr", + "order": 2, + "stops": { + "1": [ + "39-1-1128" + ], + "0": [ + "39-0-1939" + ] + } + }, + { + "stop_name": "Centre Street & South Huntington Avenue", + "branches": null, + "station": "shunt", + "order": 3, + "stops": { + "1": [ + "39-1-11131" + ], + "0": [ + "39-0-1160" + ] + } + }, + { + "stop_name": "Heath Street Loop", + "branches": null, + "station": "heath", + "order": 4, + "stops": { + "1": [ + "39-1-65741", + "39-1-6574" + ], + "0": [ + "39-0-31365" + ] + } + }, + { + "stop_name": "Brigham Circle", + "branches": null, + "station": "brghm", + "order": 5, + "stops": { + "1": [ + "39-1-1317" + ], + "0": [ + "39-0-1363" + ] + } + }, + { + "stop_name": "Huntington Avenue & Longwood Avenue", + "branches": null, + "station": "hlong", + "order": 6, + "stops": { + "1": [ + "39-1-31317" + ], + "0": [ + "39-0-91391" + ] + } + }, + { + "stop_name": "Northeastern University", + "branches": null, + "station": "nuniv", + "order": 7, + "stops": { + "1": [ + "39-1-81317" + ], + "0": [ + "39-0-41391" + ] + } + }, + { + "stop_name": "Huntington Avenue & Belvidere Street", + "branches": null, + "station": "hunbv", + "order": 8, + "stops": { + "1": [ + "39-1-11389" + ], + "0": [ + "39-0-11388" + ] + } + }, + { + "stop_name": "Copley", + "branches": null, + "station": "copst", + "order": 9, + "stops": { + "1": [ + "39-1-175" + ], + "0": [] + } + }, + { + "stop_name": "Back Bay", + "branches": null, + "station": "bbsta", + "order": 10, + "stops": { + "1": [ + "39-1-23391" + ], + "0": [ + "39-0-23391" + ] + } + } + ] + } +} diff --git a/src/bus_constants/71.json b/src/bus_constants/71.json new file mode 100644 index 000000000..e8064c5bc --- /dev/null +++ b/src/bus_constants/71.json @@ -0,0 +1,125 @@ +{ + "71": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Aberdeen & Mount Auburn", + "branches": null, + "station": "abmta", + "order": 1, + "stops": { + "1": [ + "71-1-2098" + ], + "0": [ + "71-0-2077" + ] + } + }, + { + "stop_name": "Watertown Square", + "branches": null, + "station": "wtrsq", + "order": 1, + "stops": { + "1": [ + "71-1-8178" + ], + "0": [ + "71-0-8178" + ] + } + }, + { + "stop_name": "Watertown High School", + "branches": null, + "station": "wtrhs", + "order": 2, + "stops": { + "1": [ + "71-1-2050" + ], + "0": [ + "71-0-2043" + ] + } + }, + { + "stop_name": "Mount Auburn Bridge", + "branches": null, + "station": "mtaub", + "order": 3, + "stops": { + "1": [ + "71-1-2064" + ], + "0": [ + "71-0-2030" + ] + } + }, + { + "stop_name": "Mount Auburn Hospital", + "branches": null, + "station": "mthsp", + "order": 4, + "stops": { + "1": [ + "71-1-2070" + ], + "0": [ + "71-0-2025" + ] + } + }, + { + "stop_name": "Mount Auburn @ Story Street", + "branches": null, + "station": "mtsty", + "order": 5, + "stops": { + "1": [ + "71-1-2074" + ], + "0": [ + "71-0-2020" + ] + } + }, + { + "stop_name": "Harvard Bus Tunnels", + "branches": null, + "station": "bally", + "order": 6, + "stops": { + "1": [ + "71-1-20761", + "71-1-32549" + ], + "0": [ + "71-0-2076", + "71-0-20761" + ] + } + }, + { + "stop_name": "Cambridge Common Loop", + "branches": null, + "station": "hloop", + "order": 7, + "stops": { + "1": [ + "71-1-12614" + ], + "0": [ + "71-0-12614" + ] + } + } + ] + } +} diff --git a/src/bus_constants/73.json b/src/bus_constants/73.json new file mode 100644 index 000000000..c21219464 --- /dev/null +++ b/src/bus_constants/73.json @@ -0,0 +1,111 @@ +{ + "73": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Waverley Square", + "branches": null, + "station": "wavsq", + "order": 1, + "stops": { + "1": [ + "73-1-2134" + ], + "0": [ + "73-0-2134" + ] + } + }, + { + "stop_name": "Benton Square", + "branches": null, + "station": "bntns", + "order": 2, + "stops": { + "1": [ + "73-1-2108" + ], + "0": [ + "73-0-2125" + ] + } + }, + { + "stop_name": "Mount Auburn Bridge", + "branches": null, + "station": "mtaub", + "order": 3, + "stops": { + "1": [ + "73-1-2064", + "73-1-2117" + ], + "0": [ + "73-0-2030" + ] + } + }, + { + "stop_name": "Mount Auburn Hospital", + "branches": null, + "station": "mthsp", + "order": 4, + "stops": { + "1": [ + "73-1-2070" + ], + "0": [ + "73-0-2025" + ] + } + }, + { + "stop_name": "Mount Auburn @ Story Street", + "branches": null, + "station": "mtsty", + "order": 5, + "stops": { + "1": [ + "73-1-2074" + ], + "0": [ + "73-0-2020" + ] + } + }, + { + "stop_name": "Harvard Bus Tunnels", + "branches": null, + "station": "bally", + "order": 6, + "stops": { + "1": [ + "73-1-20761" + ], + "0": [ + "73-0-2076", + "73-0-20761" + ] + } + }, + { + "stop_name": "Cambridge Common Loop", + "branches": null, + "station": "hloop", + "order": 7, + "stops": { + "1": [ + "73-1-12614" + ], + "0": [ + "73-0-12614" + ] + } + } + ] + } +} diff --git a/src/bus_constants/77.json b/src/bus_constants/77.json new file mode 100644 index 000000000..819a09ddd --- /dev/null +++ b/src/bus_constants/77.json @@ -0,0 +1,142 @@ +{ + "77": { + "type": "bus", + "direction": { + "0": "outbound", + "1": "inbound" + }, + "stations": [ + { + "stop_name": "Arlington Heights", + "branches": null, + "station": "arlht", + "order": 1, + "stops": { + "1": [ + "77-1-7922" + ], + "0": [ + "77-0-7922" + ] + } + }, + { + "stop_name": "Appleton Street & Massachusetts Avenue", + "branches": null, + "station": "appma", + "order": 2, + "stops": { + "1": [ + "77-1-2251" + ], + "0": [ + "77-0-2291" + ] + } + }, + { + "stop_name": "Arlington Center", + "branches": null, + "station": "arlct", + "order": 3, + "stops": { + "1": [ + "77-1-2258" + ], + "0": [ + "77-0-2282" + ] + } + }, + { + "stop_name": "Massachusetts Avenue @ Lake Street", + "branches": null, + "station": "make", + "order": 4, + "stops": { + "1": [ + "77-1-2265" + ], + "0": [ + "77-0-2277" + ] + } + }, + { + "stop_name": "Alewife Brook Parkway", + "branches": null, + "station": "alwpk", + "order": 5, + "stops": { + "1": [ + "77-1-22671" + ], + "0": [ + "77-0-22751" + ] + } + }, + { + "stop_name": "North Cambridge Station", + "branches": null, + "station": "ncsta", + "order": 6, + "stops": { + "1": [ + "77-1-12295", + "77-1-2296" + ], + "0": [ + "77-0-12295", + "77-0-2271", + "77-0-2320" + ] + } + }, + { + "stop_name": "Porter Square", + "branches": null, + "station": "portr", + "order": 7, + "stops": { + "1": [ + "77-1-12301" + ], + "0": [ + "77-0-23151" + ] + } + }, + { + "stop_name": "Cambridge Common Loop", + "branches": null, + "station": "hloop", + "order": 8, + "stops": { + "1": [ + "77-1-2307" + ], + "0": [ + "77-0-2310" + ] + } + }, + { + "stop_name": "Harvard Bus Tunnels", + "branches": null, + "station": "bally", + "order": 9, + "stops": { + "1": [ + "77-1-2076", + "77-1-32549" + ], + "0": [ + "77-0-20761", + "77-0-20762" + ] + } + } + ] + } +} diff --git a/src/stations.js b/src/stations.js index 9a42131a8..5ee85a24a 100644 --- a/src/stations.js +++ b/src/stations.js @@ -1,10 +1,17 @@ import { stations as rt_stations } from './constants'; import bus_1 from './bus_constants/1.json'; +import bus_15 from './bus_constants/15.json'; +import bus_22 from './bus_constants/22.json'; import bus_23 from './bus_constants/23.json'; +import bus_32 from './bus_constants/32.json'; import bus_28 from './bus_constants/28.json'; +import bus_39 from './bus_constants/39.json'; import bus_57 from './bus_constants/57.json'; import bus_66 from './bus_constants/66.json'; +import bus_71 from './bus_constants/71.json'; +import bus_73 from './bus_constants/73.json'; +import bus_77 from './bus_constants/77.json'; import bus_111 from './bus_constants/111.json'; import bus_114_116_117 from './bus_constants/114-116-117.json' From dc302a8d5d8a3d4cc0a08b2d3bd7da0712b1477d Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 20:24:07 -0500 Subject: [PATCH 092/152] Shorter names for many stations to improve display, especially on mobile. --- src/bus_constants/15.json | 4 ++-- src/bus_constants/22.json | 2 +- src/bus_constants/23.json | 6 +++--- src/bus_constants/28.json | 10 +++++----- src/bus_constants/32.json | 4 ++-- src/bus_constants/39.json | 8 ++++---- src/bus_constants/57.json | 2 +- src/bus_constants/66.json | 2 +- src/bus_constants/71.json | 2 +- src/bus_constants/73.json | 2 +- src/bus_constants/77.json | 4 ++-- 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/bus_constants/15.json b/src/bus_constants/15.json index bc1b0079f..29651c87e 100644 --- a/src/bus_constants/15.json +++ b/src/bus_constants/15.json @@ -36,7 +36,7 @@ } }, { - "stop_name": "Geneva Avenue @ Bowdoin Street", + "stop_name": "Geneva Ave @ Bowdoin St", "branches": null, "station": "genbo", "order": 2, @@ -78,7 +78,7 @@ } }, { - "stop_name": "Blue Hill Avenue @ Dudley Street", + "stop_name": "Blue Hill Ave @ Dudley", "branches": null, "station": "bhdud", "order": 5, diff --git a/src/bus_constants/22.json b/src/bus_constants/22.json index d559da09e..53732c26e 100644 --- a/src/bus_constants/22.json +++ b/src/bus_constants/22.json @@ -35,7 +35,7 @@ } }, { - "stop_name": "Blue Hill Avenue @ Talbot Avenue", + "stop_name": "Blue Hill Ave @ Talbot Ave", "branches": null, "station": "bltal", "order": 3, diff --git a/src/bus_constants/23.json b/src/bus_constants/23.json index 9b4554106..1ced30363 100644 --- a/src/bus_constants/23.json +++ b/src/bus_constants/23.json @@ -49,7 +49,7 @@ } }, { - "stop_name": "Washington Street & Columbia Road", + "stop_name": "Washington St & Columbia Rd", "branches": null, "station": "wacol", "order": 4, @@ -77,7 +77,7 @@ } }, { - "stop_name": "Warren Street @ Boston Latin Academy", + "stop_name": "Warren St @ Boston Latin", "branches": null, "station": "latac", "order": 6, @@ -92,7 +92,7 @@ } }, { - "stop_name": "Warren Street @ Walnut Avenue", + "stop_name": "Warren St @ Walnut Ave", "branches": null, "station": "warwl", "order": 7, diff --git a/src/bus_constants/28.json b/src/bus_constants/28.json index e792b4831..e177c875b 100644 --- a/src/bus_constants/28.json +++ b/src/bus_constants/28.json @@ -35,7 +35,7 @@ } }, { - "stop_name": "Morton Street @ Blue Hill Avenue", + "stop_name": "Morton St @ Blue Hill Ave", "branches": null, "station": "mortn", "order": 3, @@ -49,7 +49,7 @@ } }, { - "stop_name": "Blue Hill Avenue @ Talbot Avenue", + "stop_name": "Blue Hill Ave @ Talbot Ave", "branches": null, "station": "bltal", "order": 4, @@ -91,7 +91,7 @@ } }, { - "stop_name": "Warren Street @ Boston Latin Academy", + "stop_name": "Warren St @ Boston Latin", "branches": null, "station": "latac", "order": 7, @@ -106,7 +106,7 @@ } }, { - "stop_name": "Warren Street @ Walnut Avenue", + "stop_name": "Warren St @ Walnut Ave", "branches": null, "station": "warwl", "order": 8, @@ -134,7 +134,7 @@ } }, { - "stop_name": "Roxbury Crossing Station (Bus)", + "stop_name": "Roxbury Crossing Station", "branches": null, "station": "roxbs", "order": 10, diff --git a/src/bus_constants/32.json b/src/bus_constants/32.json index c70c125bc..1ce6e3bf6 100644 --- a/src/bus_constants/32.json +++ b/src/bus_constants/32.json @@ -50,7 +50,7 @@ } }, { - "stop_name": "Hyde Park Avenue @ American Legion Highway", + "stop_name": "Hyde Park Ave @ American Legion", "branches": null, "station": "hpaml", "order": 4, @@ -64,7 +64,7 @@ } }, { - "stop_name": "Hyde Park Avenue @ Cummins Highway", + "stop_name": "Hyde Park Ave @ Cummins Highway", "branches": null, "station": "hpave", "order": 5, diff --git a/src/bus_constants/39.json b/src/bus_constants/39.json index 01b6afdbc..e8036a63e 100644 --- a/src/bus_constants/39.json +++ b/src/bus_constants/39.json @@ -21,7 +21,7 @@ } }, { - "stop_name": "Jamaica Plain Center (Monument)", + "stop_name": "Jamaica Plain Center", "branches": null, "station": "jpctr", "order": 2, @@ -35,7 +35,7 @@ } }, { - "stop_name": "Centre Street & South Huntington Avenue", + "stop_name": "Centre St & South Huntington Ave", "branches": null, "station": "shunt", "order": 3, @@ -78,7 +78,7 @@ } }, { - "stop_name": "Huntington Avenue & Longwood Avenue", + "stop_name": "Huntington Ave & Longwood Ave", "branches": null, "station": "hlong", "order": 6, @@ -106,7 +106,7 @@ } }, { - "stop_name": "Huntington Avenue & Belvidere Street", + "stop_name": "Huntington Ave & Belvidere St", "branches": null, "station": "hunbv", "order": 8, diff --git a/src/bus_constants/57.json b/src/bus_constants/57.json index 98a509ac4..9b7c5b322 100644 --- a/src/bus_constants/57.json +++ b/src/bus_constants/57.json @@ -94,7 +94,7 @@ } }, { - "stop_name": "Commonwealth Avenue @ BU Bridge", + "stop_name": "Commonwealth Ave @ BU Bridge", "branches": null, "station": "bubrg", "order": 7, diff --git a/src/bus_constants/66.json b/src/bus_constants/66.json index 386edb4be..3df4fd73c 100644 --- a/src/bus_constants/66.json +++ b/src/bus_constants/66.json @@ -51,7 +51,7 @@ } }, { - "stop_name": "Harvard Avenue & Commonwealth Avenue", + "stop_name": "Harvard Ave & Commonwealth Ave", "branches": null, "station": "hvdcm", "order": 4, diff --git a/src/bus_constants/71.json b/src/bus_constants/71.json index e8064c5bc..a807096d6 100644 --- a/src/bus_constants/71.json +++ b/src/bus_constants/71.json @@ -77,7 +77,7 @@ } }, { - "stop_name": "Mount Auburn @ Story Street", + "stop_name": "Mount Auburn @ Story St", "branches": null, "station": "mtsty", "order": 5, diff --git a/src/bus_constants/73.json b/src/bus_constants/73.json index c21219464..89201c1d4 100644 --- a/src/bus_constants/73.json +++ b/src/bus_constants/73.json @@ -64,7 +64,7 @@ } }, { - "stop_name": "Mount Auburn @ Story Street", + "stop_name": "Mount Auburn @ Story St", "branches": null, "station": "mtsty", "order": 5, diff --git a/src/bus_constants/77.json b/src/bus_constants/77.json index 819a09ddd..55f016045 100644 --- a/src/bus_constants/77.json +++ b/src/bus_constants/77.json @@ -21,7 +21,7 @@ } }, { - "stop_name": "Appleton Street & Massachusetts Avenue", + "stop_name": "Appleton St & Mass Ave", "branches": null, "station": "appma", "order": 2, @@ -49,7 +49,7 @@ } }, { - "stop_name": "Massachusetts Avenue @ Lake Street", + "stop_name": "Mass Ave @ Lake St", "branches": null, "station": "make", "order": 4, From 8065b8316e370e094b695337671599954c69532f Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 19:29:17 -0500 Subject: [PATCH 093/152] Add column remapping to support prior years. --- server/bus/bus2train.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 5bb5685f2..98962d244 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -25,7 +25,28 @@ def load_data(input_csv, routes): """ # thinking about doing this in pandas to have all the info at once - df = pd.read_csv(input_csv, parse_dates=["service_date", "scheduled", "actual"]) + df = pd.read_csv(input_csv) + df.rename(columns={ + # This set of transformations covers prior-year bus data. + 'ServiceDate': 'service_date', + 'Route': 'route_id', + 'Direction': 'direction_id', + 'HalfTripId': 'half_trip_id', + 'Stop': 'stop_id', + 'stop_name': 'time_point_id', + 'stop_sequence': 'time_point_order', + 'Timepoint': 'time_point_id', + 'TimepointOrder': 'time_point_order', + 'PointType': 'point_type', + 'StandardType': 'standard_type', + 'Scheduled': 'scheduled', + 'Actual': 'actual', + 'ScheduledHeadway': 'scheduled_headway', + 'Headway': 'headway', + # 2020 data uses a different column name for direction. + 'direction': 'direction_id' + }, + inplace=True) # We need to keep both "Headway" AND "Schedule": both can have timepoint data. df = df.loc[df.actual.notnull()] From 8a3a5d544cd8274036c46ef09c5bec54bd3ae564 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 20:24:34 -0500 Subject: [PATCH 094/152] Remove outdated comment. --- server/bus/bus2train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 98962d244..b4c660c3f 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -43,7 +43,6 @@ def load_data(input_csv, routes): 'Actual': 'actual', 'ScheduledHeadway': 'scheduled_headway', 'Headway': 'headway', - # 2020 data uses a different column name for direction. 'direction': 'direction_id' }, inplace=True) From f381142d733813e7f7eb1ca2615d41bc01974ed9 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 21:34:13 -0500 Subject: [PATCH 095/152] Tool to fetch and extract 4 years of bus data. (This is designed to help get to a starting point on data; it should not need to be run regularly.) --- server/bus/setup_bus_input.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 server/bus/setup_bus_input.sh diff --git a/server/bus/setup_bus_input.sh b/server/bus/setup_bus_input.sh new file mode 100755 index 000000000..11f8853ab --- /dev/null +++ b/server/bus/setup_bus_input.sh @@ -0,0 +1,25 @@ +#!/bin/sh -x + +mkdir -p data/input + +#wget -O data/input/2021.zip https://www.arcgis.com/sharing/rest/content/items/2d415555f63b431597721151a7e07a3e/data +#wget -O data/input/2020.zip https://www.arcgis.com/sharing/rest/content/items/4c1293151c6c4a069d49e6b85ee68ea4/data +#wget -O data/input/2019.zip https://www.arcgis.com/sharing/rest/content/items/1bd340b39942438685d8dcdfe3f26d1a/data +#wget -O data/input/2018.zip https://www.arcgis.com/sharing/rest/content/items/d685ba39d9a54d908f49a2a762a9eb47/data + +cd data/input +for i in 2021 2020 2019 2018; do + unzip -d $i $i.zip +done +mv 2021/MBTA*/*.csv 2021/ + +mv "2020/Bus Arrival Departure Times Jan-Mar 2020.csv" "2020/2020-01-03.csv" +mv "2020/Bus Arrival Departure Times Apr-June 2020.csv" "2020/2020-04-06.csv" +mv "2020/Bus Arrival Departure Times Jul-Sep 2020.csv" "2020/2020-07-09.csv" +mv "2020/Bus Arrival Departure Times Oct-Dec 2020.csv" "2020/2020-10-12.csv" +mv "2019/MBTA Bus Arrival Departure Jan-Mar 2019.csv" "2019/2019-01-03.csv" +mv "2019/MBTA Bus Arrival Departure Apr-June 2019.csv" "2019/2019-04-06.csv" +mv "2019/MBTA Bus Arrival Departure Jul-Sept 2019.csv" "2019/2019-07-09.csv" +mv "2019/MBTA Bus Arrival Departure Oct-Dec 2019.csv" "2019/2019-10-12.csv" +mv "2018/MBTA Bus Arrival Departure Aug-Sept 2018.csv" "2018/2018-08-09.csv" +mv "2018/MBTA Bus Arrival Departure Oct-Dec 2018.csv" "2018/2018-10-12.csv" From 73d2f9c2a948fc06ab321aaced078d037d34f633 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 21:49:55 -0500 Subject: [PATCH 096/152] Add back in timestamp parsing. --- server/bus/bus2train.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index b4c660c3f..415b5121d 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -53,6 +53,11 @@ def load_data(input_csv, routes): df.route_id = df.route_id.str.lstrip("0") if routes: df = df.loc[df.route_id.isin(routes)] + + # Convert dates + df.scheduled = pd.to_datetime(df.scheduled) + df.service_date = pd.to_datetime(df.service_date) + df.actual = pd.to_datetime(df.actual) OFFSET = datetime(1900, 1, 1, 0, 0, 0) df.scheduled = df.service_date + (df.scheduled - OFFSET) From 59ba2185603758603b95f772c9bcaefe52e03cc2 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 21:59:53 -0500 Subject: [PATCH 097/152] Move gen_bus_data.sh to use the multi-year data setup by setup_bus_input. --- server/bus/gen_bus_data.sh | 7 +++++-- server/bus/setup_bus_input.sh | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/bus/gen_bus_data.sh b/server/bus/gen_bus_data.sh index caac11c81..f967a4cf4 100755 --- a/server/bus/gen_bus_data.sh +++ b/server/bus/gen_bus_data.sh @@ -1,5 +1,8 @@ #!/bin/bash -for f in $(find data/input/MBTA_Bus_Arrival_Departure_Times_2021/ -name *.csv); do - pipenv run python bus2train.py $f data/actual -r 111 # 23 # 57 # 1 28 66 114 116 117 +for y in 2018 2019 2020 2021; do + for f in $(find data/input/$y/ -name '*.csv'); do + echo "Generating stop data from $f" + pipenv run python bus2train.py $f data/output -r 1 + done done diff --git a/server/bus/setup_bus_input.sh b/server/bus/setup_bus_input.sh index 11f8853ab..3bbca741a 100755 --- a/server/bus/setup_bus_input.sh +++ b/server/bus/setup_bus_input.sh @@ -23,3 +23,4 @@ mv "2019/MBTA Bus Arrival Departure Jul-Sept 2019.csv" "2019/2019-07-09.csv" mv "2019/MBTA Bus Arrival Departure Oct-Dec 2019.csv" "2019/2019-10-12.csv" mv "2018/MBTA Bus Arrival Departure Aug-Sept 2018.csv" "2018/2018-08-09.csv" mv "2018/MBTA Bus Arrival Departure Oct-Dec 2018.csv" "2018/2018-10-12.csv" +sed -i -e 's///' 2020/2020-07-09.csv From cb5d21ab69aa1e5df7a6c145d44585e39d7a86af Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 22:06:37 -0500 Subject: [PATCH 098/152] Uncomment wget lines and add GTFS feed. --- server/bus/setup_bus_input.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/bus/setup_bus_input.sh b/server/bus/setup_bus_input.sh index 3bbca741a..3a209f7b0 100755 --- a/server/bus/setup_bus_input.sh +++ b/server/bus/setup_bus_input.sh @@ -2,10 +2,13 @@ mkdir -p data/input -#wget -O data/input/2021.zip https://www.arcgis.com/sharing/rest/content/items/2d415555f63b431597721151a7e07a3e/data -#wget -O data/input/2020.zip https://www.arcgis.com/sharing/rest/content/items/4c1293151c6c4a069d49e6b85ee68ea4/data -#wget -O data/input/2019.zip https://www.arcgis.com/sharing/rest/content/items/1bd340b39942438685d8dcdfe3f26d1a/data -#wget -O data/input/2018.zip https://www.arcgis.com/sharing/rest/content/items/d685ba39d9a54d908f49a2a762a9eb47/data +wget -O data/input/2021.zip https://www.arcgis.com/sharing/rest/content/items/2d415555f63b431597721151a7e07a3e/data +wget -O data/input/2020.zip https://www.arcgis.com/sharing/rest/content/items/4c1293151c6c4a069d49e6b85ee68ea4/data +wget -O data/input/2019.zip https://www.arcgis.com/sharing/rest/content/items/1bd340b39942438685d8dcdfe3f26d1a/data +wget -O data/input/2018.zip https://www.arcgis.com/sharing/rest/content/items/d685ba39d9a54d908f49a2a762a9eb47/data + +wget -O data/input/gtfs.zip https://cdn.mbta.com/MBTA_GTFS.zip +unzip -d data/input/MBTA_GTFS/ data/input/gtfs.zip cd data/input for i in 2021 2020 2019 2018; do From 980537b0c3debc5269b0ace75fff416b59454a15 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 22:17:44 -0500 Subject: [PATCH 099/152] Adding tools for manifest checking. --- server/bus/check_latest_manifests.sh | 9 +++++++ server/bus/compare_manifest.py | 40 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100755 server/bus/check_latest_manifests.sh create mode 100644 server/bus/compare_manifest.py diff --git a/server/bus/check_latest_manifests.sh b/server/bus/check_latest_manifests.sh new file mode 100755 index 000000000..9f50b16a4 --- /dev/null +++ b/server/bus/check_latest_manifests.sh @@ -0,0 +1,9 @@ +#!/bin/sh -x + +newfile=$1 + +for i in 1 22 23 28 57; do + mkdir -p data/output/manifests/ + pipenv run python manifest.py $newfile data/output/manifests/$i.json --checkpoints data/input/MBTA_GTFS/checkpoints.txt -r $i + pipenv run python compare_manifest.py ../../src/bus_contstants/$i.json data/output/manifests/$i.json +done diff --git a/server/bus/compare_manifest.py b/server/bus/compare_manifest.py new file mode 100644 index 000000000..7b4a6d13c --- /dev/null +++ b/server/bus/compare_manifest.py @@ -0,0 +1,40 @@ +import json +import sys + +station_stops = {} + +def runone(path, first=False): + unchanged = True + current = json.load(open(path)) + for i in list(current.values())[0]['stations']: + s = i['station'] + + if not s in station_stops: + station_stops[s] = {} + if not first: + print("Found missing station %s" % s) + unchanged = False + for direction in i['stops']: + if not direction in station_stops[s]: + station_stops[s][direction] = [] + for stop in i['stops'][direction]: + if not stop in station_stops[s][direction]: + station_stops[s][direction].append(stop) + if not first: + print("Found additional stop %s at station %s in %s" % (stop, s, path)) + unchanged = False + return unchanged + +def run(paths): + unchanged = True + runone(paths[-1], first=True) + for path in reversed(paths[1:]): + unchanged = unchanged and runone(path) + if unchanged == True: + print("No new stations/stops on route.") + else: + print("Changed?") + + +if __name__ == "__main__": + run(sys.argv[1:]) From 062f43b48e5620105a7e658f57b51229da783a4f Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 22:24:47 -0500 Subject: [PATCH 100/152] Minor fixes. --- server/bus/check_latest_manifests.sh | 7 ++++--- server/bus/compare_manifest.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/bus/check_latest_manifests.sh b/server/bus/check_latest_manifests.sh index 9f50b16a4..5595fccf9 100755 --- a/server/bus/check_latest_manifests.sh +++ b/server/bus/check_latest_manifests.sh @@ -1,9 +1,10 @@ -#!/bin/sh -x +#!/bin/sh newfile=$1 -for i in 1 22 23 28 57; do +for i in 1 111 15 22 23 28 32 39 57 66 71 73 77; do mkdir -p data/output/manifests/ pipenv run python manifest.py $newfile data/output/manifests/$i.json --checkpoints data/input/MBTA_GTFS/checkpoints.txt -r $i - pipenv run python compare_manifest.py ../../src/bus_contstants/$i.json data/output/manifests/$i.json + echo "Comparing old and new manifests for route $i" + pipenv run python compare_manifest.py ../../src/bus_constants/$i.json data/output/manifests/$i.json done diff --git a/server/bus/compare_manifest.py b/server/bus/compare_manifest.py index 7b4a6d13c..c2b9f4dae 100644 --- a/server/bus/compare_manifest.py +++ b/server/bus/compare_manifest.py @@ -27,7 +27,7 @@ def runone(path, first=False): def run(paths): unchanged = True - runone(paths[-1], first=True) + runone(paths[0], first=True) for path in reversed(paths[1:]): unchanged = unchanged and runone(path) if unchanged == True: From 83551b9509b8ea97a5dd4a0cf1fc66690bd98f5b Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 14 Dec 2021 23:10:01 -0500 Subject: [PATCH 101/152] Lint fixes. --- server/bus/bus2train.py | 5 ++--- server/bus/compare_manifest.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/server/bus/bus2train.py b/server/bus/bus2train.py index 415b5121d..134b79a49 100644 --- a/server/bus/bus2train.py +++ b/server/bus/bus2train.py @@ -44,8 +44,7 @@ def load_data(input_csv, routes): 'ScheduledHeadway': 'scheduled_headway', 'Headway': 'headway', 'direction': 'direction_id' - }, - inplace=True) + }, inplace=True) # We need to keep both "Headway" AND "Schedule": both can have timepoint data. df = df.loc[df.actual.notnull()] @@ -53,7 +52,7 @@ def load_data(input_csv, routes): df.route_id = df.route_id.str.lstrip("0") if routes: df = df.loc[df.route_id.isin(routes)] - + # Convert dates df.scheduled = pd.to_datetime(df.scheduled) df.service_date = pd.to_datetime(df.service_date) diff --git a/server/bus/compare_manifest.py b/server/bus/compare_manifest.py index c2b9f4dae..741cfcaa8 100644 --- a/server/bus/compare_manifest.py +++ b/server/bus/compare_manifest.py @@ -3,38 +3,40 @@ station_stops = {} + def runone(path, first=False): unchanged = True current = json.load(open(path)) for i in list(current.values())[0]['stations']: s = i['station'] - - if not s in station_stops: + + if s not in station_stops: station_stops[s] = {} if not first: print("Found missing station %s" % s) unchanged = False for direction in i['stops']: - if not direction in station_stops[s]: + if direction not in station_stops[s]: station_stops[s][direction] = [] for stop in i['stops'][direction]: - if not stop in station_stops[s][direction]: + if stop not in station_stops[s][direction]: station_stops[s][direction].append(stop) if not first: print("Found additional stop %s at station %s in %s" % (stop, s, path)) unchanged = False return unchanged + def run(paths): unchanged = True runone(paths[0], first=True) for path in reversed(paths[1:]): unchanged = unchanged and runone(path) - if unchanged == True: + if unchanged: print("No new stations/stops on route.") else: print("Changed?") - + if __name__ == "__main__": run(sys.argv[1:]) From 38ae7b332818e6a0efc481b06b201444d5c86163 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 15 Dec 2021 17:11:28 -0500 Subject: [PATCH 102/152] adjust chart title margin --- src/Title.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Title.js b/src/Title.js index cdc0a4fbb..c9433471b 100644 --- a/src/Title.js +++ b/src/Title.js @@ -24,7 +24,7 @@ const drawTitle = (title, location, bothStops, chart) => { ctx.save(); const leftMargin = 50; - const rightMargin = 15; + const rightMargin = 7; const minGap = 10; const vpos_row1 = 25; const vpos_row2 = 50; From 05b62368da1fc993ff79a496c96d5a50809d70e0 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 15 Dec 2021 17:11:40 -0500 Subject: [PATCH 103/152] don't show legend when not being used --- src/line.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/line.js b/src/line.js index 03db35ed8..978cd7b15 100644 --- a/src/line.js +++ b/src/line.js @@ -178,7 +178,7 @@ class SingleDayLine extends React.Component { />
- {this.props.benchmarkField && } + {this.props.useBenchmarks && }
); From 695f83b37ee11222766dee1f26a2cfb1fb089cc1 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 15 Dec 2021 17:51:20 -0500 Subject: [PATCH 104/152] toggle css improvements --- src/ui/toggle.css | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/ui/toggle.css b/src/ui/toggle.css index d49acd6db..ce1d823d7 100644 --- a/src/ui/toggle.css +++ b/src/ui/toggle.css @@ -1,8 +1,12 @@ +:root { + --toggle-timing: .1s; +} + .switch { position: relative; display: inline-block; - width: 40px; - height: 20px; + width: 42px; + height: 22px; flex-shrink: 0; } @@ -19,24 +23,23 @@ left: 0; right: 0; bottom: 0; - border-radius: 10px; - outline: none; - /* outline: 1px solid; */ + border-radius: 11px; + border: 1px solid; + width: 40px; + height: 20px; + box-sizing: content-box; background-color: #fff; - -webkit-transition: .4s; - transition: .4s; + -webkit-transition: var(--toggle-timing); + transition: var(--toggle-timing); } -/* Want toggle perimiter to show in bus mode - * Border changes size/shape of area - * Outline has square corners on some browsers - * so, using box-shadow instead */ .station-configuration .slider { - box-shadow: none; + border-style: hidden; } .station-configuration-wrapper.Bus .slider { - box-shadow: 0 0 0 1pt rgba(0, 0, 0, 0.4); + border-style: solid; + border-color: rgba(0, 0, 0, 0.4); } .slider:before { @@ -48,8 +51,8 @@ bottom: 4px; border-radius: 6px; background-color: black; - -webkit-transition: .4s; - transition: .4s; + -webkit-transition: var(--toggle-timing); + transition: var(--toggle-timing); } input:checked + .slider:before { @@ -58,6 +61,7 @@ input:checked + .slider:before { transform: translateX(18px); } -.switch input:focus + .slider:before { - background-color: skyblue; +.switch input:focus + .slider { + outline: 5px auto Highlight; + outline: 5px auto -webkit-focus-ring-color; } From c0f25aee39258deb068985064a7fb8830a86285a Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 15 Dec 2021 18:18:12 -0500 Subject: [PATCH 105/152] Add disclaimer about bus data --- src/App.css | 5 +++++ src/ChartSets.js | 2 ++ src/notes.js | 13 +++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 src/notes.js diff --git a/src/App.css b/src/App.css index 2e8cc5325..fcac3c54f 100644 --- a/src/App.css +++ b/src/App.css @@ -504,3 +504,8 @@ button:disabled, .control input[type=radio] { margin-right: 10px; } + +.bus-disclaimer { + font-style: italic; + padding-top: 30px; +} \ No newline at end of file diff --git a/src/ChartSets.js b/src/ChartSets.js index 1f14674f7..e1a0fc071 100644 --- a/src/ChartSets.js +++ b/src/ChartSets.js @@ -2,6 +2,7 @@ import React from 'react'; import { AggregateByTimeSelectable } from './SelectableCharts'; import { SingleDayLine, AggregateByDate } from './line'; import { station_direction } from './stations'; +import { BusDisclaimer } from './notes'; const dataFields = { traveltimes: { @@ -126,6 +127,7 @@ const SingleDaySet = (props) => { date={props.startDate} /> } + {props.bus_mode && }
) } diff --git a/src/notes.js b/src/notes.js new file mode 100644 index 000000000..ad158ac74 --- /dev/null +++ b/src/notes.js @@ -0,0 +1,13 @@ +import React from "react"; + +const BusDisclaimer = () => { + return( +
+ Due to data collection issues, bus data is not guaranteed to be complete for any stop or date. +
+ This may lead to inaccuracies, particularly in headway calculations. +
+ ) +}; + +export { BusDisclaimer }; \ No newline at end of file From f8e964ff6d63fd208ff042c72010e886b82c403b Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 15 Dec 2021 23:53:48 -0500 Subject: [PATCH 106/152] review nits --- src/AlertBar.js | 4 ++-- src/date.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/AlertBar.js b/src/AlertBar.js index d922342d5..a08806b37 100644 --- a/src/AlertBar.js +++ b/src/AlertBar.js @@ -9,10 +9,10 @@ function chartTimeframe(start_date) { // Set alert-bar interval to be 5:30am today to 1am tomorrow. const today = `${start_date}T00:00:00`; - let low = new Date(today); + const low = new Date(today); low.setHours(5, 30); - let high = new Date(today); + const high = new Date(today); high.setDate(high.getDate() + 1); high.setHours(1,0); diff --git a/src/date.js b/src/date.js index eed0f60d6..a869728c6 100644 --- a/src/date.js +++ b/src/date.js @@ -11,7 +11,7 @@ const RegularDateInput = (props) => { if (maxDate === "today") { const iso_date = new Date(); const offset = iso_date.getTimezoneOffset(); - const local_date = new Date(iso_date - (offset * 60 * 1000)); + const local_date = new Date(iso_date.valueOf() - (offset * 60 * 1000)); maxDate = local_date.toISOString().split("T")[0]; } From 22232e9e1b7ada5f47d4d5d11adb18735d94fe22 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 16 Dec 2021 12:28:37 -0500 Subject: [PATCH 107/152] fix back button navigation --- src/App.js | 62 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/src/App.js b/src/App.js index 059701748..4282c6af7 100644 --- a/src/App.js +++ b/src/App.js @@ -21,7 +21,7 @@ const APP_DATA_BASE_PATH = FRONTEND_TO_BACKEND_MAP.get(window.location.hostname) const MAX_AGGREGATION_MONTHS = 8; const RANGE_TOO_LARGE_ERROR = `Please select a range no larger than ${MAX_AGGREGATION_MONTHS} months.`; -const RANGE_NEGATIVE_ERROR = "Please choose an end date that comes after the selected start date."; +const RANGE_NEGATIVE_ERROR = "Oops, please ensure the start date comes before the selected end date."; const stateFromURL = (pathname, config) => { const bus_mode = (pathname === "/bus") @@ -38,6 +38,24 @@ const stateFromURL = (pathname, config) => { } }; +const urlFromState = (config) => { + const { bus_mode, line, from, to, date_start, date_end } = config; + const { fromStopIds, toStopIds } = get_stop_ids_for_stations(from, to); + const pathname = bus_mode ? "bus" : "rapidtransit" + const parts = [ + line, + fromStopIds?.[0], + toStopIds?.[0], + date_start, + date_end, + ]; + const partString = parts.map(x => x || "").join(","); + const url = `/${pathname}?config=${partString}`; + const isComplete = parts.slice(0, -1).every(x => x); + + return [url, isComplete]; +}; + const documentTitle = (config) => { return `${config.line} Line - ${config.date_start} - TransitMatters Data Dashboard`; }; @@ -100,9 +118,7 @@ class App extends React.Component { // Handle back/forward buttons window.onpopstate = (e) => { - this.setState({ - configuration: e.state.state, - }, () => { + this.setState(e.state.state, () => { this.download(); }); }; @@ -126,7 +142,7 @@ class App extends React.Component { const date_end_ts = new Date(date_end).getTime(); if (date_end_ts < date_start_ts) { return RANGE_NEGATIVE_ERROR; - } + } if (date_end_ts - date_start_ts > MAX_AGGREGATION_MONTHS * 31 * 86400 * 1000) { return RANGE_TOO_LARGE_ERROR; } @@ -134,6 +150,13 @@ class App extends React.Component { } updateConfiguration(config_change, refetch = true) { + // Save to browser history only if we are leaving a complete configuration + // so back button never takes you to a partial page + const [ url, isComplete ] = urlFromState(this.state.configuration); + if (isComplete) { + this.props.history.push(url, this.state); + } + let update = { error: null, configuration: { @@ -143,9 +166,9 @@ class App extends React.Component { }; const error = this.validateRange(update.configuration.date_start, update.configuration.date_end); - this.setState( - {error_message: error} - ); + this.setState({ + error_message: error + }); // Setting refetch to false prevents data download, but lets this.state.configuration update still if (error) { refetch = false; @@ -159,34 +182,13 @@ class App extends React.Component { update.dwells = []; } this.setState(update, () => { - this.stateToURL(); + this.props.history.replace(urlFromState(this.state.configuration)[0], this.state); if (refetch) { this.download(); } }); } - stateToURL() { - const { - bus_mode, - line, - from, - to, - date_start, - date_end, - } = this.state.configuration; - const { fromStopIds, toStopIds } = get_stop_ids_for_stations(from, to); - const pathname = bus_mode ? "bus" : "rapidtransit" - const parts = [ - line, - fromStopIds?.[0], - toStopIds?.[0], - date_start, - date_end, - ].map(x => x || "").join(","); - this.props.history.push(`/${pathname}?config=${parts}`, this.state.configuration); - } - fetchDataset(name, signal, options) { let url; // If a date_end is set, fetch aggregate data instead of single day From 9a0b56916fe0abd1595f15b7bcf5388860258042 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 16 Dec 2021 18:04:01 -0500 Subject: [PATCH 108/152] attemp to fix mobile date selection --- src/date.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/date.js b/src/date.js index a869728c6..3158ed041 100644 --- a/src/date.js +++ b/src/date.js @@ -4,7 +4,8 @@ import 'flatpickr/dist/themes/light.css'; const ua = window.navigator.userAgent; const iOSDevice = !!ua.match(/iPad/i) || !!ua.match(/iPhone/i); -const useFlatPickr = !iOSDevice; +const isMobile = /Android|webOS|iPhone|iPad|BlackBerry|IEMobile|Opera Mini/i.test(ua); +const useFlatPickr = !isMobile; const RegularDateInput = (props) => { let maxDate = props.options.maxDate; From 98deaa9e45b7e73416e87b4027434bc21bb1f7db Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 16 Dec 2021 18:14:20 -0500 Subject: [PATCH 109/152] fix non-flatpickr css --- src/App.css | 2 +- src/date.js | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/App.css b/src/App.css index fcac3c54f..140f1989a 100644 --- a/src/App.css +++ b/src/App.css @@ -156,7 +156,7 @@ } select, -.flatpickr-input, +.option-date input, button { border-radius: 4px; appearance: none; diff --git a/src/date.js b/src/date.js index 3158ed041..4cc769e3f 100644 --- a/src/date.js +++ b/src/date.js @@ -3,7 +3,6 @@ import flatpickr from 'flatpickr'; import 'flatpickr/dist/themes/light.css'; const ua = window.navigator.userAgent; -const iOSDevice = !!ua.match(/iPad/i) || !!ua.match(/iPhone/i); const isMobile = /Android|webOS|iPhone|iPad|BlackBerry|IEMobile|Opera Mini/i.test(ua); const useFlatPickr = !isMobile; From e6b0b6e6d13b32e91d695aa87ca35493baeb85d2 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 16 Dec 2021 21:16:37 -0500 Subject: [PATCH 110/152] move presets to allow for bus --- src/App.js | 2 +- src/constants.js | 43 ------------------------------------- src/presets.js | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ src/stations.js | 1 + 4 files changed, 57 insertions(+), 44 deletions(-) create mode 100644 src/presets.js diff --git a/src/App.js b/src/App.js index 4282c6af7..089cb49ed 100644 --- a/src/App.js +++ b/src/App.js @@ -9,7 +9,7 @@ import AlertBar from './AlertBar'; import ProgressBar from './ui/ProgressBar'; import './App.css'; import Select from './Select'; -import { configPresets } from './constants'; +import { configPresets } from './presets'; const FRONTEND_TO_BACKEND_MAP = new Map([ ["localhost", ""], // this becomes a relative path that is proxied through CRA:3000 to python on :5000 diff --git a/src/constants.js b/src/constants.js index 97fe35e11..cb976fbf8 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1158,46 +1158,3 @@ export const stations = { ] } }; - -const createConfigPresetValue = (line, fromStationName, toStationName, date_start, date_end = undefined) => { - const fromStation = stations[line].stations.find(s => s.stop_name === fromStationName); - const toStation = stations[line].stations.find(s => s.stop_name === toStationName); - const bus_mode = stations[line].type === "bus"; - return { - bus_mode, - line, - date_start, - date_end, - from: fromStation, - to: toStation, - } -}; - -const TODAY = new Date().toISOString().split("T")[0]; - -export const configPresets = [ - { - label: "[New!] April 2021 — Orange Line slow zone", - value: createConfigPresetValue("Orange", "Oak Grove", "Wellington", '2021-03-01', TODAY), - }, - { - label: "[New!] December 2020 — Orange Line slow zone", - value: createConfigPresetValue("Orange", "Community College", "North Station", '2020-11-01', '2021-02-17'), - }, - { - label: "[New!] Spring 2020 — Green Line (E-branch) COVID-19 pandemic effect", - value: createConfigPresetValue("Green", "Mission Park", "Government Center", '2020-01-01', '2020-05-31'), - }, - { - label: "March 30, 2021 — Red Line Power Issues", - value: createConfigPresetValue("Red", "Andrew", "Park Street", '2021-03-30'), - }, - { - label: "March 16, 2021 — Orange Line Derailment", - value: createConfigPresetValue("Orange", "Downtown Crossing", "Community College", '2021-03-16'), - }, - { - label: "April 2, 2021 — Blue Line Daytime Maintenance", - value: createConfigPresetValue("Blue", "Revere Beach", "State Street", '2021-04-02'), - }, -]; diff --git a/src/presets.js b/src/presets.js new file mode 100644 index 000000000..d4a01fceb --- /dev/null +++ b/src/presets.js @@ -0,0 +1,55 @@ +import { stations } from "./stations"; + + +const createConfigPresetValue = (line, fromStationName, toStationName, date_start, date_end = undefined) => { + const fromStation = stations[line].stations.find(s => s.stop_name === fromStationName); + const toStation = stations[line].stations.find(s => s.stop_name === toStationName); + const bus_mode = stations[line].type === "bus"; + return { + bus_mode, + line, + date_start, + date_end, + from: fromStation, + to: toStation, + } +}; + +const TODAY = new Date().toISOString().split("T")[0]; + +export const configPresets = [ + { + label: "June to December 2021 — Orange Line slow zones", + value: createConfigPresetValue("Orange", "Downtown Crossing", "Green Street", "2021-06-01", "2021-12-31") + }, + { + label: "September 9, 2021 — Route 28 first day of school traffic", + value: createConfigPresetValue("28", "Mattapan Station", "Nubian Station", "2021-09-09") + }, + /** OLD STUFF + { + label: "[New!] April 2021 — Orange Line slow zone", + value: createConfigPresetValue("Orange", "Oak Grove", "Wellington", '2021-03-01', TODAY), + }, + { + label: "[New!] December 2020 — Orange Line slow zone", + value: createConfigPresetValue("Orange", "Community College", "North Station", '2020-11-01', '2021-02-17'), + }, + { + label: "[New!] Spring 2020 — Green Line (E-branch) COVID-19 pandemic effect", + value: createConfigPresetValue("Green", "Mission Park", "Government Center", '2020-01-01', '2020-05-31'), + }, + { + label: "March 30, 2021 — Red Line Power Issues", + value: createConfigPresetValue("Red", "Andrew", "Park Street", '2021-03-30'), + }, + { + label: "March 16, 2021 — Orange Line Derailment", + value: createConfigPresetValue("Orange", "Downtown Crossing", "Community College", '2021-03-16'), + }, + { + label: "April 2, 2021 — Blue Line Daytime Maintenance", + value: createConfigPresetValue("Blue", "Revere Beach", "State Street", '2021-04-02'), + }, + */ +]; diff --git a/src/stations.js b/src/stations.js index 094c96908..5e5dc188b 100644 --- a/src/stations.js +++ b/src/stations.js @@ -64,6 +64,7 @@ const get_stop_ids_for_stations = (from, to) => { } export { + stations, all_lines, bus_lines, subway_lines, From 87f999a08bb95e92e9ec59c64bb1955c242e2023 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Thu, 16 Dec 2021 22:26:18 -0500 Subject: [PATCH 111/152] shrink title until it fits --- src/Title.js | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/src/Title.js b/src/Title.js index c9433471b..d5858280a 100644 --- a/src/Title.js +++ b/src/Title.js @@ -4,12 +4,15 @@ const getLineColor = (lineName) => colorsForLine[lineName] || 'black'; const titleColor = 'gray'; const parse_location_description = (location, bothStops) => { - let result = []; - + /** Example return values: + * [["Harvard", "red"], [" to ", "gray"], ["Park St", "red"]] + * or + * [["Harvard", "red"], [" southbound", "gray"]] + */ + const result = []; const lineColor = getLineColor(location['line']); result.push([location['from'], lineColor]); - if (bothStops) { result.push([' to ', titleColor]); result.push([location['to'], lineColor]); @@ -19,8 +22,20 @@ const parse_location_description = (location, bothStops) => { return result; }; +function font(size = 16) { + // This is the default from chartjs, but now we can adjust size. + return `bold ${size}px "Helvetica Neue", "Helvetica", "Arial", sans-serif`; +} + +const calcLocationWidth = (words, ctx) => { + // input: result of parse_location_description + // output depends on ctx's current font + return words.map(x => ctx.measureText(x[0]).width) + .reduce((a,b) => a + b, 0); +} + const drawTitle = (title, location, bothStops, chart) => { - let ctx = chart.chart.ctx; + const ctx = chart.chart.ctx; ctx.save(); const leftMargin = 50; @@ -28,13 +43,15 @@ const drawTitle = (title, location, bothStops, chart) => { const minGap = 10; const vpos_row1 = 25; const vpos_row2 = 50; + + let fontSize = 16; + ctx.font = font(fontSize); let position; - + + const locationDescr = parse_location_description(location, bothStops); const titleWidth = ctx.measureText(title).width; - const locationWidth = parse_location_description(location, bothStops) - .map(x => ctx.measureText(x[0]).width) - .reduce((a,b) => a + b, 0); + let locationWidth = calcLocationWidth(locationDescr, ctx); if ((leftMargin + titleWidth + minGap + locationWidth + rightMargin) > chart.chart.width) { // small screen: centered title stacks vertically @@ -42,10 +59,17 @@ const drawTitle = (title, location, bothStops, chart) => { ctx.fillStyle = titleColor; ctx.fillText(title, chart.chart.width/2, vpos_row1); + // Location info might be too long. Shrink the font until it fits. + while (locationWidth > chart.chart.width && fontSize >= 8) { + fontSize -= 1; + ctx.font = font(fontSize); + locationWidth = calcLocationWidth(locationDescr, ctx); + } + // we want centered, but have to write 1 word at a time bc colors, so... ctx.textAlign = 'left'; position = chart.chart.width/2 - locationWidth/2; - for (const [word, color] of parse_location_description(location, bothStops)) { + for (const [word, color] of locationDescr) { ctx.fillStyle = color; ctx.fillText(word, position, vpos_row2); position += ctx.measureText(word).width; @@ -53,7 +77,7 @@ const drawTitle = (title, location, bothStops, chart) => { } else { // larger screen - // Primary chart title goes left left + // Primary chart title goes left ctx.textAlign = 'left'; ctx.fillStyle = titleColor; ctx.fillText(title, leftMargin, vpos_row2); @@ -61,7 +85,7 @@ const drawTitle = (title, location, bothStops, chart) => { // location components are aligned right ctx.textAlign = 'right'; position = chart.chart.width - rightMargin; - for (const [word, color] of parse_location_description(location, bothStops).reverse()) { + for (const [word, color] of locationDescr.reverse()) { ctx.fillStyle = color; ctx.fillText(word, position, vpos_row2); position -= ctx.measureText(word).width; From 2c7e12ae21a42416dbfbbe873ef9341c0cf50aad Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Fri, 17 Dec 2021 18:57:35 -0500 Subject: [PATCH 112/152] daily charts show date --- src/line.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/line.js b/src/line.js index 978cd7b15..4ac23047b 100644 --- a/src/line.js +++ b/src/line.js @@ -153,7 +153,11 @@ class SingleDayLine extends React.Component { tooltipFormat: "LTS" // locale time with seconds }, scaleLabel: { - labelString: "Time of day", + labelString: ( + new Date(`${this.props.date}T00:00:00`) + .toLocaleDateString(undefined, { + weekday: 'long', year: 'numeric', month: 'long', day: 'numeric' + })) }, // make sure graph shows /at least/ 6am today to 1am tomorrow afterDataLimits: (axis) => { From cb49ffbbd8447c7b3aebcc783127386bcf413ea2 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Fri, 17 Dec 2021 14:27:02 -0500 Subject: [PATCH 113/152] encapsulate progress bar --- src/App.js | 77 +++++++++------------------------- src/presets.js | 2 +- src/ui/ProgressBar.css | 5 +-- src/ui/ProgressBar.js | 94 +++++++++++++++++++++++++++++++++++------- 4 files changed, 101 insertions(+), 77 deletions(-) diff --git a/src/App.js b/src/App.js index 089cb49ed..e8069917f 100644 --- a/src/App.js +++ b/src/App.js @@ -6,7 +6,7 @@ import { withRouter } from 'react-router-dom'; import { lookup_station_by_id, get_stop_ids_for_stations } from './stations'; import { recognize } from './AlertFilter'; import AlertBar from './AlertBar'; -import ProgressBar from './ui/ProgressBar'; +import { ProgressBar, progressBarRate } from './ui/ProgressBar'; import './App.css'; import Select from './Select'; import { configPresets } from './presets'; @@ -96,7 +96,7 @@ class App extends React.Component { alerts: [], datasetLoadingState: {}, - progress: 0, + progressBarKey: 0, }; ReactGA.initialize("UA-71173708-2"); @@ -126,7 +126,6 @@ class App extends React.Component { this.download = this.download.bind(this); this.updateConfiguration = this.updateConfiguration.bind(this); - this.progressTimer = null; this.fetchControllers = []; } @@ -181,12 +180,18 @@ class App extends React.Component { update.traveltimes = []; update.dwells = []; } - this.setState(update, () => { - this.props.history.replace(urlFromState(this.state.configuration)[0], this.state); - if (refetch) { - this.download(); - } - }); + this.setState((state, props) => ( + { + progressBarKey: state.progressBarKey + 1, + ...update, + }), + // callback once state is updated + () => { + this.props.history.replace(urlFromState(this.state.configuration)[0], this.state); + if (refetch) { + this.download(); + } + }); } fetchDataset(name, signal, options) { @@ -259,27 +264,6 @@ class App extends React.Component { return !!this.state.configuration.date_end; } - restartProgressBar() { - // Start the progress bar at 0 - this.setState({ - progress: 0, - }, () => { - this.progressTimer = setInterval(() => { - // Increment up until 95% - if(this.state.progress < 95) { - this.setState({ - progress: this.state.progress + this.progressBarRate(), - }); - } - else { - // Stop at 90%, the progress bar will be cleared when everything is actually done loading - clearInterval(this.progressTimer); - this.progressTimer = null; - } - }, 1000); - }); - } - download() { // End all existing fetches while(this.fetchControllers.length > 0) { @@ -289,19 +273,9 @@ class App extends React.Component { const controller = new AbortController(); this.fetchControllers.push(controller); - // Stop existing progress bar timer - if(this.progressTimer !== null) { - clearInterval(this.progressTimer); - this.progressTimer = null; - } - const { configuration } = this.state; const { fromStopIds, toStopIds } = get_stop_ids_for_stations(configuration.from, configuration.to); if (configuration.date_start && fromStopIds && toStopIds) { - if (configuration.date_end) { - this.restartProgressBar(); - } - this.fetchDataset('headways', controller.signal, { stop: fromStopIds, }); @@ -365,22 +339,6 @@ class App extends React.Component {
} - progressBarRate() { - // Single day: no rate - if(!this.isAggregation()) { - return null; - } - - // Aggregation: fake rate based on how many days - const {date_start, date_end} = this.state.configuration; - const ms = (new Date(date_end) - new Date(date_start)); - const days = ms / (1000*60*60*24); - const months = days / 30; - - const total_seconds_expected = 3.0 * months; - return 100 / total_seconds_expected; // % per second - } - renderCharts() { const propsToPass = { bus_mode: this.state.configuration.bus_mode, @@ -422,7 +380,12 @@ class App extends React.Component { isLoading={this.getIsLoadingDataset("alerts")} isHidden={hasNoLoadedCharts} />} - {canShowCharts && !this.getDoneLoading() && this.isAggregation() && } + {canShowCharts && this.isAggregation() && !this.getDoneLoading() && + + }
{!canShowCharts && this.renderEmptyState(error_message)} {canShowCharts && this.renderCharts()} diff --git a/src/presets.js b/src/presets.js index d4a01fceb..13873e880 100644 --- a/src/presets.js +++ b/src/presets.js @@ -20,7 +20,7 @@ const TODAY = new Date().toISOString().split("T")[0]; export const configPresets = [ { label: "June to December 2021 — Orange Line slow zones", - value: createConfigPresetValue("Orange", "Downtown Crossing", "Green Street", "2021-06-01", "2021-12-31") + value: createConfigPresetValue("Orange", "Downtown Crossing", "Green Street", "2021-06-01", TODAY) }, { label: "September 9, 2021 — Route 28 first day of school traffic", diff --git a/src/ui/ProgressBar.css b/src/ui/ProgressBar.css index d2cfb7f84..be21be289 100644 --- a/src/ui/ProgressBar.css +++ b/src/ui/ProgressBar.css @@ -3,9 +3,8 @@ position: absolute; background-color: gray; color: black; - border-radius: 10px; + border-radius: 0px 10px 10px 0px; width: 0%; height: 8px; transition: width 0.5s ease-in-out; - visibility: hidden;; -} \ No newline at end of file +} diff --git a/src/ui/ProgressBar.js b/src/ui/ProgressBar.js index f0ee8ba58..08723d7bb 100644 --- a/src/ui/ProgressBar.js +++ b/src/ui/ProgressBar.js @@ -1,19 +1,81 @@ import React from "react"; import './ProgressBar.css'; -const ProgressBar = (props) => { - const { progress } = props; - if (!(progress >= 0 && progress < 100)) { - return (null); - } - - return ( -
- ); -}; - -export default ProgressBar; +function progressBarRate(date_start, date_end) { + // Single day: no rate + if(!date_end) { + return null; + } + // Aggregation: fake rate based on how many days + const ms = (new Date(date_end) - new Date(date_start)); + const days = ms / (1000*60*60*24); + const months = days / 30; + const total_seconds_expected = 3.0 * months; + return 100 / total_seconds_expected; +} + + +class ProgressBar extends React.Component{ + /** + * This is now a completely uncontrolled component. + * Once mounted, it will run until 95% or it's unmounted. + * key is used by App.js to make React mount a new one each time. + * props: rate, key + */ + constructor(props) { + super(props); + this.timer = null; + this.state = { + progress: 0 + } + } + + componentDidMount() { + this.startTimer(); + } + + componentWillUnmount() { + this.stopTimer(); + } + + startTimer() { + // shouldn't be necessary to reset, but just in case + this.stopTimer(); + this.setState({ + progress: 0, + }); + this.timer = setInterval( + () => this.tick(), + 1000 + ); + } + + stopTimer() { + if (this.timer) { + clearInterval(this.timer); + } + this.timer = null; + } + + tick() { + this.setState((state, props) => { + const newProgress = state.progress + props.rate; + if (newProgress < 95) { + return { progress: newProgress }; + } else { + this.stopTimer(); + return { progress: 95 }; + } + }); + } + + render() { + return ( +
+ ) + } +} + +export { ProgressBar, progressBarRate }; From 02911c8cdd585edb30ee743df5ad8bfff1966ef4 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 18 Dec 2021 20:27:00 -0500 Subject: [PATCH 114/152] reorganize directory structure --- src/App.js | 6 +++--- src/ChartSets.js | 6 +++--- src/StationConfiguration.js | 7 ++++--- src/{ => alerts}/AlertBar.js | 0 src/{ => alerts}/AlertFilter.js | 0 src/{ => alerts}/LegendAlerts.js | 0 src/{ => charts}/Legend.js | 0 src/{ => charts}/SelectableCharts.js | 20 +------------------- src/{ => charts}/Title.js | 2 +- src/{ => charts}/line.js | 0 src/{ => inputs}/Select.js | 0 src/{ => inputs}/date.js | 0 src/inputs/radio.js | 22 ++++++++++++++++++++++ src/{ui => inputs}/toggle.css | 0 src/{ => ui}/notes.js | 0 15 files changed, 34 insertions(+), 29 deletions(-) rename src/{ => alerts}/AlertBar.js (100%) rename src/{ => alerts}/AlertFilter.js (100%) rename src/{ => alerts}/LegendAlerts.js (100%) rename src/{ => charts}/Legend.js (100%) rename src/{ => charts}/SelectableCharts.js (84%) rename src/{ => charts}/Title.js (98%) rename src/{ => charts}/line.js (100%) rename src/{ => inputs}/Select.js (100%) rename src/{ => inputs}/date.js (100%) create mode 100644 src/inputs/radio.js rename src/{ui => inputs}/toggle.css (100%) rename src/{ => ui}/notes.js (100%) diff --git a/src/App.js b/src/App.js index e8069917f..8b081fcb3 100644 --- a/src/App.js +++ b/src/App.js @@ -4,11 +4,11 @@ import { SingleDaySet, AggregateSet } from './ChartSets'; import StationConfiguration from './StationConfiguration'; import { withRouter } from 'react-router-dom'; import { lookup_station_by_id, get_stop_ids_for_stations } from './stations'; -import { recognize } from './AlertFilter'; -import AlertBar from './AlertBar'; +import { recognize } from './alerts/AlertFilter'; +import AlertBar from './alerts/AlertBar'; import { ProgressBar, progressBarRate } from './ui/ProgressBar'; import './App.css'; -import Select from './Select'; +import Select from './inputs/Select'; import { configPresets } from './presets'; const FRONTEND_TO_BACKEND_MAP = new Map([ diff --git a/src/ChartSets.js b/src/ChartSets.js index e1a0fc071..fb389b8a2 100644 --- a/src/ChartSets.js +++ b/src/ChartSets.js @@ -1,8 +1,8 @@ import React from 'react'; -import { AggregateByTimeSelectable } from './SelectableCharts'; -import { SingleDayLine, AggregateByDate } from './line'; +import { AggregateByTimeSelectable } from './charts/SelectableCharts'; +import { SingleDayLine, AggregateByDate } from './charts/line'; import { station_direction } from './stations'; -import { BusDisclaimer } from './notes'; +import { BusDisclaimer } from './ui/notes'; const dataFields = { traveltimes: { diff --git a/src/StationConfiguration.js b/src/StationConfiguration.js index a02c7ea3c..3c9d6248d 100644 --- a/src/StationConfiguration.js +++ b/src/StationConfiguration.js @@ -1,11 +1,12 @@ import React from 'react'; import classNames from 'classnames'; -import DatePicker from './date'; -import Select from './Select'; +import DatePicker from './inputs/date'; +import Select from './inputs/Select'; +import './inputs/toggle.css'; + import { bus_lines, subway_lines, options_station } from './stations'; import { busDateRange, trainDateRange } from './constants'; -import './ui/toggle.css'; const options_lines = (is_bus) => { diff --git a/src/AlertBar.js b/src/alerts/AlertBar.js similarity index 100% rename from src/AlertBar.js rename to src/alerts/AlertBar.js diff --git a/src/AlertFilter.js b/src/alerts/AlertFilter.js similarity index 100% rename from src/AlertFilter.js rename to src/alerts/AlertFilter.js diff --git a/src/LegendAlerts.js b/src/alerts/LegendAlerts.js similarity index 100% rename from src/LegendAlerts.js rename to src/alerts/LegendAlerts.js diff --git a/src/Legend.js b/src/charts/Legend.js similarity index 100% rename from src/Legend.js rename to src/charts/Legend.js diff --git a/src/SelectableCharts.js b/src/charts/SelectableCharts.js similarity index 84% rename from src/SelectableCharts.js rename to src/charts/SelectableCharts.js index a9233e963..329491050 100644 --- a/src/SelectableCharts.js +++ b/src/charts/SelectableCharts.js @@ -1,24 +1,6 @@ import React from 'react'; -import classNames from 'classnames'; import { AggregateByTime, AggregateByDate } from './line'; - -const RadioForm = (props) => { - const { options, onChange, checked, className } = props; - - return ( -
- {options.map((opt, index) => - - )} -
- ) -} +import RadioForm from '../inputs/radio'; const dayOptions = [ {value: "weekday", label: "Weekday", titleSuffix: "(Weekday)"}, diff --git a/src/Title.js b/src/charts/Title.js similarity index 98% rename from src/Title.js rename to src/charts/Title.js index d5858280a..02c20357a 100644 --- a/src/Title.js +++ b/src/charts/Title.js @@ -1,4 +1,4 @@ -import { colorsForLine } from './constants.js'; +import { colorsForLine } from '../constants.js'; const getLineColor = (lineName) => colorsForLine[lineName] || 'black'; const titleColor = 'gray'; diff --git a/src/line.js b/src/charts/line.js similarity index 100% rename from src/line.js rename to src/charts/line.js diff --git a/src/Select.js b/src/inputs/Select.js similarity index 100% rename from src/Select.js rename to src/inputs/Select.js diff --git a/src/date.js b/src/inputs/date.js similarity index 100% rename from src/date.js rename to src/inputs/date.js diff --git a/src/inputs/radio.js b/src/inputs/radio.js new file mode 100644 index 000000000..a179ac717 --- /dev/null +++ b/src/inputs/radio.js @@ -0,0 +1,22 @@ +import React from "react"; +import classNames from 'classnames'; + +const RadioForm = (props) => { + const { options, onChange, checked, className } = props; + + return ( +
+ {options.map((opt, index) => + + )} +
+ ) +} + +export default RadioForm; \ No newline at end of file diff --git a/src/ui/toggle.css b/src/inputs/toggle.css similarity index 100% rename from src/ui/toggle.css rename to src/inputs/toggle.css diff --git a/src/notes.js b/src/ui/notes.js similarity index 100% rename from src/notes.js rename to src/ui/notes.js From 25c7a6f5e8f994214790de57d3608e26d8e6a6db Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 18 Dec 2021 23:07:33 -0500 Subject: [PATCH 115/152] add no-data error message --- src/App.js | 2 +- src/charts/error.js | 15 +++++++++++++++ src/charts/line.js | 18 ++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 src/charts/error.js diff --git a/src/App.js b/src/App.js index 8b081fcb3..cc34fd645 100644 --- a/src/App.js +++ b/src/App.js @@ -364,7 +364,7 @@ class App extends React.Component { render() { const { configuration, error_message } = this.state; const { from, to, date_start } = configuration; - const canShowCharts = from && to && !error_message; + const canShowCharts = from && to && date_start && !error_message; const canShowAlerts = from && to && date_start && !this.isAggregation(); const recognized_alerts = this.state.alerts?.filter(recognize); const hasNoLoadedCharts = ['traveltimes', 'dwells', 'headways'] diff --git a/src/charts/error.js b/src/charts/error.js new file mode 100644 index 000000000..1240e21b6 --- /dev/null +++ b/src/charts/error.js @@ -0,0 +1,15 @@ +const writeError = (chart) => { + const ctx = chart.chart.ctx; + ctx.save(); + + ctx.font = `16px bold "Helvetica Neue", "Helvetica", "Arial", sans-serif`; + ctx.textAlign = 'center'; + ctx.fillText("No data available. Try another stop/date.", + chart.chart.width / 2, + chart.chart.height / 2); + + ctx.restore(); + +} + +export default writeError; \ No newline at end of file diff --git a/src/charts/line.js b/src/charts/line.js index 4ac23047b..52565d824 100644 --- a/src/charts/line.js +++ b/src/charts/line.js @@ -4,6 +4,7 @@ import { Line, Chart, defaults } from 'react-chartjs-2'; import merge from 'lodash.merge'; import {Legend, LegendLongTerm} from './Legend'; import drawTitle from './Title'; +import writeError from './error'; Chart.Tooltip.positioners.first = (tooltipItems, eventPos) => { let x = eventPos.x; @@ -177,7 +178,12 @@ class SingleDayLine extends React.Component { } }} plugins={[{ - afterDraw: (chart) => drawTitle(this.props.title, this.props.location, this.props.titleBothStops, chart) + afterDraw: (chart) => { + drawTitle(this.props.title, this.props.location, this.props.titleBothStops, chart); + if (!this.props.isLoading && !this.props.data.length) { + writeError(chart); + } + } }]} />
@@ -263,12 +269,20 @@ class AggregateLine extends React.Component { // force graph to show startDate to endDate, even if missing data min: this.props.xMin, max: this.props.xMax, + }, + scaleLabel: { + labelString: this.props.xLabel, } }] } }} plugins={[{ - afterDraw: (chart) => drawTitle(this.props.title, this.props.location, this.props.titleBothStops, chart) + afterDraw: (chart) => { + drawTitle(this.props.title, this.props.location, this.props.titleBothStops, chart); + if (!this.props.isLoading && !this.props.data.length) { + writeError(chart); + } + } }]} />
From 3c1f39562eec30556630c42e41c9c54fd3f935e5 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sat, 18 Dec 2021 23:08:07 -0500 Subject: [PATCH 116/152] add date range to AggregateByTime chart --- src/ChartSets.js | 4 +++- src/charts/SelectableCharts.js | 2 ++ src/charts/line.js | 20 +++++++++++++++----- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/ChartSets.js b/src/ChartSets.js index fb389b8a2..bc5e433db 100644 --- a/src/ChartSets.js +++ b/src/ChartSets.js @@ -77,10 +77,12 @@ const AggregateSet = (props) => {
) diff --git a/src/charts/SelectableCharts.js b/src/charts/SelectableCharts.js index 329491050..2e0056b6f 100644 --- a/src/charts/SelectableCharts.js +++ b/src/charts/SelectableCharts.js @@ -42,6 +42,8 @@ class AggregateByTimeSelectable extends React.Component { location={this.props.location} titleBothStops={this.props.titleBothStops} isLoading={this.props.isLoading} + startDate={this.props.startDate} + endDate={this.props.endDate} > { return {x, y}; }; +const prettyDate = (dateString, with_dow) => { + const options = { + year: 'numeric', + month: 'long', + day: 'numeric', + weekday: with_dow ? 'long' : undefined, + }; + return new Date(`${dateString}T00:00:00`) + .toLocaleDateString(undefined, // user locale/language + options); +} + const departure_from_normal_string = (metric, benchmark) => { const ratio = metric / benchmark; if (!isFinite(ratio) || ratio <= 1.25) { @@ -154,11 +166,7 @@ class SingleDayLine extends React.Component { tooltipFormat: "LTS" // locale time with seconds }, scaleLabel: { - labelString: ( - new Date(`${this.props.date}T00:00:00`) - .toLocaleDateString(undefined, { - weekday: 'long', year: 'numeric', month: 'long', day: 'numeric' - })) + labelString: prettyDate(this.props.date, true) }, // make sure graph shows /at least/ 6am today to 1am tomorrow afterDataLimits: (axis) => { @@ -305,6 +313,7 @@ const AggregateByDate = (props) => { xMin={new Date(`${props.startDate}T00:00:00`)} xMax={new Date(`${props.endDate}T00:00:00`)} fillColor={"rgba(191,200,214,0.5)"} + xLabel="" /> ) } @@ -317,6 +326,7 @@ const AggregateByTime = (props) => { timeUnit={'hour'} timeFormat={'LT'} // momentjs format: locale time fillColor={"rgba(136,174,230,0.5)"} + xLabel={`${prettyDate(props.startDate, false)} – ${prettyDate(props.endDate, false)}`} // xMin, xMax? /> ) From 02269767a0e6da2d4a3620deeedff6bf8f09d7b0 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 19 Dec 2021 21:43:48 -0500 Subject: [PATCH 117/152] no more guesswork in chart title margins --- src/charts/Title.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/charts/Title.js b/src/charts/Title.js index 02c20357a..ad2e8d4ed 100644 --- a/src/charts/Title.js +++ b/src/charts/Title.js @@ -38,8 +38,8 @@ const drawTitle = (title, location, bothStops, chart) => { const ctx = chart.chart.ctx; ctx.save(); - const leftMargin = 50; - const rightMargin = 7; + const leftMargin = chart.scales['x-axis-0'].left; + const rightMargin = chart.chart.width - chart.scales['x-axis-0'].right; const minGap = 10; const vpos_row1 = 25; const vpos_row2 = 50; From 9bcacd0e6c521cdbf626b32f39837aec2f9a0ae3 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 19 Dec 2021 22:06:53 -0500 Subject: [PATCH 118/152] fix small screen error msg --- src/charts/error.js | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/charts/error.js b/src/charts/error.js index 1240e21b6..c6152e07b 100644 --- a/src/charts/error.js +++ b/src/charts/error.js @@ -1,15 +1,33 @@ +const errorMsg = "No data available. Try another stop or date."; +const txtColor = "gray"; + +function font(size=16) { + return `bold ${size}px "Helvetica Neue", "Helvetica", "Arial", sans-serif`; +} + const writeError = (chart) => { const ctx = chart.chart.ctx; ctx.save(); - ctx.font = `16px bold "Helvetica Neue", "Helvetica", "Arial", sans-serif`; + const xAxis = chart.scales['x-axis-0']; + const yAxis = chart.scales['y-axis-0']; + const centerX = (xAxis.left + xAxis.right) / 2; + const centerY = (yAxis.bottom + yAxis.top) / 2; + const maxWidth = (xAxis.right - xAxis.left); + + /* shrink font on smaller screens */ + let fontSize = 16; + ctx.font = font(fontSize); + while (ctx.measureText(errorMsg).width > maxWidth) { + fontSize -= 1; + ctx.font = font(fontSize); + } + + ctx.fillStyle = txtColor; ctx.textAlign = 'center'; - ctx.fillText("No data available. Try another stop/date.", - chart.chart.width / 2, - chart.chart.height / 2); + ctx.fillText(errorMsg, centerX, centerY); ctx.restore(); - } export default writeError; \ No newline at end of file From 28afe00a4fe36d4838210d04782573defa5b88f6 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 19 Dec 2021 22:20:18 -0500 Subject: [PATCH 119/152] relocate line naming --- src/StationConfiguration.js | 24 ++++++++---------------- src/stations.js | 9 +++++++++ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/StationConfiguration.js b/src/StationConfiguration.js index 3c9d6248d..2bb1381b6 100644 --- a/src/StationConfiguration.js +++ b/src/StationConfiguration.js @@ -5,26 +5,18 @@ import DatePicker from './inputs/date'; import Select from './inputs/Select'; import './inputs/toggle.css'; -import { bus_lines, subway_lines, options_station } from './stations'; +import { bus_lines, subway_lines, options_station, line_name } from './stations'; import { busDateRange, trainDateRange } from './constants'; const options_lines = (is_bus) => { - if (is_bus) { - return bus_lines().map((line) => { - return { - value: line, - label: line.includes("/") ? `Routes ${line}` : `Route ${line}` - } - }); - } else { - return subway_lines().map((line) => { - return { - value: line, - label: `${line} Line` - } - }); - } + const lines = is_bus ? bus_lines() : subway_lines(); + return lines.map((line) => { + return { + value: line, + label: line_name(line) + } + }); }; const options_station_ui = (line) => { diff --git a/src/stations.js b/src/stations.js index 5e5dc188b..fe234e9d5 100644 --- a/src/stations.js +++ b/src/stations.js @@ -63,6 +63,14 @@ const get_stop_ids_for_stations = (from, to) => { } } +const line_name = (line) => { + if (stations[line].type === "bus") { + return line.includes("/") ? `Routes ${line}` : `Route ${line}`; + } else { + return `${line} Line`; + } +} + export { stations, all_lines, @@ -72,4 +80,5 @@ export { station_direction, lookup_station_by_id, get_stop_ids_for_stations, + line_name }; From 159e05332fba9b5440bcf3a9d1215072a574069b Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 19 Dec 2021 23:14:21 -0500 Subject: [PATCH 120/152] fixes to documentTitle and GA --- src/App.js | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/App.js b/src/App.js index cc34fd645..335607013 100644 --- a/src/App.js +++ b/src/App.js @@ -3,7 +3,7 @@ import ReactGA from 'react-ga'; import { SingleDaySet, AggregateSet } from './ChartSets'; import StationConfiguration from './StationConfiguration'; import { withRouter } from 'react-router-dom'; -import { lookup_station_by_id, get_stop_ids_for_stations } from './stations'; +import { lookup_station_by_id, get_stop_ids_for_stations, line_name } from './stations'; import { recognize } from './alerts/AlertFilter'; import AlertBar from './alerts/AlertBar'; import { ProgressBar, progressBarRate } from './ui/ProgressBar'; @@ -19,12 +19,15 @@ const FRONTEND_TO_BACKEND_MAP = new Map([ ]); const APP_DATA_BASE_PATH = FRONTEND_TO_BACKEND_MAP.get(window.location.hostname); +const RAPIDTRANSIT_PATH = "/rapidtransit"; +const BUS_PATH = "/bus"; + const MAX_AGGREGATION_MONTHS = 8; const RANGE_TOO_LARGE_ERROR = `Please select a range no larger than ${MAX_AGGREGATION_MONTHS} months.`; const RANGE_NEGATIVE_ERROR = "Oops, please ensure the start date comes before the selected end date."; const stateFromURL = (pathname, config) => { - const bus_mode = (pathname === "/bus") + const bus_mode = (pathname === BUS_PATH) const [line, from_id, to_id, date_start, date_end] = config.split(","); const from = lookup_station_by_id(line, from_id); const to = lookup_station_by_id(line, to_id); @@ -41,7 +44,7 @@ const stateFromURL = (pathname, config) => { const urlFromState = (config) => { const { bus_mode, line, from, to, date_start, date_end } = config; const { fromStopIds, toStopIds } = get_stop_ids_for_stations(from, to); - const pathname = bus_mode ? "bus" : "rapidtransit" + const path = bus_mode ? BUS_PATH : RAPIDTRANSIT_PATH; const parts = [ line, fromStopIds?.[0], @@ -50,14 +53,16 @@ const urlFromState = (config) => { date_end, ]; const partString = parts.map(x => x || "").join(","); - const url = `/${pathname}?config=${partString}`; + const url = `${path}?config=${partString}`; const isComplete = parts.slice(0, -1).every(x => x); return [url, isComplete]; }; const documentTitle = (config) => { - return `${config.line} Line - ${config.date_start} - TransitMatters Data Dashboard`; + const line = line_name(config.line); + const date_info = `${config.date_start}` + (config.date_end ? ` to ${config.date_end}` : ""); + return `${line} - ${date_info} - TransitMatters Data Dashboard`; }; const showBetaTag = () => { @@ -100,7 +105,7 @@ class App extends React.Component { }; ReactGA.initialize("UA-71173708-2"); - ReactGA.pageview("/rapidtransit"); + ReactGA.pageview(props.location.pathname); const url_config = new URLSearchParams(props.location.search).get("config"); if (typeof url_config === "string") { @@ -188,6 +193,7 @@ class App extends React.Component { // callback once state is updated () => { this.props.history.replace(urlFromState(this.state.configuration)[0], this.state); + document.title = documentTitle(this.state.configuration); if (refetch) { this.download(); } @@ -297,9 +303,8 @@ class App extends React.Component { this.fetchDataset('alerts', controller.signal, { route: configuration.line, }); - ReactGA.pageview(window.location.pathname + window.location.search); - document.title = documentTitle(this.state.configuration); } + ReactGA.pageview(window.location.pathname + window.location.search); } } From 9a2ff3c5f7a4916a0b588e40f5d0c311b7afbc84 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 19 Dec 2021 23:31:05 -0500 Subject: [PATCH 121/152] documentTitle bug fixes --- src/App.js | 13 ++++++++++--- src/stations.js | 3 +++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/App.js b/src/App.js index 335607013..7eafdb48f 100644 --- a/src/App.js +++ b/src/App.js @@ -60,9 +60,16 @@ const urlFromState = (config) => { }; const documentTitle = (config) => { - const line = line_name(config.line); - const date_info = `${config.date_start}` + (config.date_end ? ` to ${config.date_end}` : ""); - return `${line} - ${date_info} - TransitMatters Data Dashboard`; + const pieces = ["TransitMatters Data Dashboard"]; + if (config.line) { + pieces.splice(0, 0, line_name(config.line)); + } + if (config.date_start && config.date_end) { + pieces.splice(1, 0, `${config.date_start} to ${config.date_end}`); + } else if (config.date_start) { + pieces.splice(1, 0, config.date_start.toString()); + } + return pieces.join(' - '); }; const showBetaTag = () => { diff --git a/src/stations.js b/src/stations.js index fe234e9d5..9434cce0f 100644 --- a/src/stations.js +++ b/src/stations.js @@ -64,6 +64,9 @@ const get_stop_ids_for_stations = (from, to) => { } const line_name = (line) => { + if (!line) { + return ""; + } if (stations[line].type === "bus") { return line.includes("/") ? `Routes ${line}` : `Route ${line}`; } else { From df159b5b4684746bb86f827b429ad4a64aa10f8d Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Sun, 19 Dec 2021 23:40:40 -0500 Subject: [PATCH 122/152] adjust headwayTitle logic --- src/ChartSets.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ChartSets.js b/src/ChartSets.js index bc5e433db..7142dc4b3 100644 --- a/src/ChartSets.js +++ b/src/ChartSets.js @@ -25,6 +25,12 @@ const dataFields = { } } +const headwayTitle = { + // indexed by bus_mode + true: "Time between buses (headways)", + false: "Time between trains (headways)" +} + function getLocationDescription(from, to, line) { if (from && to) { return { @@ -39,7 +45,6 @@ function getLocationDescription(from, to, line) { const AggregateSet = (props) => { const locationDescription = getLocationDescription(props.from, props.to, props.line); - const headwayTitle = props.bus_mode ? "Time between buses (headways)" : "Time between trains (headways)"; return(
{ endDate={props.endDate} /> { // TODO: fix this by clearing existing data in updateConfiguration const anyTravelBenchmarks = Array.isArray(props.traveltimes) && props.traveltimes.some(e => e.benchmark_travel_time_sec > 0); const anyHeadwayBenchmarks = props.headways.some(e => e.benchmark_headway_time_sec > 0); - const headwayTitle = props.bus_mode ? "Time between buses (headways)" : "Time between trains (headways)"; return(
{ /> Date: Mon, 20 Dec 2021 09:26:21 -0500 Subject: [PATCH 123/152] Minor cleanups to data import tool. --- server/bus/setup_bus_input.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/server/bus/setup_bus_input.sh b/server/bus/setup_bus_input.sh index 3a209f7b0..a405b28f2 100755 --- a/server/bus/setup_bus_input.sh +++ b/server/bus/setup_bus_input.sh @@ -14,16 +14,20 @@ cd data/input for i in 2021 2020 2019 2018; do unzip -d $i $i.zip done + mv 2021/MBTA*/*.csv 2021/ +rmdir 2021/MBTA_Bus_Arrival_Depature_Times_2021/ + +# Rename files so they have consistent names and no spaces. +mv "2020/Bus Arrival Departure Times Jan-Mar 2020.csv" "2020/2020-Q1.csv" +mv "2020/Bus Arrival Departure Times Apr-June 2020.csv" "2020/2020-Q2.csv" +mv "2020/Bus Arrival Departure Times Jul-Sep 2020.csv" "2020/2020-Q3.csv" +mv "2020/Bus Arrival Departure Times Oct-Dec 2020.csv" "2020/2020-Q4.csv" +mv "2019/MBTA Bus Arrival Departure Jan-Mar 2019.csv" "2019/2019-Q1.csv" +mv "2019/MBTA Bus Arrival Departure Apr-June 2019.csv" "2019/2019-Q2.csv" +mv "2019/MBTA Bus Arrival Departure Jul-Sept 2019.csv" "2019/2019-Q3.csv" +mv "2019/MBTA Bus Arrival Departure Oct-Dec 2019.csv" "2019/2019-Q4.csv" +mv "2018/MBTA Bus Arrival Departure Aug-Sept 2018.csv" "2018/2018-Q3.csv" +mv "2018/MBTA Bus Arrival Departure Oct-Dec 2018.csv" "2018/2018-Q4.csv" +sed -i -e 's///' 2020/2020-Q3.csv -mv "2020/Bus Arrival Departure Times Jan-Mar 2020.csv" "2020/2020-01-03.csv" -mv "2020/Bus Arrival Departure Times Apr-June 2020.csv" "2020/2020-04-06.csv" -mv "2020/Bus Arrival Departure Times Jul-Sep 2020.csv" "2020/2020-07-09.csv" -mv "2020/Bus Arrival Departure Times Oct-Dec 2020.csv" "2020/2020-10-12.csv" -mv "2019/MBTA Bus Arrival Departure Jan-Mar 2019.csv" "2019/2019-01-03.csv" -mv "2019/MBTA Bus Arrival Departure Apr-June 2019.csv" "2019/2019-04-06.csv" -mv "2019/MBTA Bus Arrival Departure Jul-Sept 2019.csv" "2019/2019-07-09.csv" -mv "2019/MBTA Bus Arrival Departure Oct-Dec 2019.csv" "2019/2019-10-12.csv" -mv "2018/MBTA Bus Arrival Departure Aug-Sept 2018.csv" "2018/2018-08-09.csv" -mv "2018/MBTA Bus Arrival Departure Oct-Dec 2018.csv" "2018/2018-10-12.csv" -sed -i -e 's///' 2020/2020-07-09.csv From 3542b955f34ddde56bd072d1864faae6811e337d Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 09:32:23 -0500 Subject: [PATCH 124/152] Adding supprot for passing routes as args. --- server/bus/gen_bus_data.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/bus/gen_bus_data.sh b/server/bus/gen_bus_data.sh index f967a4cf4..1a421b8b5 100755 --- a/server/bus/gen_bus_data.sh +++ b/server/bus/gen_bus_data.sh @@ -1,8 +1,13 @@ -#!/bin/bash +#!/bin/bash -x + +routes="$@" +if [ -z "$routes" ]; then + routes="1" +fi for y in 2018 2019 2020 2021; do for f in $(find data/input/$y/ -name '*.csv'); do echo "Generating stop data from $f" - pipenv run python bus2train.py $f data/output -r 1 + pipenv run python bus2train.py $f data/output -r $routes done done From 101418d00aaff003133e6296093799b1d68bb817 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 09:34:36 -0500 Subject: [PATCH 125/152] Formatting improvements. --- server/bus/compare_manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/bus/compare_manifest.py b/server/bus/compare_manifest.py index 741cfcaa8..21687b011 100644 --- a/server/bus/compare_manifest.py +++ b/server/bus/compare_manifest.py @@ -13,7 +13,7 @@ def runone(path, first=False): if s not in station_stops: station_stops[s] = {} if not first: - print("Found missing station %s" % s) + print(" + Found new station %s" % s) unchanged = False for direction in i['stops']: if direction not in station_stops[s]: @@ -22,7 +22,7 @@ def runone(path, first=False): if stop not in station_stops[s][direction]: station_stops[s][direction].append(stop) if not first: - print("Found additional stop %s at station %s in %s" % (stop, s, path)) + print(" + Found additional stop %s at station %s in %s" % (stop, s, path)) unchanged = False return unchanged From c4f8ddfc57e5a4d56448444369ae92536815a8c6 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 09:50:37 -0500 Subject: [PATCH 126/152] Check for any removed stops before checking for new ones. --- server/bus/compare_manifest.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/server/bus/compare_manifest.py b/server/bus/compare_manifest.py index 21687b011..f4f627788 100644 --- a/server/bus/compare_manifest.py +++ b/server/bus/compare_manifest.py @@ -3,10 +3,22 @@ station_stops = {} - def runone(path, first=False): unchanged = True current = json.load(open(path)) + # Print any removed stops first + if not first: + my_stations = list(current.values())[0]['stations'] + stat_map = dict(map(lambda x: (x['station'], x), my_stations)) + for s in station_stops: + if s not in stat_map: + print(" + Station %s removed in file %s. (Stops: %s)" % (s, path, station_stops[s])) + continue + for d in station_stops[s]: + for stop in station_stops[s][d]: + if not stop in stat_map[s]['stops'][d]: + print(" + Stop %s removed from %s in file %s" % (stop, s, path)) + for i in list(current.values())[0]['stations']: s = i['station'] From 08a248147af266a9567eb4102d1bca0489581a2d Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 09:54:51 -0500 Subject: [PATCH 127/152] Further minor name improvements. --- src/bus_constants/111.json | 2 +- src/bus_constants/39.json | 2 +- src/bus_constants/57.json | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bus_constants/111.json b/src/bus_constants/111.json index c833e9531..1fe540485 100644 --- a/src/bus_constants/111.json +++ b/src/bus_constants/111.json @@ -122,7 +122,7 @@ } }, { - "stop_name": "Causeway Street & Commercial Street", + "stop_name": "Causeway St & Commercial St", "branches": null, "station": "cacom", "order": 9, diff --git a/src/bus_constants/39.json b/src/bus_constants/39.json index e8036a63e..3156e4ad9 100644 --- a/src/bus_constants/39.json +++ b/src/bus_constants/39.json @@ -35,7 +35,7 @@ } }, { - "stop_name": "Centre St & South Huntington Ave", + "stop_name": "Centre St & S Huntington", "branches": null, "station": "shunt", "order": 3, diff --git a/src/bus_constants/57.json b/src/bus_constants/57.json index 9b7c5b322..163596441 100644 --- a/src/bus_constants/57.json +++ b/src/bus_constants/57.json @@ -80,7 +80,7 @@ } }, { - "stop_name": "Brighton & Commonwealth Avenue", + "stop_name": "Brighton & Comm Ave", "branches": null, "station": "brcom", "order": 6, @@ -94,7 +94,7 @@ } }, { - "stop_name": "Commonwealth Ave @ BU Bridge", + "stop_name": "Comm Ave @ BU Bridge", "branches": null, "station": "bubrg", "order": 7, From 121b3e1c4ac3c0139d10122f575622c94a445eea Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 09:57:59 -0500 Subject: [PATCH 128/152] Change split route to use two different points rather than combined. --- src/bus_constants/111.json | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/bus_constants/111.json b/src/bus_constants/111.json index 1fe540485..a164d6be9 100644 --- a/src/bus_constants/111.json +++ b/src/bus_constants/111.json @@ -79,15 +79,22 @@ } }, { - "stop_name": "[INB] Third & Chestnut | [OB] Broadway & Beacon", - "_comment": "The route splits with differing timepoints. Hopefully this is clear yet concise", + "stop_name": "Third & Chestnut (Inbound)", "branches": null, - "station": "thche|bwybc", + "station": "thche", "order": 6, "stops": { "1": [ "111-1-5607" - ], + ] + } + }, + { + "stop_name": "Broadway & Beacon (Outbound)", + "branches": null, + "station": "bwybc", + "order": 6, + "stops": { "0": [ "111-0-5611" ] From 6250f9c3e5084473507bda995cfb0951da4baf1a Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 09:58:47 -0500 Subject: [PATCH 129/152] Commonwealth -> Comm --- src/bus_constants/66.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bus_constants/66.json b/src/bus_constants/66.json index 3df4fd73c..52b017a30 100644 --- a/src/bus_constants/66.json +++ b/src/bus_constants/66.json @@ -51,7 +51,7 @@ } }, { - "stop_name": "Harvard Ave & Commonwealth Ave", + "stop_name": "Harvard Ave & Comm Ave", "branches": null, "station": "hvdcm", "order": 4, From 8c90146bb35a3b137642c0f5c6f8c308e0afc6b4 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 09:59:31 -0500 Subject: [PATCH 130/152] Mount Auburn -> Mt Auburn --- src/bus_constants/71.json | 8 ++++---- src/bus_constants/73.json | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bus_constants/71.json b/src/bus_constants/71.json index a807096d6..2ada6ec16 100644 --- a/src/bus_constants/71.json +++ b/src/bus_constants/71.json @@ -7,7 +7,7 @@ }, "stations": [ { - "stop_name": "Aberdeen & Mount Auburn", + "stop_name": "Aberdeen & Mt Auburn", "branches": null, "station": "abmta", "order": 1, @@ -49,7 +49,7 @@ } }, { - "stop_name": "Mount Auburn Bridge", + "stop_name": "Mt Auburn Bridge", "branches": null, "station": "mtaub", "order": 3, @@ -63,7 +63,7 @@ } }, { - "stop_name": "Mount Auburn Hospital", + "stop_name": "Mt Auburn Hospital", "branches": null, "station": "mthsp", "order": 4, @@ -77,7 +77,7 @@ } }, { - "stop_name": "Mount Auburn @ Story St", + "stop_name": "Mt Auburn @ Story St", "branches": null, "station": "mtsty", "order": 5, diff --git a/src/bus_constants/73.json b/src/bus_constants/73.json index 89201c1d4..176957e47 100644 --- a/src/bus_constants/73.json +++ b/src/bus_constants/73.json @@ -35,7 +35,7 @@ } }, { - "stop_name": "Mount Auburn Bridge", + "stop_name": "Mt Auburn Bridge", "branches": null, "station": "mtaub", "order": 3, @@ -50,7 +50,7 @@ } }, { - "stop_name": "Mount Auburn Hospital", + "stop_name": "Mt Auburn Hospital", "branches": null, "station": "mthsp", "order": 4, @@ -64,7 +64,7 @@ } }, { - "stop_name": "Mount Auburn @ Story St", + "stop_name": "Mt Auburn @ Story St", "branches": null, "station": "mtsty", "order": 5, From f2043fe4668a68def9be93d9501b06fd2f5c8099 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 10:11:34 -0500 Subject: [PATCH 131/152] Lint fixes. --- server/bus/compare_manifest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/bus/compare_manifest.py b/server/bus/compare_manifest.py index f4f627788..d407a92a3 100644 --- a/server/bus/compare_manifest.py +++ b/server/bus/compare_manifest.py @@ -3,6 +3,7 @@ station_stops = {} + def runone(path, first=False): unchanged = True current = json.load(open(path)) @@ -16,7 +17,7 @@ def runone(path, first=False): continue for d in station_stops[s]: for stop in station_stops[s][d]: - if not stop in stat_map[s]['stops'][d]: + if stop not in stat_map[s]['stops'][d]: print(" + Stop %s removed from %s in file %s" % (stop, s, path)) for i in list(current.values())[0]['stations']: From 7090bdc75e36e58b0f7f3ffc2cbba6d77ff81d60 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 11:41:03 -0500 Subject: [PATCH 132/152] Change to - for removals. --- server/bus/compare_manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/bus/compare_manifest.py b/server/bus/compare_manifest.py index d407a92a3..a096d503c 100644 --- a/server/bus/compare_manifest.py +++ b/server/bus/compare_manifest.py @@ -13,12 +13,12 @@ def runone(path, first=False): stat_map = dict(map(lambda x: (x['station'], x), my_stations)) for s in station_stops: if s not in stat_map: - print(" + Station %s removed in file %s. (Stops: %s)" % (s, path, station_stops[s])) + print(" - Station %s removed in file %s. (Stops: %s)" % (s, path, station_stops[s])) continue for d in station_stops[s]: for stop in station_stops[s][d]: if stop not in stat_map[s]['stops'][d]: - print(" + Stop %s removed from %s in file %s" % (stop, s, path)) + print(" - Stop %s removed from %s in file %s" % (stop, s, path)) for i in list(current.values())[0]['stations']: s = i['station'] From 37c665d585c768869e6c799977109aa30f2cc801 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Mon, 20 Dec 2021 12:43:59 -0500 Subject: [PATCH 133/152] update bus date range --- src/constants.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/constants.js b/src/constants.js index cb976fbf8..77af4ce1e 100644 --- a/src/constants.js +++ b/src/constants.js @@ -12,8 +12,8 @@ export const trainDateRange = { }; export const busDateRange = { - minDate: "2021-01-01", - maxDate: "2021-09-30" + minDate: "2018-08-01", + maxDate: "2021-10-31" }; export const stations = { From 9a6eb82cf50c71290aeabae590dcc5557e57fd91 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 13:18:01 -0500 Subject: [PATCH 134/152] Eliminate duplicate order, and add extra keys to prevent any trouble. --- src/bus_constants/111.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/bus_constants/111.json b/src/bus_constants/111.json index a164d6be9..e84539d91 100644 --- a/src/bus_constants/111.json +++ b/src/bus_constants/111.json @@ -86,18 +86,20 @@ "stops": { "1": [ "111-1-5607" - ] + ], + "0": [] } }, { "stop_name": "Broadway & Beacon (Outbound)", "branches": null, "station": "bwybc", - "order": 6, + "order": 6.5, "stops": { "0": [ "111-0-5611" - ] + ], + "1": [] } }, { From 3092cc3409f5f0f04c4b0de3cdcf1f9c1280ccf0 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 13:20:55 -0500 Subject: [PATCH 135/152] Add in new stations to the list now that data is uploaded. --- src/stations.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/stations.js b/src/stations.js index 5ee85a24a..09d9e7d12 100644 --- a/src/stations.js +++ b/src/stations.js @@ -17,10 +17,17 @@ import bus_114_116_117 from './bus_constants/114-116-117.json' const stations = {...rt_stations, ...bus_1, + ...bus_15, + ...bus_22, ...bus_23, ...bus_28, + ...bus_32, + ...bus_39, ...bus_57, ...bus_66, + ...bus_71, + ...bus_73, + ...bus_77, ...bus_111, ...bus_114_116_117 }; From bc1175c3241b7a6055c761faab511885a63cb7f1 Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 14:34:14 -0500 Subject: [PATCH 136/152] Abbreviations for Route 1. --- src/bus_constants/1.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bus_constants/1.json b/src/bus_constants/1.json index cb935e7aa..6dffaa6db 100644 --- a/src/bus_constants/1.json +++ b/src/bus_constants/1.json @@ -21,7 +21,7 @@ } }, { - "stop_name": "Massachusetts Avenue & Putnam Avenue", + "stop_name": "Mass Ave & Putnam Ave", "branches": null, "station": "maput", "order": 2, @@ -49,7 +49,7 @@ } }, { - "stop_name": "MIT @ Massachusetts Avenue", + "stop_name": "MIT @ Mass Ave", "branches": null, "station": "mit", "order": 4, @@ -77,7 +77,7 @@ } }, { - "stop_name": "Massachusetts Avenue (Orange Line)", + "stop_name": "Mass Ave (Orange Line)", "branches": null, "station": "masta", "order": 6, @@ -91,7 +91,7 @@ } }, { - "stop_name": "Massachusetts Avenue @ Washington", + "stop_name": "Mass Ave @ Washington", "branches": null, "station": "wasma", "order": 7, @@ -105,7 +105,7 @@ } }, { - "stop_name": "Melnea Cass Boulevard @ Washington Street", + "stop_name": "Melnea Cass @ Washington", "branches": null, "station": "melwa", "order": 8, From 6417d5b2c3457e2d7a92d59d3d18abaf18a3aebe Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 15:11:35 -0500 Subject: [PATCH 137/152] Tabs -> Spaces. --- src/bus_constants/111.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bus_constants/111.json b/src/bus_constants/111.json index e84539d91..c8c371d50 100644 --- a/src/bus_constants/111.json +++ b/src/bus_constants/111.json @@ -87,7 +87,7 @@ "1": [ "111-1-5607" ], - "0": [] + "0": [] } }, { @@ -99,7 +99,7 @@ "0": [ "111-0-5611" ], - "1": [] + "1": [] } }, { From 75fb7cb9171bc9697c23bfd2618dc08c1e8c542b Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 19:11:23 -0500 Subject: [PATCH 138/152] Remove St. Peter's Square, which is close to kane. --- src/bus_constants/15.json | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/bus_constants/15.json b/src/bus_constants/15.json index 29651c87e..0218f5345 100644 --- a/src/bus_constants/15.json +++ b/src/bus_constants/15.json @@ -21,20 +21,6 @@ ] } }, - { - "stop_name": "Saint Peter's Square (Dorchester)", - "branches": null, - "station": "peter", - "order": 1, - "stops": { - "1": [ - "15-1-15100" - ], - "0": [ - "15-0-15100" - ] - } - }, { "stop_name": "Geneva Ave @ Bowdoin St", "branches": null, From 512b09f840b6edf8c679080fb72b0c30405c3ecf Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 19:12:10 -0500 Subject: [PATCH 139/152] Reorder station imports. --- src/stations.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stations.js b/src/stations.js index 09d9e7d12..c3cb7e7da 100644 --- a/src/stations.js +++ b/src/stations.js @@ -4,8 +4,8 @@ import bus_1 from './bus_constants/1.json'; import bus_15 from './bus_constants/15.json'; import bus_22 from './bus_constants/22.json'; import bus_23 from './bus_constants/23.json'; -import bus_32 from './bus_constants/32.json'; import bus_28 from './bus_constants/28.json'; +import bus_32 from './bus_constants/32.json'; import bus_39 from './bus_constants/39.json'; import bus_57 from './bus_constants/57.json'; import bus_66 from './bus_constants/66.json'; From 00a73d34d56a11f890bf752527e5daba2629682f Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Mon, 20 Dec 2021 19:26:12 -0500 Subject: [PATCH 140/152] Eliminate uncommon short-term stop. --- src/bus_constants/71.json | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/bus_constants/71.json b/src/bus_constants/71.json index 2ada6ec16..364686331 100644 --- a/src/bus_constants/71.json +++ b/src/bus_constants/71.json @@ -6,20 +6,6 @@ "1": "inbound" }, "stations": [ - { - "stop_name": "Aberdeen & Mt Auburn", - "branches": null, - "station": "abmta", - "order": 1, - "stops": { - "1": [ - "71-1-2098" - ], - "0": [ - "71-0-2077" - ] - } - }, { "stop_name": "Watertown Square", "branches": null, From 43dca0cced6053963916fe01600a8cdfcbf98f8c Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Tue, 21 Dec 2021 00:28:20 -0500 Subject: [PATCH 141/152] fix short-circuiting bug in compare_manifest --- server/bus/compare_manifest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/bus/compare_manifest.py b/server/bus/compare_manifest.py index a096d503c..271e2dcd4 100644 --- a/server/bus/compare_manifest.py +++ b/server/bus/compare_manifest.py @@ -26,7 +26,7 @@ def runone(path, first=False): if s not in station_stops: station_stops[s] = {} if not first: - print(" + Found new station %s" % s) + print(" + Found new station %s" % s) unchanged = False for direction in i['stops']: if direction not in station_stops[s]: @@ -35,7 +35,7 @@ def runone(path, first=False): if stop not in station_stops[s][direction]: station_stops[s][direction].append(stop) if not first: - print(" + Found additional stop %s at station %s in %s" % (stop, s, path)) + print(" + Found additional stop %s at station %s in %s" % (stop, s, path)) unchanged = False return unchanged @@ -44,7 +44,7 @@ def run(paths): unchanged = True runone(paths[0], first=True) for path in reversed(paths[1:]): - unchanged = unchanged and runone(path) + unchanged = runone(path) and unchanged if unchanged: print("No new stations/stops on route.") else: From 39fea69b619bc856462dd4bfedd0cea6fdddb604 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Tue, 21 Dec 2021 00:28:38 -0500 Subject: [PATCH 142/152] update gen_manifests.sh for new tooling --- server/bus/gen_manifests.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/bus/gen_manifests.sh b/server/bus/gen_manifests.sh index 39d408857..efd14d381 100755 --- a/server/bus/gen_manifests.sh +++ b/server/bus/gen_manifests.sh @@ -1,10 +1,10 @@ #!/bin/bash -route="111" +for route in 1 111 15 22 23 28 32 39 57 66 71 73 77 114 116 117; do + mkdir -p data/output/manifests/$route -mkdir data/output/manifests/$route - -for f in $(find data/input/MBTA_Bus_Arrival_Departure_Times_2021/ -name *.csv); do - month=$(echo $f | rev | cut -d- -f1 | rev | cut -d. -f1) - pipenv run python manifest.py $f "data/output/manifests/$route/$route_$month.json" --checkpoints "data/MBTA_GTFS/checkpoints.txt" -r $route + for f in $(find data/input/ -name *.csv); do + month=$(echo $f | cut -d/ -f4 | cut -d. -f1) + pipenv run python manifest.py $f "data/output/manifests/$route/$route_$month.json" --checkpoints "data/input/MBTA_GTFS/checkpoints.txt" -r $route + done done From 5616ec42914e5db06a331b594ceda56f0cfb910e Mon Sep 17 00:00:00 2001 From: Christopher Schmidt Date: Tue, 21 Dec 2021 12:52:39 -0500 Subject: [PATCH 143/152] Change name to include Inbound --- src/bus_constants/39.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bus_constants/39.json b/src/bus_constants/39.json index 3156e4ad9..9432f1a9f 100644 --- a/src/bus_constants/39.json +++ b/src/bus_constants/39.json @@ -120,7 +120,7 @@ } }, { - "stop_name": "Copley", + "stop_name": "Copley (Inbound)", "branches": null, "station": "copst", "order": 9, From 175db41a1bd3e7b95530f12a8b268dff841bb822 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Tue, 21 Dec 2021 13:54:01 -0500 Subject: [PATCH 144/152] oops one more bug fix (inbound/outbound stop errors) --- src/App.js | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/App.js b/src/App.js index 7eafdb48f..85b38e640 100644 --- a/src/App.js +++ b/src/App.js @@ -24,7 +24,8 @@ const BUS_PATH = "/bus"; const MAX_AGGREGATION_MONTHS = 8; const RANGE_TOO_LARGE_ERROR = `Please select a range no larger than ${MAX_AGGREGATION_MONTHS} months.`; -const RANGE_NEGATIVE_ERROR = "Oops, please ensure the start date comes before the selected end date."; +const RANGE_NEGATIVE_ERROR = "Please ensure the start date comes before the selected end date."; +const INVALID_STOP_ERROR = "Invalid stop selection. Please check the inbound/outbound nature of your selected stops."; const stateFromURL = (pathname, config) => { const bus_mode = (pathname === BUS_PATH) @@ -160,6 +161,17 @@ class App extends React.Component { return null; } + validateStops(from, to) { + /* A selected stop might have no stop ids depending on direction (e.g. inbound-only). */ + const { fromStopIds, toStopIds } = get_stop_ids_for_stations(from, to); + if (from && to) { + if (!fromStopIds.length || !toStopIds.length) { + return INVALID_STOP_ERROR; + } + } + return null; + } + updateConfiguration(config_change, refetch = true) { // Save to browser history only if we are leaving a complete configuration // so back button never takes you to a partial page @@ -176,7 +188,8 @@ class App extends React.Component { } }; - const error = this.validateRange(update.configuration.date_start, update.configuration.date_end); + const error = this.validateRange(update.configuration.date_start, update.configuration.date_end) || + this.validateStops(update.configuration.from, update.configuration.to); this.setState({ error_message: error }); From f20eca89e7196dcaeb4e7b44de8ba89eb21d9c6c Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 22 Dec 2021 00:13:51 -0500 Subject: [PATCH 145/152] minor changes for review --- src/charts/Title.js | 4 ++-- src/charts/error.js | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/charts/Title.js b/src/charts/Title.js index ad2e8d4ed..0c9d41587 100644 --- a/src/charts/Title.js +++ b/src/charts/Title.js @@ -22,9 +22,9 @@ const parse_location_description = (location, bothStops) => { return result; }; -function font(size = 16) { +function font(size_px = 16) { // This is the default from chartjs, but now we can adjust size. - return `bold ${size}px "Helvetica Neue", "Helvetica", "Arial", sans-serif`; + return `bold ${size_px}px "Helvetica Neue", "Helvetica", "Arial", sans-serif`; } const calcLocationWidth = (words, ctx) => { diff --git a/src/charts/error.js b/src/charts/error.js index c6152e07b..e582a979d 100644 --- a/src/charts/error.js +++ b/src/charts/error.js @@ -1,11 +1,11 @@ const errorMsg = "No data available. Try another stop or date."; const txtColor = "gray"; -function font(size=16) { - return `bold ${size}px "Helvetica Neue", "Helvetica", "Arial", sans-serif`; +function font(size_px=16) { + return `bold ${size_px}px "Helvetica Neue", "Helvetica", "Arial", sans-serif`; } -const writeError = (chart) => { +function writeError(chart) { const ctx = chart.chart.ctx; ctx.save(); From 611aef14d71ba93a63fe22cb2056b5145a4594eb Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 29 Dec 2021 21:49:43 -0500 Subject: [PATCH 146/152] clean up routing, w/ bus_mode definition --- src/App.js | 6 +++++- src/index.js | 24 +++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/App.js b/src/App.js index 85b38e640..b23c2271e 100644 --- a/src/App.js +++ b/src/App.js @@ -28,7 +28,7 @@ const RANGE_NEGATIVE_ERROR = "Please ensure the start date comes before the sele const INVALID_STOP_ERROR = "Invalid stop selection. Please check the inbound/outbound nature of your selected stops."; const stateFromURL = (pathname, config) => { - const bus_mode = (pathname === BUS_PATH) + const bus_mode = (pathname === BUS_PATH); const [line, from_id, to_id, date_start, date_end] = config.split(","); const from = lookup_station_by_id(line, from_id); const to = lookup_station_by_id(line, to_id); @@ -53,6 +53,9 @@ const urlFromState = (config) => { date_start, date_end, ]; + if (!parts.some(x => x)) { + return [path, false]; + } const partString = parts.map(x => x || "").join(","); const url = `${path}?config=${partString}`; const isComplete = parts.slice(0, -1).every(x => x); @@ -100,6 +103,7 @@ class App extends React.Component { this.state = { configuration: { + bus_mode: props.bus_mode, show_alerts: true, }, error_message: null, diff --git a/src/index.js b/src/index.js index a0b2b8dcf..5051288d3 100644 --- a/src/index.js +++ b/src/index.js @@ -3,9 +3,23 @@ import ReactDOM from 'react-dom'; import './index.css'; import App from './App'; import OpenSource from './OpenSource'; -import { BrowserRouter, Route } from 'react-router-dom'; +import { BrowserRouter, Route, Switch } from 'react-router-dom'; -ReactDOM.render( - - -, document.getElementById('root')); +ReactDOM.render( + + + + + + + + + + + + + + + + , +document.getElementById('root')); From b555bb651732a76692a8e369bd87cd6b7860a952 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 5 Jan 2022 02:53:39 -0500 Subject: [PATCH 147/152] add Nov 2021 bus data; fix typo --- server/bus/check_latest_manifests.sh | 2 +- server/bus/setup_bus_input.sh | 2 +- src/constants.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/bus/check_latest_manifests.sh b/server/bus/check_latest_manifests.sh index 5595fccf9..638d8b237 100755 --- a/server/bus/check_latest_manifests.sh +++ b/server/bus/check_latest_manifests.sh @@ -2,7 +2,7 @@ newfile=$1 -for i in 1 111 15 22 23 28 32 39 57 66 71 73 77; do +for i in 1 15 22 23 28 32 39 57 66 71 73 77 111; do mkdir -p data/output/manifests/ pipenv run python manifest.py $newfile data/output/manifests/$i.json --checkpoints data/input/MBTA_GTFS/checkpoints.txt -r $i echo "Comparing old and new manifests for route $i" diff --git a/server/bus/setup_bus_input.sh b/server/bus/setup_bus_input.sh index a405b28f2..33c086af4 100755 --- a/server/bus/setup_bus_input.sh +++ b/server/bus/setup_bus_input.sh @@ -16,7 +16,7 @@ for i in 2021 2020 2019 2018; do done mv 2021/MBTA*/*.csv 2021/ -rmdir 2021/MBTA_Bus_Arrival_Depature_Times_2021/ +rmdir 2021/MBTA_Bus_Arrival_Departure_Times_2021/ # Rename files so they have consistent names and no spaces. mv "2020/Bus Arrival Departure Times Jan-Mar 2020.csv" "2020/2020-Q1.csv" diff --git a/src/constants.js b/src/constants.js index 77af4ce1e..86283ca52 100644 --- a/src/constants.js +++ b/src/constants.js @@ -13,7 +13,7 @@ export const trainDateRange = { export const busDateRange = { minDate: "2018-08-01", - maxDate: "2021-10-31" + maxDate: "2021-11-30" }; export const stations = { From 67ad4982faaedb4a2b6321eae703266eabf2af27 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Wed, 5 Jan 2022 03:30:46 -0500 Subject: [PATCH 148/152] update presets --- src/presets.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/presets.js b/src/presets.js index 13873e880..63b71e5a0 100644 --- a/src/presets.js +++ b/src/presets.js @@ -26,6 +26,14 @@ export const configPresets = [ label: "September 9, 2021 — Route 28 first day of school traffic", value: createConfigPresetValue("28", "Mattapan Station", "Nubian Station", "2021-09-09") }, + { + label: "October 2021 — Route 22 Before Columbus Ave Bus Lanes", + value: createConfigPresetValue("22", "Jackson Square Station", "Franklin Park", "2021-10-01", "2021-10-30") + }, + { + label: "November 2021 — Route 22 After Columbus Ave Bus Lanes", + value: createConfigPresetValue("22", "Jackson Square Station", "Franklin Park", "2021-11-01", "2021-11-30") + } /** OLD STUFF { label: "[New!] April 2021 — Orange Line slow zone", From 938be47d7ba3c069fe7a3dbb74e663948ea9ecfe Mon Sep 17 00:00:00 2001 From: Preston Mueller Date: Sun, 23 Jan 2022 23:33:37 -0500 Subject: [PATCH 149/152] Switch from Google Analytics to GoatCounter (#136) * Switch from Google Analytics to GoatCounter * Update src/App.js Co-authored-by: austinjpaul * remove ga from package-lock Co-authored-by: austinjpaul --- package-lock.json | 5 ----- package.json | 1 - public/index.html | 7 ++++++- src/App.js | 19 ++++++++++++++----- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index a887baf15..02564d041 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13369,11 +13369,6 @@ "resolved": "https://registry.npmjs.org/react-error-overlay/-/react-error-overlay-6.0.9.tgz", "integrity": "sha512-nQTTcUu+ATDbrSD1BZHr5kgSD4oF8OFjxun8uAaL8RwPBacGBNPf/yAuVVdx17N8XNzRDMrZ9XcKZHCjPW+9ew==" }, - "react-ga": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/react-ga/-/react-ga-3.3.0.tgz", - "integrity": "sha512-o8RScHj6Lb8cwy3GMrVH6NJvL+y0zpJvKtc0+wmH7Bt23rszJmnqEQxRbyrqUzk9DTJIHoP42bfO5rswC9SWBQ==" - }, "react-input-autosize": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/react-input-autosize/-/react-input-autosize-3.0.0.tgz", diff --git a/package.json b/package.json index a2aa9f4f2..7ea5a58c3 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,6 @@ "react": "^17.0.0", "react-chartjs-2": "^2.11.1", "react-dom": "^17.0.0", - "react-ga": "^3.3.0", "react-router-dom": "^5.1.2", "react-scripts": "^4.0.3", "react-select": "^4.0.0" diff --git a/public/index.html b/public/index.html index b702ce18c..711b85eb7 100644 --- a/public/index.html +++ b/public/index.html @@ -10,10 +10,15 @@ - + TransitMatters Data Dashboard + + diff --git a/src/App.js b/src/App.js index b23c2271e..f5e27cf83 100644 --- a/src/App.js +++ b/src/App.js @@ -1,5 +1,4 @@ import React from 'react'; -import ReactGA from 'react-ga'; import { SingleDaySet, AggregateSet } from './ChartSets'; import StationConfiguration from './StationConfiguration'; import { withRouter } from 'react-router-dom'; @@ -11,10 +10,12 @@ import './App.css'; import Select from './inputs/Select'; import { configPresets } from './presets'; +const PRODUCTION = "dashboard.transitmatters.org"; + const FRONTEND_TO_BACKEND_MAP = new Map([ ["localhost", ""], // this becomes a relative path that is proxied through CRA:3000 to python on :5000 ["127.0.0.1", ""], - ["dashboard.transitmatters.org", "https://dashboard-api2.transitmatters.org"], + [PRODUCTION, "https://dashboard-api2.transitmatters.org"], ["dashboard-beta.transitmatters.org", "https://dashboard-api-beta.transitmatters.org"] ]); const APP_DATA_BASE_PATH = FRONTEND_TO_BACKEND_MAP.get(window.location.hostname); @@ -116,8 +117,11 @@ class App extends React.Component { progressBarKey: 0, }; - ReactGA.initialize("UA-71173708-2"); - ReactGA.pageview(props.location.pathname); + if (window.location.hostname === PRODUCTION) { + window.goatcounter.count({ + path: props.location.pathname, + }); + } const url_config = new URLSearchParams(props.location.search).get("config"); if (typeof url_config === "string") { @@ -328,7 +332,12 @@ class App extends React.Component { route: configuration.line, }); } - ReactGA.pageview(window.location.pathname + window.location.search); + + if (window.location.hostname === PRODUCTION) { + window.goatcounter.count({ + path: window.location.pathname + window.location.search, + }); + } } } From d4a9933336c7452eb9b4f4dd6c59d9de06aedbb6 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Mon, 24 Jan 2022 00:12:49 -0500 Subject: [PATCH 150/152] debug goatcounter loading --- public/index.html | 2 +- src/App.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/public/index.html b/public/index.html index 711b85eb7..21253857b 100644 --- a/public/index.html +++ b/public/index.html @@ -18,7 +18,7 @@ - + diff --git a/src/App.js b/src/App.js index f5e27cf83..1045d75da 100644 --- a/src/App.js +++ b/src/App.js @@ -118,7 +118,7 @@ class App extends React.Component { }; if (window.location.hostname === PRODUCTION) { - window.goatcounter.count({ + window.goatcounter?.count({ path: props.location.pathname, }); } @@ -334,7 +334,7 @@ class App extends React.Component { } if (window.location.hostname === PRODUCTION) { - window.goatcounter.count({ + window.goatcounter?.count({ path: window.location.pathname + window.location.search, }); } From 40630cef58f7698206d253bccad75ef6933dbcb8 Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Mon, 24 Jan 2022 01:06:27 -0500 Subject: [PATCH 151/152] this is ridiculous --- src/App.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/App.js b/src/App.js index 1045d75da..09221d5f7 100644 --- a/src/App.js +++ b/src/App.js @@ -118,7 +118,7 @@ class App extends React.Component { }; if (window.location.hostname === PRODUCTION) { - window.goatcounter?.count({ + window.goatcounter?.count?.({ path: props.location.pathname, }); } @@ -334,7 +334,7 @@ class App extends React.Component { } if (window.location.hostname === PRODUCTION) { - window.goatcounter?.count({ + window.goatcounter?.count?.({ path: window.location.pathname + window.location.search, }); } From 5fe30f3340660fb8687661a28a861a95eecf732c Mon Sep 17 00:00:00 2001 From: Austin Paul Date: Mon, 24 Jan 2022 22:02:52 -0500 Subject: [PATCH 152/152] update presets for release --- src/presets.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/presets.js b/src/presets.js index 63b71e5a0..33de5d622 100644 --- a/src/presets.js +++ b/src/presets.js @@ -19,20 +19,24 @@ const TODAY = new Date().toISOString().split("T")[0]; export const configPresets = [ { - label: "June to December 2021 — Orange Line slow zones", - value: createConfigPresetValue("Orange", "Downtown Crossing", "Green Street", "2021-06-01", TODAY) + label: "Jan 12, 2022 — Red Line door problem", + value: createConfigPresetValue("Red", "Quincy Adams", "South Station", "2022-01-12") }, { - label: "September 9, 2021 — Route 28 first day of school traffic", + label: "June 2021 to Present — Orange Line slow zones", + value: createConfigPresetValue("Orange", "Downtown Crossing", "Green Street", "2021-06-01", "2022-01-23") + }, + { + label: "Sept 9, 2021 — Route 28 first day of school traffic", value: createConfigPresetValue("28", "Mattapan Station", "Nubian Station", "2021-09-09") }, { - label: "October 2021 — Route 22 Before Columbus Ave Bus Lanes", - value: createConfigPresetValue("22", "Jackson Square Station", "Franklin Park", "2021-10-01", "2021-10-30") + label: "Oct 19, 2021 — Route 1 bus bunching", + value: createConfigPresetValue("1", "Harvard", "Hynes Station", "2021-10-19") }, { - label: "November 2021 — Route 22 After Columbus Ave Bus Lanes", - value: createConfigPresetValue("22", "Jackson Square Station", "Franklin Park", "2021-11-01", "2021-11-30") + label: "Oct to Nov 2021 — Route 22 Columbus Ave bus lane", + value: createConfigPresetValue("22", "Jackson Square Station", "Franklin Park", "2021-10-01", "2021-11-30") } /** OLD STUFF {