[Draft] Changes needed to add TTC

backend/agencies/muni.yaml

@@ -6,10 +6,11 @@ gtfs_url: http://gtfs.sfmta.com/transitdata/google_transit.zip
 route_id_gtfs_field: route_short_name
 stop_id_gtfs_field: stop_code
 default_directions:
-  '0':
-    title_prefix: Outbound
-  '1':
-    title_prefix: Inbound
+  - directions:
+      '0':
+        title_prefix: Outbound
+      '1':
+        title_prefix: Inbound
 custom_directions:
   '38':
     - id: "1-48th"

backend/agencies/ttc.yaml

@@ -3,8 +3,100 @@ provider: nextbus
 nextbus_agency_id: ttc
 gtfs_url: http://opendata.toronto.ca/TTC/routes/OpenData_TTC_Schedules.zip
 route_id_gtfs_field: route_short_name
+stop_id_gtfs_field: stop_code
 timezone_id: America/Toronto
 js_properties:
   title: TTC
   initialMapCenter: { lat: 43.684, lng: -79.39 }
   initialMapZoom: 12
+  default_disabled_routes: [ # exclude subway lines as they're not on nextbus
+    '1',
+    '2',
+    '3',
+    '4',
+  ]
+  routeSortingKey: 'id'
+default_day_start_hour: 4
+custom_day_start_hours:
+  - start_hour: 1 # Blue Night Network (300 series) routes run from 1:30-6 am (1:30-8am Sundays)
+    routes: [
+      # bus
+      '300', '302', '307', '312', '315', '320', '322', '324', '325',
+      '329', '332', '334', '335', '336', '337', '339', '341', '343',
+      '352', '353', '354', '363', '365', '384', '385', '395', '396',
+      # streetcar
+      '301', '304', '306', '310',
+    ]
+default_directions:
+  - directions:
+      '0':
+        title_prefix: Eastbound
+      '1':
+        title_prefix: Westbound
+    routes: [
+      # subway
+      '2', '3', '4',
+      # bus
+      '8', '10', '12', '14', '15', '23', '26', '32', '34',
+      '36', '38', '39', '40', '42', '48', '49', '50', '52', '53', '54',
+      '59', '60', '62', '78', '82', '84', '85', '86', '87', '94', '95', '96',
+      '98', '107', '108', '109', '113', '115', '116', '117', '119', '120', '121',
+      '122', '123', '124', '125', '127', '130', '131', '132', '134', '135',
+      '143', '145', '161', '162', '165', '171', '176', '189', '300', '312',
+      '315', '332', '334', '336', '339', '352', '353', '354', '384', '385',
+      '395', '396', '905', '913', '939', '952', '953', '954', '960', '984',
+      '985', '986', '995', '996',
+      # streetcar
+      '301', '304', '306',
+      '501', '502', '503', '504', '505', '506', '508', '509', '512',
+      # discontinued routes (exist in old OpenTransit data), for use in versioned
+      # ttc configs as custom_default_directions can be used for all versions.
+      '190', '191', '192', '193', '196', '198', '199', '185', '186',
+      '514', 
+    ]
+  - directions:
+      '0':
+        title_prefix: Southbound
+      '1':
+        title_prefix: Northbound 
+custom_directions:
+  '25': # Don Mills
+    - id: "25A-NB"
+      title: "25A Northbound to Steeles via Don Mills Stn"
+      gtfs_direction_id: "1"
+      included_stop_ids: ["1977", "14625", "1975"]
+      # Wynford (South of Line 4), Van Horne (North of Line 4), and bay at Don Mills Stn
+    - id: "25B-NB"
+      title: "25B Northbound to Don Mills Stn"
+      gtfs_direction_id: "1"
+      included_stop_ids: ["1977"]
+      excluded_stop_ids: ["1975", "14625"] # exclude bay at Don Mills Stn
+    - id: "25C-NB"
+      title: "25C Northbound to Steeles"
+      gtfs_direction_id: "1"
+      included_stop_ids: ["1975"]
+      excluded_stop_ids: ["1977"]
+    - id: "25A-NB-ST"
+      title: "25A Northbound to Steeles from Fenelon Loop (School Tripper)"
+      gtfs_direction_id: "1"
+      included_stop_ids: ["15056"]
+    - id: "25A-SB"
+      title: "25A Southbound to Pape Stn via Don Mills Stn"
+      gtfs_direction_id: "0"
+      included_stop_ids: ["1976", "1978"] # Van Horne and Wynford
+      excluded_stop_ids: ["1950"]
+    - id: "25B-SB"
+      title: "25B Southbound to Pape Stn"
+      gtfs_direction_id: "0"
+      included_stop_ids: ["1978"]
+      excluded_stop_ids: ["1976"]
+    - id: "25C-SB"
+      title: "25C Southbound to Don Mills Stn"
+      gtfs_direction_id: "0"
+      included_stop_ids: ["1976"]
+      excluded_stop_ids: ["1978"]
+    - id: "25B-SB-ST"
+      title: "25B Southbound to Pape Stn from Fenelon Loop (School Tripper)"
+      gtfs_direction_id: "0"
+      included_stop_ids: ["15056", "1978"]
+

backend/models/config.py

@@ -46,7 +46,9 @@ def __init__(self, conf):
         # by finding the most common GTFS shape_id for each direction_id.
         self.custom_directions = conf.get('custom_directions', {})
 
-        # map of GTFS direction_id (string) to object with metadata about that direction ID.
+        # array of objects containing a directions map (GTFS direction_id (string) to object with metadata about
+        # that direction ID) and a routes list (listing the routes matched to the directions map, by default
+        # all routes are matched to the object)
         # `title_prefix` property will be prepended to the title of the direction for display in the UI.
         self.default_directions = conf.get('default_directions', {})
 

backend/models/gtfs.py

@@ -116,7 +116,7 @@ def __init__(self, agency: config.Agency):
         download_gtfs_data(agency, gtfs_cache_dir)
 
         self.feed = ptg.load_geo_feed(gtfs_cache_dir, {})
-
+        print(self.feed.routes.head())
         self.errors = []
         self.stop_times_by_trip = None
         self.stops_df = None
@@ -735,7 +735,8 @@ def get_custom_direction_data(self, custom_direction_info, route_trips_df, route
             if len(excluded_stop_ids) > 0:
                 error_message += f" excluding {','.join(excluded_stop_ids)}"
 
-            self.errors.append(error_message)
+            # Redundant custom directions shouldn't cause an exception
+            # self.errors.append(error_message)
             print(f'  {error_message}')
             return None
         elif len(matching_shapes) > 1:
@@ -753,10 +754,11 @@ def get_custom_direction_data(self, custom_direction_info, route_trips_df, route
             gtfs_shape_id=matching_shape_id,
             gtfs_direction_id=gtfs_direction_id,
             stop_ids=matching_shape['stop_ids'],
-            title=custom_direction_info.get('title', None)
+            route_id=route_id,
+            title=custom_direction_info.get('title', None),
         )
 
-    def get_default_direction_data(self, direction_id, route_trips_df):
+    def get_default_direction_data(self, direction_id, route_trips_df, route_id):
         print(f' default direction = {direction_id}')
 
         route_direction_id_values = route_trips_df['direction_id'].values
@@ -775,14 +777,22 @@ def get_default_direction_data(self, direction_id, route_trips_df):
             id=direction_id,
             gtfs_shape_id=best_shape_id,
             gtfs_direction_id=direction_id,
-            stop_ids=best_shape['stop_ids']
+            stop_ids=best_shape['stop_ids'],
+            route_id=route_id,
         )
 
-    def get_direction_data(self, id, gtfs_shape_id, gtfs_direction_id, stop_ids, title = None):
+    def get_direction_data(self, id, gtfs_shape_id, gtfs_direction_id, stop_ids, route_id, title=None):
         agency = self.agency
         if title is None:
-            default_direction_info = agency.default_directions.get(gtfs_direction_id, {})
-            title_prefix = default_direction_info.get('title_prefix', None)
+            # use the first directions map each route matches.
+            title_prefix = None
+            for default_direction in agency.default_directions:
+                if 'routes' not in default_direction or route_id in default_direction['routes']:
+                    title_prefix = default_direction['directions'][id].get(
+                        'title_prefix',
+                        None,
+                    )
+                    break
 
             last_stop_id = stop_ids[-1]
             last_stop = self.get_stop_row(last_stop_id)
@@ -869,9 +879,11 @@ def project_xy(lon, lat, z=None):
 
             stop_geometry = get_stop_geometry(stop_xy, shape_lines_xy, shape_cumulative_dist, start_index)
 
-            if stop_geometry['offset'] > 100:
-                print(f"    !! bad geometry for stop {stop_id}: {stop_geometry['offset']} m from route line segment")
-                continue
+            if stop_geometry['offset'] > 300:
+                # Throw as skipping it will result in speed metrics API calls failing
+                raise Exception(
+                    f"Bad geometry for stop {stop_id}: {stop_geometry['offset']}m from route line segment"
+                )
 
             dir_data['stop_geometry'][stop_id] = stop_geometry
 
@@ -941,7 +953,7 @@ def get_route_data(self, route):
                     route_data['directions'].append(custom_direction_data)
         else:
             route_data['directions'] = [
-                self.get_default_direction_data(direction_id, route_trips_df)
+                self.get_default_direction_data(direction_id, route_trips_df, route_id)
                 for direction_id in np.unique(route_trips_df['direction_id'].values)
             ]
 
@@ -1058,7 +1070,7 @@ def get_sort_key(route_data):
             return route_data['title']
         return sorted(routes_data, key=get_sort_key)
 
-    def save_routes(self, save_to_s3, d):
+    def save_routes(self, save_to_s3, d, included_stop_ids):
         agency = self.agency
         agency_id = agency.id
         routes_df = self.get_gtfs_routes()
@@ -1070,6 +1082,11 @@ def save_routes(self, save_to_s3, d):
             ))
             return
 
+        # Only return routes that we want to include
+        routes_df = routes_df[
+            routes_df[agency.route_id_gtfs_field].isin(included_stop_ids)
+        ]
+
         routes_data = [
             self.get_route_data(route)
             for route in routes_df.itertuples()

backend/models/metrics.py

@@ -568,7 +568,9 @@ def get_average_speed(self, units=constants.MILES_PER_HOUR, scheduled=False):
         last_stop_geometry = dir_info.get_stop_geometry(last_stop_id)
 
         if first_stop_geometry is None or last_stop_geometry is None:
-            return None
+            raise Exception(
+                f'Missing stop geometry on route {self.route_id}, {self.direction_id}, Stop {first_stop_id} to {last_stop_id}'
+            )
 
         dist = last_stop_geometry['distance'] - first_stop_geometry['distance']
         if dist <= 0:

backend/models/trynapi.py

@@ -287,7 +287,12 @@ def get_state_raw(agency_id, start_time, end_time, route_ids):
     print(params)
 
     query_url = f"{trynapi_url}/graphql?query={query}"
-    r = requests.get(query_url)
+    try:
+        r = requests.get(query_url)
+    except Exception as exc:
+        print(exc)
+        print('Ensure tryn-api is running and that you set the TRYNAPI_URL environment variable')
+        exit(1)
 
     print(f"   response length = {len(r.text)}")
 

backend/requirements.txt

@@ -13,7 +13,7 @@ Jinja2>=2.10.1
 MarkupSafe==1.1.0
 numpy==1.16.1
 pandas==0.24.1
-partridge==1.1.0
+git+git://github.com/EddyIonescu/partridge.git
 python-dateutil==2.8.0
 pytz==2018.9
 requests==2.22.0

backend/save_routes.py

@@ -42,22 +42,30 @@
     parser.add_argument('--s3', dest='s3', action='store_true', help='store in s3')
     parser.add_argument('--timetables', dest='timetables', action='store_true', help='also save timetables')
     parser.add_argument('--scheduled-stats', dest='scheduled_stats', action='store_true', help='also compute scheduled stats if the timetable has new dates (requires --timetables)')
+    parser.add_argument('--routes', dest='routes', required=False, help='Comma-separated string of routes to include, otherwise include all')
     parser.set_defaults(s3=False)
     parser.set_defaults(timetables=False)
     parser.set_defaults(scheduled_stats=False)
+    parser.set_defaults(routes=None)
 
     args = parser.parse_args()
 
     agencies = [config.get_agency(args.agency)] if args.agency is not None else config.agencies
 
     save_to_s3 = args.s3
+    routes = args.routes
+
     d = date.today()
 
+
     errors = []
 
     for agency in agencies:
         scraper = gtfs.GtfsScraper(agency)
-        scraper.save_routes(save_to_s3, d)
+        include_route_ids = []
+        if routes is not None:
+            include_route_ids = routes.split(',')
+        scraper.save_routes(save_to_s3, d, include_route_ids)
 
         if args.timetables:
             timetables_updated = scraper.save_timetables(save_to_s3=save_to_s3, skip_existing=True)

docs/agencies.md

@@ -39,6 +39,26 @@ default_directions:
     title_prefix: Inbound
 ```
 
+`custom_default_directions` - an extension of `default_directions`; it is an optional object with keys for each `default_directions` object you provide that also contains a `route` field where you provide the list of routes that the rule should apply to. This field can be provided in addition to `default_directions`. In the example below, a direction_id of '0' can have multiple meanings such as Outbound or Eastbound depending on the route:
+```
+  'inbound-outbound':
+    '0':
+      title_prefix: Outbound
+    '1':
+      title_prefix: Inbound
+    routes: [
+      'KT', 'L', 'M', 'N', 'J', 'E', 'F'
+    ]
+  'east-west':
+    '0':
+      title_prefix: Eastbound
+    '1':
+      title_prefix: Westbound
+    routes: [
+      '2', '3', '4', '501', '502', '503', '504', '505', '506', '508', '509', '512'
+    ]
+```
+
 `custom_directions` - an optional object that allows manually defining directions for certain routes, in order to support routes with multiple branches. Each key in this object is a route ID, and the value is an array of directions for that route, with the properties `id`, `title` (optional), `gtfs_direction_id`, `included_stop_ids` (optional), and `excluded_stop_ids` (optional).
 
 The `gtfs_direction_id`, `included_stop_ids`, and `excluded_stop_ids` properties are used to filter the shape_ids referred to by trips.txt in the GTFS feed. Each custom direction should match at least one shape in the GTFS feed. If a custom direction can be matched to multiple shapes, it is matched to the one associated with the most trips in the GTFS feed. If a custom direction cannot be matched to any shapes, the direction will be omitted and an exception will be raised by save_routes.py after saving the new routes file. If included_stop_ids contains multiple stop IDs, they must be listed in order.

frontend/src/components/RouteTable.jsx

@@ -15,6 +15,7 @@ import { connect } from 'react-redux';
 import Navlink from 'redux-first-router-link';
 import { filterRoutes } from '../helpers/routeCalculations';
 import DateTimeRangeControls from './DateTimeRangeControls';
+import { Agencies } from '../config';
 
 function getComparisonFunction(order, orderBy) {
   // Sort null values to bottom regardless of ascending/descending
@@ -236,7 +237,10 @@ const useStyles = makeStyles(theme => ({
 function RouteTable(props) {
   const classes = useStyles();
   const [order, setOrder] = React.useState('asc');
-  const [orderBy, setOrderBy] = React.useState('title');
+  const agency = Agencies[0];
+  const [orderBy, setOrderBy] = React.useState(
+    agency.routeSortingKey || 'title',
+  );
   const theme = useTheme();
 
   const { statsByRouteId } = props;
@@ -248,6 +252,7 @@ function RouteTable(props) {
   }
 
   let routes = props.routes ? filterRoutes(props.routes) : [];
+  console.log(routes);
   const spiderLines = props.spiderSelection.nearbyLines;
 
   // filter the route list down to the spider routes if needed