Add TTC continued

- add custom default directions
- use our fork of remix/partridge to use a patch for buggy CSVs
- use the TTC route number for sorting on the route-table

backend/agencies/muni.yaml

@@ -6,10 +6,11 @@ gtfs_url: http://gtfs.sfmta.com/transitdata/google_transit.zip
 route_id_gtfs_field: route_short_name
 stop_id_gtfs_field: stop_code
 default_directions:
-  '0':
-    title_prefix: Outbound
-  '1':
-    title_prefix: Inbound
+  - directions:
+      '0':
+        title_prefix: Outbound
+      '1':
+        title_prefix: Inbound
 custom_directions:
   '38':
     - id: "1-48th"

backend/agencies/ttc.yaml

@@ -15,6 +15,7 @@ js_properties:
     '3',
     '4',
   ]
+  routeSortingKey: 'id'
 default_day_start_hour: 4
 custom_day_start_hours:
   - start_hour: 1 # Blue Night Network (300 series) routes run from 1:30-6 am (1:30-8am Sundays)
@@ -27,16 +28,11 @@ custom_day_start_hours:
       '301', '304', '306', '310',
     ]
 default_directions:
-  '0':
-    title_prefix: Southbound
-  '1':
-    title_prefix: Northbound
-custom_default_directions:
-  'east-west':
-    '0':
-      title_prefix: Eastbound
-    '1':
-      title_prefix: Westbound
+  - directions:
+      '0':
+        title_prefix: Eastbound
+      '1':
+        title_prefix: Westbound
     routes: [
       # subway
       '2', '3', '4',
@@ -58,6 +54,11 @@ custom_default_directions:
       '190', '191', '192', '193', '196', '198', '199', '185', '186',
       '514', 
     ]
+  - directions:
+      '0':
+        title_prefix: Southbound
+      '1':
+        title_prefix: Northbound 
 custom_directions:
   '25': # Don Mills
     - id: "25A-NB"

backend/models/config.py

@@ -46,14 +46,12 @@ def __init__(self, conf):
         # by finding the most common GTFS shape_id for each direction_id.
         self.custom_directions = conf.get('custom_directions', {})
 
-        # map of GTFS direction_id (string) to object with metadata about that direction ID.
+        # array of objects containing a directions map (GTFS direction_id (string) to object with metadata about
+        # that direction ID) and a routes list (listing the routes matched to the directions map, by default
+        # all routes are matched to the object)
         # `title_prefix` property will be prepended to the title of the direction for display in the UI.
         self.default_directions = conf.get('default_directions', {})
 
-        # map of custom default direction name to object containing a default_directions object
-        # and routes, a list of route IDs that should use this type of directions.
-        self.custom_default_directions = conf.get('custom_default_directions', {})
-
         self.conf = conf
 
     def get_route_list(self):

backend/models/gtfs.py

@@ -116,7 +116,7 @@ def __init__(self, agency: config.Agency):
         download_gtfs_data(agency, gtfs_cache_dir)
 
         self.feed = ptg.load_geo_feed(gtfs_cache_dir, {})
-
+        print(self.feed.routes.head())
         self.errors = []
         self.stop_times_by_trip = None
         self.stops_df = None
@@ -735,7 +735,8 @@ def get_custom_direction_data(self, custom_direction_info, route_trips_df, route
             if len(excluded_stop_ids) > 0:
                 error_message += f" excluding {','.join(excluded_stop_ids)}"
 
-            self.errors.append(error_message)
+            # Redundant custom directions shouldn't cause an exception
+            # self.errors.append(error_message)
             print(f'  {error_message}')
             return None
         elif len(matching_shapes) > 1:
@@ -753,10 +754,11 @@ def get_custom_direction_data(self, custom_direction_info, route_trips_df, route
             gtfs_shape_id=matching_shape_id,
             gtfs_direction_id=gtfs_direction_id,
             stop_ids=matching_shape['stop_ids'],
-            title=custom_direction_info.get('title', None)
+            route_id=route_id,
+            title=custom_direction_info.get('title', None),
         )
 
-    def get_default_direction_data(self, direction_id, route_trips_df):
+    def get_default_direction_data(self, direction_id, route_trips_df, route_id):
         print(f' default direction = {direction_id}')
 
         route_direction_id_values = route_trips_df['direction_id'].values
@@ -775,21 +777,22 @@ def get_default_direction_data(self, direction_id, route_trips_df):
             id=direction_id,
             gtfs_shape_id=best_shape_id,
             gtfs_direction_id=direction_id,
-            stop_ids=best_shape['stop_ids']
+            stop_ids=best_shape['stop_ids'],
+            route_id=route_id,
         )
 
-    def get_direction_data(self, id, gtfs_shape_id, gtfs_direction_id, stop_ids, title = None):
+    def get_direction_data(self, id, gtfs_shape_id, gtfs_direction_id, stop_ids, route_id, title=None):
         agency = self.agency
         if title is None:
-            default_direction_info = agency.default_directions.get(gtfs_direction_id, {})
-            custom_default_directions = agency.custom_default_directions
-            for custom_default_direction_key in custom_default_directions:
-                custom_default_direction = custom_default_directions[
-                    custom_default_direction_key
-                ]
-                if id in custom_default_direction['routes']:
-                    default_direction_info = custom_default_direction.get(gtfs_direction_id, {})
-            title_prefix = default_direction_info.get('title_prefix', None)
+            # use the first directions map each route matches.
+            title_prefix = None
+            for default_direction in agency.default_directions:
+                if 'routes' not in default_direction or route_id in default_direction['routes']:
+                    title_prefix = default_direction['directions'][id].get(
+                        'title_prefix',
+                        None,
+                    )
+                    break
 
             last_stop_id = stop_ids[-1]
             last_stop = self.get_stop_row(last_stop_id)
@@ -950,7 +953,7 @@ def get_route_data(self, route):
                     route_data['directions'].append(custom_direction_data)
         else:
             route_data['directions'] = [
-                self.get_default_direction_data(direction_id, route_trips_df)
+                self.get_default_direction_data(direction_id, route_trips_df, route_id)
                 for direction_id in np.unique(route_trips_df['direction_id'].values)
             ]
 
@@ -1067,7 +1070,7 @@ def get_sort_key(route_data):
             return route_data['title']
         return sorted(routes_data, key=get_sort_key)
 
-    def save_routes(self, save_to_s3, d):
+    def save_routes(self, save_to_s3, d, included_stop_ids):
         agency = self.agency
         agency_id = agency.id
         routes_df = self.get_gtfs_routes()
@@ -1079,6 +1082,11 @@ def save_routes(self, save_to_s3, d):
             ))
             return
 
+        # Only return routes that we want to include
+        routes_df = routes_df[
+            routes_df[agency.route_id_gtfs_field].isin(included_stop_ids)
+        ]
+
         routes_data = [
             self.get_route_data(route)
             for route in routes_df.itertuples()

backend/models/metrics.py

@@ -569,7 +569,7 @@ def get_average_speed(self, units=constants.MILES_PER_HOUR, scheduled=False):
 
         if first_stop_geometry is None or last_stop_geometry is None:
             raise Exception(
-                f'Missing stop geometry on route {route_id}, {direction_id}, Stop {first_stop_id} to {last_stop_id}'
+                f'Missing stop geometry on route {self.route_id}, {self.direction_id}, Stop {first_stop_id} to {last_stop_id}'
             )
 
         dist = last_stop_geometry['distance'] - first_stop_geometry['distance']

backend/requirements.txt

@@ -13,7 +13,7 @@ Jinja2>=2.10.1
 MarkupSafe==1.1.0
 numpy==1.16.1
 pandas==0.24.1
-partridge==1.1.0
+git+git://github.com/EddyIonescu/partridge.git
 python-dateutil==2.8.0
 pytz==2018.9
 requests==2.22.0

backend/save_routes.py

@@ -42,22 +42,30 @@
     parser.add_argument('--s3', dest='s3', action='store_true', help='store in s3')
     parser.add_argument('--timetables', dest='timetables', action='store_true', help='also save timetables')
     parser.add_argument('--scheduled-stats', dest='scheduled_stats', action='store_true', help='also compute scheduled stats if the timetable has new dates (requires --timetables)')
+    parser.add_argument('--routes', dest='routes', required=False, help='Comma-separated string of routes to include, otherwise include all')
     parser.set_defaults(s3=False)
     parser.set_defaults(timetables=False)
     parser.set_defaults(scheduled_stats=False)
+    parser.set_defaults(routes=None)
 
     args = parser.parse_args()
 
     agencies = [config.get_agency(args.agency)] if args.agency is not None else config.agencies
 
     save_to_s3 = args.s3
+    routes = args.routes
+
     d = date.today()
 
+
     errors = []
 
     for agency in agencies:
         scraper = gtfs.GtfsScraper(agency)
-        scraper.save_routes(save_to_s3, d)
+        include_route_ids = []
+        if routes is not None:
+            include_route_ids = routes.split(',')
+        scraper.save_routes(save_to_s3, d, include_route_ids)
 
         if args.timetables:
             timetables_updated = scraper.save_timetables(save_to_s3=save_to_s3, skip_existing=True)

frontend/src/components/RouteTable.jsx

@@ -15,6 +15,7 @@ import { connect } from 'react-redux';
 import Navlink from 'redux-first-router-link';
 import { filterRoutes } from '../helpers/routeCalculations';
 import DateTimeRangeControls from './DateTimeRangeControls';
+import { Agencies } from '../config';
 
 function getComparisonFunction(order, orderBy) {
   // Sort null values to bottom regardless of ascending/descending
@@ -236,7 +237,10 @@ const useStyles = makeStyles(theme => ({
 function RouteTable(props) {
   const classes = useStyles();
   const [order, setOrder] = React.useState('asc');
-  const [orderBy, setOrderBy] = React.useState('title');
+  const agency = Agencies[0];
+  const [orderBy, setOrderBy] = React.useState(
+    agency.routeSortingKey || 'title',
+  );
   const theme = useTheme();
 
   const { statsByRouteId } = props;
@@ -248,6 +252,7 @@ function RouteTable(props) {
   }
 
   let routes = props.routes ? filterRoutes(props.routes) : [];
+  console.log(routes);
   const spiderLines = props.spiderSelection.nearbyLines;
 
   // filter the route list down to the spider routes if needed