Merge pull request #599 from ibi-group/feature/DT-423-rename-stop-are…

…as-location-groups

Flex spec v2

pom.xml

@@ -271,7 +271,7 @@
         <dependency>
             <groupId>com.github.ibi-group</groupId>
             <artifactId>gtfs-lib</artifactId>
-            <version>2bd924c6e5f5244a279c9d94298c14c0b3d47b35</version>
+            <version>c9908447dad0720cd606c3c753752e6850f7b572</version>
             <!-- Latest dev-flex build on jitpack.io -->
             <!-- Exclusions added in order to silence SLF4J warnings about multiple bindings:
                 http://www.slf4j.org/codes.html#multiple_bindings
@@ -455,6 +455,11 @@
             <artifactId>gson</artifactId>
             <version>2.8.9</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.17.0</version>
+        </dependency>
     </dependencies>
 
 </project>

src/main/java/com/conveyal/datatools/manager/DataManager.java

@@ -210,12 +210,11 @@ static void registerRoutes() throws IOException {
             // NOTE: fare_attributes controller handles updates to nested table fare_rules.
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_ATTRIBUTES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE);
-            // NOTE: AREA and STOP_AREAS are additions for GTFS Flex but belong to the Fares v2 specification:
-            new EditorControllerImpl(EDITOR_API_PREFIX, Table.STOP_AREAS, DataManager.GTFS_DATA_SOURCE);
-            new EditorControllerImpl(EDITOR_API_PREFIX, Table.AREA, DataManager.GTFS_DATA_SOURCE);
             // NOTE: Booking rules, locations and location shapes are GTFS Flex additions.
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.BOOKING_RULES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATIONS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUP, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUP_STOPS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_SHAPES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTES, DataManager.GTFS_DATA_SOURCE);
             // NOTE: Patterns controller handles updates to nested tables shapes, pattern stops, and frequencies.

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java

@@ -40,7 +40,7 @@ public class MergeFeedsResult implements Serializable {
      * Track various table ids for resolving foreign references.
      */
     public Set<String> stopIds = new HashSet<>();
-    public Set<String> stopAreaIds = new HashSet<>();
+    public Set<String> locationGroupStopIds = new HashSet<>();
 
     /**
      * Track the set of route IDs to end up in the merged feed in order to determine which route_attributes

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java

@@ -246,8 +246,8 @@ private void setForeignReferenceKeyValues() {
             case "stops":
                 mergeFeedsResult.stopIds.add(getIdWithScope(keyValue));
                 break;
-            case "stop_areas":
-                mergeFeedsResult.stopAreaIds.add(getIdWithScope(keyValue));
+            case "location_group_stops":
+                mergeFeedsResult.locationGroupStopIds.add(getIdWithScope(keyValue));
                 break;
             default:
                 // nothing.
@@ -272,16 +272,10 @@ private Table getReferenceTable(FieldContext fieldContext, Field field) {
                     getTableScopedValue(Table.CALENDAR_DATES, fieldContext.getValue())
                 );
             case STOP_TIMES_STOP_ID_KEY:
-                return ReferenceTableDiscovery.getStopTimeStopIdReferenceTable(
+            case LOCATION_GROUP_STOPS_STOP_ID_KEY:
+                return ReferenceTableDiscovery.getStopReferenceTable(
                     fieldContext.getValueToWrite(),
-                    mergeFeedsResult,
-                    feedMergeContext.locationIds
-                );
-            case STOP_AREA_STOP_ID_KEY:
-                return ReferenceTableDiscovery.getStopAreaAreaIdReferenceTable(
-                    fieldContext.getValueToWrite(),
-                    mergeFeedsResult,
-                    feedMergeContext.locationIds
+                    mergeFeedsResult
                 );
             default:
                 return null;

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java

@@ -3,7 +3,6 @@
 import com.conveyal.gtfs.loader.Field;
 import com.conveyal.gtfs.loader.Table;
 
-import java.util.Set;
 import java.util.stream.Collectors;
 
 import static com.conveyal.datatools.manager.jobs.feedmerge.MergeLineContext.SERVICE_ID;
@@ -27,23 +26,20 @@ public enum ReferenceTableKey {
                 Table.SCHEDULE_EXCEPTIONS.name
             )
         ),
-        STOP_AREA_STOP_ID_KEY(
+        LOCATION_GROUP_STOPS_STOP_ID_KEY(
             String.join(
                 REF_TABLE_SEPARATOR,
-                Table.STOP_AREAS.name,
+                Table.LOCATION_GROUP_STOPS.name,
                 "stop_id",
-                Table.STOPS.name,
-                Table.LOCATIONS.name
+                Table.STOPS.name
             )
         ),
         STOP_TIMES_STOP_ID_KEY(
             String.join(
                 REF_TABLE_SEPARATOR,
                 Table.STOP_TIMES.name,
                 "stop_id",
-                Table.STOPS.name,
-                Table.LOCATIONS.name,
-                Table.STOP_AREAS.name
+                Table.STOPS.name
             )
         );
 
@@ -112,35 +108,11 @@ public static Table getTripServiceIdReferenceTable(
     }
 
     /**
-     * Define the reference table for a stop area's area id. This will either be a stop or location.
+     * Defines the reference table as a stop if the field value matches a stop id.
      */
-    public static Table getStopAreaAreaIdReferenceTable(
-        String fieldValue,
-        MergeFeedsResult mergeFeedsResult,
-        Set<String> locationIds
-    ) {
-        if (mergeFeedsResult.stopIds.contains(fieldValue)) {
-            return Table.STOPS;
-        } else if (locationIds.contains(fieldValue)) {
-            return Table.LOCATIONS;
-        }
-        return null;
-    }
-
-    /**
-     * Define the reference table for a stop time's stop id. This will either be a stop, location or stop area.
-     */
-    public static Table getStopTimeStopIdReferenceTable(
-        String fieldValue,
-        MergeFeedsResult mergeFeedsResult,
-        Set<String> locationIds
-    ) {
+    public static Table getStopReferenceTable(String fieldValue, MergeFeedsResult mergeFeedsResult) {
         if (mergeFeedsResult.stopIds.contains(fieldValue)) {
             return Table.STOPS;
-        } else if (locationIds.contains(fieldValue)) {
-            return Table.LOCATIONS;
-        } else if (mergeFeedsResult.stopAreaIds.contains(fieldValue)) {
-            return Table.STOP_AREAS;
         }
         return null;
     }

src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java

@@ -97,7 +97,6 @@ void canMergeRegional() throws SQLException {
 
         // Ensure the feed has the row counts we expect.
         assertRowCount(r1.agency, r2.agency, merged.agency, "Agency");
-        assertRowCount(r1.area, r2.area, merged.area, "Area");
         assertRowCount(r1.attributions, r2.attributions, merged.attributions, "Attributions");
         assertRowCount(r1.bookingRules, r2.bookingRules, merged.bookingRules, "Booking rules");
         assertRowCount(r1.calendar, r2.calendar, merged.calendar, "Calendar");
@@ -106,11 +105,12 @@ void canMergeRegional() throws SQLException {
         assertRowCount(r1.fareRules, r2.fareRules, merged.fareRules, "Fare rules");
         assertRowCount(r1.frequencies, r2.frequencies, merged.frequencies, "Frequencies");
         assertRowCount(r1.locations, r2.locations, merged.locations, "Locations");
+        assertRowCount(r1.locationGroup, r2.locationGroup, merged.locationGroup, "Location Groups");
+        assertRowCount(r1.locationGroupStops, r2.locationGroupStops, merged.locationGroupStops, "Location Group Stops");
         assertRowCount(r1.locationShapes, r2.locationShapes, merged.locationShapes, "Location shapes");
         assertRowCount(r1.routes, r2.routes, merged.routes, "Routes");
         assertRowCount(r1.shapes, r2.shapes, merged.shapes, "Shapes");
         assertRowCount(r1.stops, r2.stops, merged.stops, "Stops");
-        assertRowCount(r1.stopAreas, r2.stopAreas, merged.stopAreas, "Stop areas");
         assertRowCount(r1.stopTimes, r2.stopTimes, merged.stopTimes, "Stop times");
         assertRowCount(r1.trips, r2.trips, merged.trips, "Trips");
         assertRowCount(r1.translations, r2.translations, merged.translations, "Translations");

src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java

@@ -102,8 +102,7 @@ void canNormalizeField(TransformationCase transformationCase) throws IOException
     private static Stream<Arguments> createNormalizedFieldCases() {
         return Stream.of(
             Arguments.of(new TransformationCase("routes", "route_long_name", "Route", "Rte")),
-            Arguments.of(new TransformationCase("booking_rules", "pickup_message", "Message", "Msg")),
-            Arguments.of(new TransformationCase("areas", "area_name", "Area", "Place"))
+            Arguments.of(new TransformationCase("booking_rules", "pickup_message", "Message", "Msg"))
         );
     }
 

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/location_group_stops.txt

@@ -0,0 +1,2 @@
+location_group_id,stop_id
+1,123

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/location_groups.txt

@@ -0,0 +1,2 @@
+location_group_id,location_group_name
+1,"Area referencing a stop"

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/locations.geojson

@@ -5,7 +5,7 @@
       "id": "area_1469-2",
       "properties": {
         "stop_name": "Templeboy to Ballisodare",
-        "stop_desc": "Templeboy to Ballisodare Door-to-door pickup area"
+        "stop_desc": "Templeboy to Ballisodare Door-to-door pickup locationGroup"
       },
       "type": "Feature",
       "geometry": {
@@ -25,7 +25,7 @@
       "id": "area_1469-1",
       "properties": {
         "stop_name": "Dromore West to Templeboy",
-        "stop_desc": "Dromore West to Templeboy Door-to-door pickup area"
+        "stop_desc": "Dromore West to Templeboy Door-to-door pickup locationGroup"
       },
       "type": "Feature",
       "geometry": {
@@ -172,7 +172,7 @@
 	  "id": "area_246",
       "properties": {
         "stop_name": "Moville to Letterkenny",
-        "stop_desc": "Moville to Letterkenny Door-to-door pickup area"
+        "stop_desc": "Moville to Letterkenny Door-to-door pickup locationGroup"
       },
       "type": "Feature",
       "geometry": {
@@ -187,7 +187,7 @@
       "id": "area_251",
       "properties": {
         "stop_name": "Muff to Moville",
-        "stop_desc": "Muff to Moville Door-to-door pickup area"
+        "stop_desc": "Muff to Moville Door-to-door pickup locationGroup"
       },
       "type": "Feature",
       "geometry": {

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt

@@ -1,7 +1,7 @@
-trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window,mean_duration_factor,mean_duration_offset,safe_duration_factor,safe_duration_offset
-a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1,60,1,60
-a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00,1,60,1,60
-frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00,1,60,1,60
-frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60
-frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60
-frequency-trip,,,3,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00,1,60,1,60
+trip_id,arrival_time,departure_time,stop_id,location_group_id,location_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window
+a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,,,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1
+a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,,,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00
+frequency-trip,08:00:00,08:00:00,4u6g,,,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00
+frequency-trip,08:29:00,08:29:00,1234,,,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00
+frequency-trip,08:31:00,08:31:00,,,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00
+frequency-trip,,,,1,,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/location_group_stops.txt

@@ -0,0 +1,2 @@
+location_group_id,stop_id
+1,123

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/location_groups.txt

@@ -0,0 +1,2 @@
+location_group_id,location_group_name
+1,"Area referencing a stop"

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/locations.geojson

@@ -5,7 +5,7 @@
       "id": "area_1469-2",
       "properties": {
         "stop_name": "Templeboy to Ballisodare",
-        "stop_desc": "Templeboy to Ballisodare Door-to-door pickup area"
+        "stop_desc": "Templeboy to Ballisodare Door-to-door pickup locationGroup"
       },
       "type": "Feature",
       "geometry": {
@@ -25,7 +25,7 @@
       "id": "area_1469-1",
       "properties": {
         "stop_name": "Dromore West to Templeboy",
-        "stop_desc": "Dromore West to Templeboy Door-to-door pickup area"
+        "stop_desc": "Dromore West to Templeboy Door-to-door pickup locationGroup"
       },
       "type": "Feature",
       "geometry": {
@@ -172,7 +172,7 @@
 	  "id": "area_246",
       "properties": {
         "stop_name": "Moville to Letterkenny",
-        "stop_desc": "Moville to Letterkenny Door-to-door pickup area"
+        "stop_desc": "Moville to Letterkenny Door-to-door pickup locationGroup"
       },
       "type": "Feature",
       "geometry": {
@@ -187,7 +187,7 @@
       "id": "area_251",
       "properties": {
         "stop_name": "Muff to Moville",
-        "stop_desc": "Muff to Moville Door-to-door pickup area"
+        "stop_desc": "Muff to Moville Door-to-door pickup locationGroup"
       },
       "type": "Feature",
       "geometry": {

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt

@@ -1,7 +1,7 @@
-trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window,mean_duration_factor,mean_duration_offset,safe_duration_factor,safe_duration_offset
-a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1,60,1,60
-a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00,1,60,1,60
-frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00,1,60,1,60
-frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60
-frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60
-frequency-trip,,,3,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00,1,60,1,60
+trip_id,arrival_time,departure_time,stop_id,location_group_id,location_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window
+a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,,,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1
+a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,,,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00
+frequency-trip,08:00:00,08:00:00,4u6g,,,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00
+frequency-trip,08:29:00,08:29:00,1234,,,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00
+frequency-trip,08:31:00,08:31:00,,,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00
+frequency-trip,,,,1,,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00