GTFS-RT extension to add completely new routes

docs/BuildConfiguration.md

@@ -51,6 +51,7 @@ Sections follow that describe particular settings in more depth.
 | [transitServiceEnd](#transitServiceEnd)                                  |  `duration` | Limit the import of transit services to the given end date.                                                                                                    | *Optional* | `"P3Y"`                           |  2.0  |
 | [transitServiceStart](#transitServiceStart)                              |  `duration` | Limit the import of transit services to the given START date.                                                                                                  | *Optional* | `"-P1Y"`                          |  2.0  |
 | [writeCachedElevations](#writeCachedElevations)                          |  `boolean`  | Reusing elevation data from previous builds                                                                                                                    | *Optional* | `false`                           |  2.0  |
+| [alwaysGenerateTransfersFeeds](#alwaysGenerateTransfersFeeds)            |  `string[]` | Always generate transfers to stops in this feed even when they don't have a trip pattern using them.                                                           | *Optional* |                                   |  2.3  |
 | [boardingLocationTags](#boardingLocationTags)                            |  `string[]` | What OSM tags should be looked on for the source of matching stops to platforms and stops.                                                                     | *Optional* |                                   |  2.2  |
 | [dataOverlay](sandbox/DataOverlay.md)                                    |   `object`  | Config for the DataOverlay Sandbox module                                                                                                                      | *Optional* |                                   |  2.2  |
 | [dem](#dem)                                                              |  `object[]` | Specify parameters for DEM extracts.                                                                                                                           | *Optional* |                                   |  2.2  |
@@ -689,6 +690,19 @@ become stale and not include new OSM ways. Therefore, periodic update of this ca
 recommended.
 
 
+<h3 id="alwaysGenerateTransfersFeeds">alwaysGenerateTransfersFeeds</h3>
+
+**Since version:** `2.3` ∙ **Type:** `string[]` ∙ **Cardinality:** `Optional`   
+**Path:** / 
+
+Always generate transfers to stops in this feed even when they don't have a trip pattern using them.
+
+By default OTP doesn't generate a transfer to a stop that doesn't have a trip pattern using it.
+However, there are feeds that have stops that exist just so that realtime updaters can create
+trips to them on the fly.
+In such a case you _do_ want to generate the transfers anyway.
+
+
 <h3 id="boardingLocationTags">boardingLocationTags</h3>
 
 **Since version:** `2.2` ∙ **Type:** `string[]` ∙ **Cardinality:** `Optional`   

src/main/java/org/opentripplanner/framework/io/HttpUtils.java

@@ -60,10 +60,6 @@ public static InputStream getData(URI uri, Map<String, String> requestHeaderValu
     return getData(uri, DEFAULT_TIMEOUT, requestHeaderValues);
   }
 
-  public static List<Header> getHeaders(URI uri) {
-    return getHeaders(uri, DEFAULT_TIMEOUT, null);
-  }
-
   public static List<Header> getHeaders(
     URI uri,
     Duration timeout,

src/main/java/org/opentripplanner/graph_builder/module/DirectTransferGenerator.java

@@ -6,6 +6,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.opentripplanner.framework.application.OTPFeature;
 import org.opentripplanner.framework.logging.ProgressTracker;
@@ -45,19 +46,32 @@ public class DirectTransferGenerator implements GraphBuilderModule {
   private final Graph graph;
   private final TransitModel transitModel;
   private final DataImportIssueStore issueStore;
+  private final Set<String> feedsToExcludeFromPatternCheck;
 
   public DirectTransferGenerator(
     Graph graph,
     TransitModel transitModel,
     DataImportIssueStore issueStore,
     Duration radiusByDuration,
     List<RouteRequest> transferRequests
+  ) {
+    this(graph, transitModel, issueStore, radiusByDuration, transferRequests, Set.of());
+  }
+
+  public DirectTransferGenerator(
+    Graph graph,
+    TransitModel transitModel,
+    DataImportIssueStore issueStore,
+    Duration radiusByDuration,
+    List<RouteRequest> transferRequests,
+    Set<String> feedsToExcludeFromPatternCheck
   ) {
     this.graph = graph;
     this.transitModel = transitModel;
     this.issueStore = issueStore;
     this.radiusByDuration = radiusByDuration;
     this.transferRequests = transferRequests;
+    this.feedsToExcludeFromPatternCheck = feedsToExcludeFromPatternCheck;
   }
 
   @Override
@@ -72,7 +86,8 @@ public void buildGraph() {
       new DefaultTransitService(transitModel),
       radiusByDuration,
       null,
-      graph.hasStreets
+      graph.hasStreets,
+      feedsToExcludeFromPatternCheck
     );
     if (nearbyStopFinder.useStreets) {
       LOG.info("Creating direct transfer edges between stops using the street network from OSM...");

src/main/java/org/opentripplanner/graph_builder/module/NearbyStopFinder.java

@@ -64,6 +64,8 @@ public class NearbyStopFinder {
   private final Duration durationLimit;
   private final DataOverlayContext dataOverlayContext;
 
+  private final Set<String> feedsToExcludeFromPatternCheck;
+
   private DirectGraphFinder directGraphFinder;
 
   /**
@@ -76,12 +78,14 @@ public NearbyStopFinder(
     TransitService transitService,
     Duration durationLimit,
     DataOverlayContext dataOverlayContext,
-    boolean useStreets
+    boolean useStreets,
+    Set<String> feedsToExcludeFromPatternCheck
   ) {
     this.transitService = transitService;
     this.dataOverlayContext = dataOverlayContext;
     this.useStreets = useStreets;
     this.durationLimit = durationLimit;
+    this.feedsToExcludeFromPatternCheck = feedsToExcludeFromPatternCheck;
 
     if (!useStreets) {
       // We need to accommodate straight line distance (in meters) but when streets are present we
@@ -109,6 +113,11 @@ public Set<NearbyStop> findNearbyStopsConsideringPatterns(
     /* Track the closest stop on each flex trip nearby. */
     MinMap<FlexTrip<?, ?>, NearbyStop> closestStopForFlexTrip = new MinMap<>();
 
+    // Generally we don't want to generate a transfer when there are no patterns at the stop, however
+    // there are special stops which are purely there so you can add a trip via a realtime updater
+    // so for those you _do_ want to generate a transfer.
+    List<NearbyStop> overrideStops = new ArrayList<>();
+
     /* Iterate over nearby stops via the street network or using straight-line distance, depending on the graph. */
     for (NearbyStop nearbyStop : findNearbyStops(
       vertex,
@@ -130,6 +139,11 @@ public Set<NearbyStop> findNearbyStopsConsideringPatterns(
           }
         }
       }
+
+      if (feedsToExcludeFromPatternCheck.contains(ts1.getId().getFeedId())) {
+        overrideStops.add(nearbyStop);
+      }
+
       if (OTPFeature.FlexRouting.isOn()) {
         for (FlexTrip<?, ?> trip : transitService.getFlexIndex().getFlexTripsByStop(ts1)) {
           if (
@@ -147,6 +161,7 @@ public Set<NearbyStop> findNearbyStopsConsideringPatterns(
     Set<NearbyStop> uniqueStops = new HashSet<>();
     uniqueStops.addAll(closestStopForFlexTrip.values());
     uniqueStops.addAll(closestStopForPattern.values());
+    uniqueStops.addAll(overrideStops);
     return uniqueStops;
   }
 

src/main/java/org/opentripplanner/graph_builder/module/configure/GraphBuilderModules.java

@@ -204,7 +204,8 @@ static DirectTransferGenerator provideDirectTransferGenerator(
       transitModel,
       issueStore,
       maxTransferDuration,
-      config.transferRequests
+      config.transferRequests,
+      config.alwaysGenerateTransfersFeeds
     );
   }
 

src/main/java/org/opentripplanner/routing/algorithm/raptoradapter/router/street/AccessEgressRouter.java

@@ -2,6 +2,7 @@
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 import org.opentripplanner.ext.dataoverlay.routing.DataOverlayContext;
 import org.opentripplanner.graph_builder.module.NearbyStopFinder;
 import org.opentripplanner.routing.api.request.RouteRequest;
@@ -38,7 +39,8 @@ public static Collection<NearbyStop> streetSearch(
       transitService,
       request.preferences().street().maxAccessEgressDuration().valueOf(streetRequest.mode()),
       dataOverlayContext,
-      true
+      true,
+      Set.of()
     );
     List<NearbyStop> nearbyStopList = nearbyStopFinder.findNearbyStopsViaStreets(
       fromTarget ? verticesContainer.getToVertices() : verticesContainer.getFromVertices(),

src/main/java/org/opentripplanner/standalone/config/BuildConfig.java

@@ -5,6 +5,7 @@
 import static org.opentripplanner.standalone.config.framework.json.OtpVersion.V2_0;
 import static org.opentripplanner.standalone.config.framework.json.OtpVersion.V2_1;
 import static org.opentripplanner.standalone.config.framework.json.OtpVersion.V2_2;
+import static org.opentripplanner.standalone.config.framework.json.OtpVersion.V2_3;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.MissingNode;
@@ -165,6 +166,7 @@ public class BuildConfig implements OtpDataStoreConfig {
   public final DemExtractParametersList dem;
   public final OsmExtractParametersList osm;
   public final TransitFeeds transitFeeds;
+  public final Set<String> alwaysGenerateTransfersFeeds;
   public boolean staticParkAndRide;
   public boolean staticBikeParkAndRide;
   public double distanceBetweenElevationSamples;
@@ -648,6 +650,22 @@ that we support remote input files (cloud storage or arbitrary URLs) not all dat
       """
         )
         .asUri(null);
+    alwaysGenerateTransfersFeeds =
+      root
+        .of("alwaysGenerateTransfersFeeds")
+        .since(V2_3)
+        .summary(
+          "Always generate transfers to stops in this feed even when they don't have a trip pattern using them."
+        )
+        .description(
+          """
+      By default OTP doesn't generate a transfer to a stop that doesn't have a trip pattern using it.
+      However, there are feeds that have stops that exist just so that realtime updaters can create
+      trips to them on the fly.
+      In such a case you _do_ want to generate the transfers anyway.
+      """
+        )
+        .asStringSet(Set.of());
 
     osmDefaults = OsmConfig.mapOsmDefaults(root, "osmDefaults");
     osm = OsmConfig.mapOsmConfig(root, "osm", osmDefaults);

src/main/java/org/opentripplanner/transit/model/network/StopPattern.java

@@ -168,11 +168,11 @@ public StopLocation getStop(int stopPosInPattern) {
     return stops[stopPosInPattern];
   }
 
-  PickDrop getPickup(int stopPosInPattern) {
+  public PickDrop getPickup(int stopPosInPattern) {
     return pickups[stopPosInPattern];
   }
 
-  PickDrop getDropoff(int stopPosInPattern) {
+  public PickDrop getDropoff(int stopPosInPattern) {
     return dropoffs[stopPosInPattern];
   }
 

src/main/java/org/opentripplanner/updater/trip/AddedRoute.java

@@ -0,0 +1,59 @@
+package org.opentripplanner.updater.trip;
+
+import com.google.transit.realtime.GtfsRealtime;
+import de.mfdz.MfdzRealtimeExtensions;
+import java.util.Objects;
+import org.opentripplanner.transit.model.organization.Agency;
+
+final class AddedRoute {
+
+  //bus
+  public static final int FALLBACK_ROUTE_TYPE = 3;
+  private final String routeUrl;
+  private final String agencyId;
+  private final Integer routeType;
+  private final String routeLongName;
+
+  public AddedRoute(String routeUrl, String agencyId, Integer routeType, String routeLongName) {
+    this.routeUrl = routeUrl;
+    this.agencyId = agencyId;
+    this.routeType = routeType;
+    this.routeLongName = routeLongName;
+  }
+
+  public boolean matchesAgencyId(Agency agency) {
+    return agencyId != null && agencyId.equals(agency.getId().getId());
+  }
+
+  /**
+   * If the route type is not defined in {@see MfdzRealtimeExtensions} then we fall back to 3 (bus).
+   */
+  public int routeType() {
+    return Objects.requireNonNullElse(routeType, FALLBACK_ROUTE_TYPE);
+  }
+
+  public String routeUrl() {
+    return routeUrl;
+  }
+
+  public String agencyId() {
+    return agencyId;
+  }
+
+  public String routeLongName() {
+    return routeLongName;
+  }
+
+  static AddedRoute ofTripDescriptor(GtfsRealtime.TripDescriptor tripDescriptor) {
+    if (tripDescriptor.hasExtension(MfdzRealtimeExtensions.tripDescriptor)) {
+      var ext = tripDescriptor.getExtension(MfdzRealtimeExtensions.tripDescriptor);
+      var url = ext.getRouteUrl();
+      var agencyId = ext.getAgencyId();
+      var routeType = ext.getRouteType();
+      var routeName = ext.getRouteLongName();
+      return new AddedRoute(url, agencyId, routeType, routeName);
+    } else {
+      return new AddedRoute(null, null, null, null);
+    }
+  }
+}

src/main/java/org/opentripplanner/updater/trip/AddedStopTime.java

@@ -0,0 +1,54 @@
+package org.opentripplanner.updater.trip;
+
+import com.google.transit.realtime.GtfsRealtime;
+import de.mfdz.MfdzRealtimeExtensions;
+import de.mfdz.MfdzRealtimeExtensions.StopTimePropertiesExtension;
+import java.util.Objects;
+import javax.annotation.Nullable;
+import org.opentripplanner.model.PickDrop;
+
+final class AddedStopTime {
+
+  @Nullable
+  private final PickDrop pickup;
+
+  @Nullable
+  private final PickDrop dropOff;
+
+  public static final PickDrop DEFAULT_PICK_DROP = PickDrop.SCHEDULED;
+
+  AddedStopTime(@Nullable PickDrop pickup, @Nullable PickDrop dropOff) {
+    this.pickup = pickup;
+    this.dropOff = dropOff;
+  }
+
+  PickDrop pickup() {
+    return Objects.requireNonNullElse(pickup, DEFAULT_PICK_DROP);
+  }
+
+  PickDrop dropOff() {
+    return Objects.requireNonNullElse(dropOff, DEFAULT_PICK_DROP);
+  }
+
+  static AddedStopTime ofStopTimeProperties(
+    GtfsRealtime.TripUpdate.StopTimeUpdate.StopTimeProperties props
+  ) {
+    if (props.hasExtension(MfdzRealtimeExtensions.stopTimeProperties)) {
+      var ext = props.getExtension(MfdzRealtimeExtensions.stopTimeProperties);
+      var pickup = ext.getPickupType();
+      var dropOff = ext.getDropoffType();
+      return new AddedStopTime(toPickDrop(pickup), toPickDrop(dropOff));
+    } else {
+      return new AddedStopTime(null, null);
+    }
+  }
+
+  private static PickDrop toPickDrop(StopTimePropertiesExtension.DropOffPickupType gtfs) {
+    return switch (gtfs) {
+      case REGULAR -> PickDrop.SCHEDULED;
+      case NONE -> PickDrop.NONE;
+      case PHONE_AGENCY -> PickDrop.CALL_AGENCY;
+      case COORDINATE_WITH_DRIVER -> PickDrop.COORDINATE_WITH_DRIVER;
+    };
+  }
+}

src/main/java/org/opentripplanner/updater/trip/GtfsRealtimeHttpTripUpdateSource.java

@@ -1,9 +1,11 @@
 package org.opentripplanner.updater.trip;
 
+import com.google.protobuf.ExtensionRegistry;
 import com.google.transit.realtime.GtfsRealtime;
 import com.google.transit.realtime.GtfsRealtime.FeedEntity;
 import com.google.transit.realtime.GtfsRealtime.FeedMessage;
 import com.google.transit.realtime.GtfsRealtime.TripUpdate;
+import de.mfdz.MfdzRealtimeExtensions;
 import java.io.InputStream;
 import java.net.URI;
 import java.util.ArrayList;
@@ -27,10 +29,12 @@ public class GtfsRealtimeHttpTripUpdateSource implements TripUpdateSource {
    * previous updates should be disregarded
    */
   private boolean fullDataset = true;
+  private ExtensionRegistry registry = ExtensionRegistry.newInstance();
 
   public GtfsRealtimeHttpTripUpdateSource(Parameters config) {
     this.feedId = config.getFeedId();
     this.url = config.getUrl();
+    MfdzRealtimeExtensions.registerAllExtensions(registry);
   }
 
   @Override
@@ -40,7 +44,7 @@ public List<TripUpdate> getUpdates() {
     List<TripUpdate> updates = null;
     fullDataset = true;
     try {
-      InputStream is = HttpUtils.getData(
+      InputStream is = HttpUtils.openInputStream(
         URI.create(url),
         Map.of(
           "Accept",
@@ -49,7 +53,7 @@ public List<TripUpdate> getUpdates() {
       );
       if (is != null) {
         // Decode message
-        feedMessage = FeedMessage.parseFrom(is);
+        feedMessage = FeedMessage.parseFrom(is, registry);
         feedEntityList = feedMessage.getEntityList();
 
         // Change fullDataset value if this is an incremental update