DIRTY - Do not merge - Log failed itin checks and bus notifs

.github/workflows/maven-ci.yml

@@ -56,38 +56,6 @@ jobs:
           TWILIO_ACCOUNT_SID: ${{ secrets.TWILIO_ACCOUNT_SID }}
           TWILIO_AUTH_TOKEN: ${{ secrets.TWILIO_AUTH_TOKEN }}
           TWILIO_VERIFICATION_SERVICE_SID: ${{ secrets.TWILIO_VERIFICATION_SERVICE_SID }}
-      - name: Run E2E tests
-        run: mvn --no-transfer-progress test
-        env:
-          OTP_ADMIN_DASHBOARD_NAME: OTP Admin Dashboard
-          OTP_ADMIN_DASHBOARD_FROM_EMAIL: OTP Admin Dashboard <[email protected]>
-          OTP_UI_NAME: Trip Planner
-          AUTH0_API_CLIENT: ${{ secrets.AUTH0_API_CLIENT }}
-          AUTH0_API_SECRET: ${{ secrets.AUTH0_API_SECRET }}
-          AUTH0_DOMAIN: ${{ secrets.AUTH0_DOMAIN }}
-          AWS_PROFILE: default
-          AWS_REGION: us-east-1
-          BUGSNAG_API_KEY: ${{ secrets.BUGSNAG_API_KEY }}
-          BUGSNAG_ORGANIZATION: ${{ secrets.BUGSNAG_ORGANIZATION }}
-          BUGSNAG_PROJECT_NOTIFIER_API_KEY: ${{ secrets.BUGSNAG_PROJECT_NOTIFIER_API_KEY }}
-          CONNECTED_DATA_PLATFORM_S3_BUCKET_NAME: ${{ secrets.CONNECTED_DATA_PLATFORM_S3_BUCKET_NAME }}
-          CONNECTED_DATA_PLATFORM_S3_FOLDER_NAME: ${{ secrets.CONNECTED_DATA_PLATFORM_S3_FOLDER_NAME }}
-          CONNECTED_DATA_PLATFORM_TRIP_HISTORY_UPLOAD_JOB_FREQUENCY_IN_MINUTES: ${{ secrets.CONNECTED_DATA_PLATFORM_TRIP_HISTORY_UPLOAD_JOB_FREQUENCY_IN_MINUTES }}
-          DEFAULT_USAGE_PLAN_ID: ${{ secrets.DEFAULT_USAGE_PLAN_ID }}
-          MONGO_DB_NAME: ${{ secrets.MONGO_DB_NAME }}
-          NOTIFICATION_FROM_EMAIL: ${{ secrets.NOTIFICATION_FROM_EMAIL }}
-          NOTIFICATION_FROM_PHONE: ${{ secrets.NOTIFICATION_FROM_PHONE }}
-          OTP_ADMIN_DASHBOARD_URL: ${{ secrets.OTP_ADMIN_DASHBOARD_URL }}
-          OTP_API_ROOT: ${{ secrets.OTP_API_ROOT }}
-          OTP2_API_ROOT: ${{ secrets.OTP_API_ROOT }}
-          OTP_PLAN_ENDPOINT: ${{ secrets.OTP_PLAN_ENDPOINT }}
-          OTP_TIMEZONE: ${{ secrets.OTP_TIMEZONE }}
-          OTP_UI_URL: ${{ secrets.OTP_UI_URL }}
-          SPARKPOST_KEY: ${{ secrets.SPARKPOST_KEY }}
-          TWILIO_ACCOUNT_SID: ${{ secrets.TWILIO_ACCOUNT_SID }}
-          TWILIO_AUTH_TOKEN: ${{ secrets.TWILIO_AUTH_TOKEN }}
-          TWILIO_VERIFICATION_SERVICE_SID: ${{ secrets.TWILIO_VERIFICATION_SERVICE_SID }}
-          RUN_E2E: true
 
       # if this point it reached, the CI build has succeeded
 

src/main/java/org/opentripplanner/middleware/controllers/api/MonitoredTripController.java

@@ -104,7 +104,7 @@ MonitoredTrip postCreateHook(MonitoredTrip monitoredTrip, Request req) {
      * monitored trip job, so return the trip as found in the database after the job completes.
      */
     private MonitoredTrip runCheckMonitoredTrip(MonitoredTrip monitoredTrip) throws Exception {
-        new CheckMonitoredTrip(monitoredTrip, false).run();
+        new CheckMonitoredTrip(monitoredTrip, true).run();
         return Persistence.monitoredTrips.getById(monitoredTrip.id);
     }
 

src/main/java/org/opentripplanner/middleware/models/ItineraryExistence.java

@@ -11,6 +11,9 @@
 import org.opentripplanner.middleware.otp.response.TripPlan;
 import org.opentripplanner.middleware.utils.DateTimeUtils;
 import org.opentripplanner.middleware.utils.ItineraryUtils;
+import org.opentripplanner.middleware.utils.JsonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.time.DayOfWeek;
 import java.time.ZonedDateTime;
@@ -29,6 +32,8 @@
  * particular day of the week.
  */
 public class ItineraryExistence extends Model {
+    private static final Logger LOG = LoggerFactory.getLogger(ItineraryExistence.class);
+
     /**
      * Initial set of requests on which to base the itinerary existence checks. We do not want these persisted.
      */
@@ -186,7 +191,9 @@ public String getInvalidDaysOfWeekMessage() {
     public void checkExistence(MonitoredTrip trip) {
         // TODO: Consider multi-threading?
         // Check existence of itinerary in the response for each OTP request.
+        int index = 0;
         for (OtpRequest otpRequest : otpRequests) {
+            index++;
             boolean hasMatchingItinerary = false;
             DayOfWeek dayOfWeek = otpRequest.dateTime.getDayOfWeek();
             // Get existing result for day of week if a date for that day of week has already been processed, or create
@@ -199,31 +206,43 @@ public void checkExistence(MonitoredTrip trip) {
 
             // Send off each plan query to OTP.
             OtpResponse response = this.otpResponseProvider.apply(otpRequest);
-            TripPlan plan = response.plan;
-
-            // Handle response if valid itineraries exist.
-            if (plan != null && plan.itineraries != null) {
-                for (Itinerary itineraryCandidate : plan.itineraries) {
-                    // If a matching itinerary on the same service day as the request date is found,
-                    // save the date with the matching itinerary.
-                    // (The matching itinerary will replace the original trip.itinerary.)
-                    if (
-                        ItineraryUtils.occursOnSameServiceDay(itineraryCandidate, otpRequest.dateTime, tripIsArriveBy) &&
-                        ItineraryUtils.itinerariesMatch(referenceItinerary, itineraryCandidate)
-                    ) {
-                        result.handleValidDate(otpRequest.dateTime, itineraryCandidate);
-                        hasMatchingItinerary = true;
+            if (response == null) {
+                LOG.warn("Itinerary existence check failed on {} for trip {} - OTP response was null.", dayOfWeek , trip.id);
+            } else {
+                TripPlan plan = response.plan;
+
+                // Handle response if valid itineraries exist.
+                if (plan != null && plan.itineraries != null) {
+                    for (Itinerary itineraryCandidate : plan.itineraries) {
+                        // If a matching itinerary on the same service day as the request date is found,
+                        // save the date with the matching itinerary.
+                        // (The matching itinerary will replace the original trip.itinerary.)
+                        if (
+                            ItineraryUtils.occursOnSameServiceDay(itineraryCandidate, otpRequest.dateTime, tripIsArriveBy) &&
+                                ItineraryUtils.itinerariesMatch(referenceItinerary, itineraryCandidate)
+                        ) {
+                            result.handleValidDate(otpRequest.dateTime, itineraryCandidate);
+                            hasMatchingItinerary = true;
+                        }
+                    }
+                }
+
+                if (!hasMatchingItinerary) {
+                    // If no match was found for the date, mark day of week as non-existent for the itinerary.
+                    result.handleInvalidDate(otpRequest.dateTime);
+
+                    // Log if the itinerary didn't exist "today"
+                    if (index == 1) {
+                        LOG.warn("Itinerary existence check failed 'today' for trip {} - params: {}", trip.id, JsonUtils.toJson(trip.otp2QueryParams));
+                        LOG.warn("Itinerary existence check failed 'today' for trip {} - saved itinerary: {}", trip.id, JsonUtils.toJson(trip.itinerary));
+                        LOG.warn("Itinerary existence check failed 'today' for trip {} - OTP itineraries: {}", trip.id, JsonUtils.toJson(plan.itineraries));
                     }
                 }
-            }
-            if (!hasMatchingItinerary) {
-                // If no match was found for the date, mark day of week as non-existent for the itinerary.
-                result.handleInvalidDate(otpRequest.dateTime);
             }
         }
         if (!allMonitoredDaysAreValid(trip)) {
             this.message = String.format(
-                "The trip is not possible on the following days of the week you have selected: %s",
+                "The trip is not possible on the following days of the week you have selected: %s. Real-time conditions might have changed since you last planned this trip, so try returning to the trip planner, perform the itinerary search again, and save the result.",
                 getInvalidDaysOfWeekMessage()
             );
             this.error = true;

src/main/java/org/opentripplanner/middleware/tripmonitor/jobs/CheckMonitoredTrip.java

@@ -19,6 +19,7 @@
 import org.opentripplanner.middleware.utils.DateTimeUtils;
 import org.opentripplanner.middleware.utils.I18nUtils;
 import org.opentripplanner.middleware.utils.ItineraryUtils;
+import org.opentripplanner.middleware.utils.JsonUtils;
 import org.opentripplanner.middleware.utils.NotificationUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -259,7 +260,10 @@ private boolean isTrackingOngoing() {
      */
     private boolean makeOTPRequestAndUpdateMatchingItineraryInternal() {
         OtpResponse otpResponse = otpResponseProvider.get();
-        if (otpResponse == null) return false;
+        if (otpResponse == null) {
+            LOG.warn("No comparison itinerary found for trip {} - OTP response was null.", trip.id);
+            return false;
+        }
         for (int i = 0; i < otpResponse.plan.itineraries.size(); i++) {
             Itinerary candidateItinerary = otpResponse.plan.itineraries.get(i);
             if (ItineraryUtils.itinerariesMatch(trip.itinerary, candidateItinerary)) {
@@ -301,7 +305,9 @@ private boolean makeOTPRequestAndUpdateMatchingItineraryInternal() {
         }
 
         // If this point is reached, a matching itinerary was not found
-        LOG.warn("No comparison itinerary found in otp response for trip");
+        LOG.warn("No comparison itinerary found for trip {} - params: {}", trip.id, JsonUtils.toJson(trip.otp2QueryParams));
+        LOG.warn("No comparison itinerary found for trip {} - saved itinerary: {}", trip.id, JsonUtils.toJson(trip.itinerary));
+        LOG.warn("No comparison itinerary found for trip {} - OTP itineraries: {}", trip.id, JsonUtils.toJson(otpResponse.plan.itineraries));
 
         if (hasReachedMaxItineraryChecks()) {
             // Check whether this trip should no longer ever be checked due to not having matching itineraries on any