refactor(Fixed merge conflicts):

src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java

@@ -120,9 +120,9 @@ public enum NewGTFSErrorType {
     FIELD_VALUE_TOO_LONG(Priority.MEDIUM, "Field value has too many characters."),
 
     // Shared Stops-specifc errors.
-    MULTIPLE_SHARED_STOPS_GROUPS(Priority.HIGH, "A GTFS stop belongs to more than one shared-stop group."),
-    SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS(Priority.HIGH, "A Shared-stop group has multiple primary stops."),
-    SHARED_STOP_GROUP_ENTITY_DOES_NOT_EXIST(Priority.MEDIUM, "The stop referenced by a shared-stop does not exist."),
+    MULTIPLE_SHARED_STOPS_GROUPS(Priority.HIGH, "A GTFS stop belongs to more than one shared-stop group, or belongs to the same shared-stop group twice."),
+    SHARED_STOP_GROUP_MULTIPLE_PRIMARY_STOPS(Priority.HIGH, "A shared-stop group has multiple primary stops."),
+    SHARED_STOP_GROUP_ENTITY_DOES_NOT_EXIST(Priority.MEDIUM, "The stop referenced by a shared-stop does not exist in the feed it was said to exist in."),
 
     // Unknown errors.
     OTHER(Priority.LOW, "Other errors.");

src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java

@@ -487,6 +487,17 @@ private void populateDefaultEditorValues(Connection connection, String tablePref
             LOG.info(updateOtherSql);
             int calendarsUpdated = statement.executeUpdate(updateOtherSql);
             LOG.info("Updated description for {} calendars", calendarsUpdated);
+
+            // Check if there are duplicate descriptions, in which case set the description to be the service_id which is unique
+            String avoidDuplicateDescriptionSql = String.format(
+                    "update %1$scalendar set description = service_id " +
+                    "from (select description, count(*) as duplicate_count from %1$scalendar group by description) as duplicate_descriptions " +
+                    "where %1$scalendar.description = duplicate_descriptions.description and duplicate_descriptions.duplicate_count > 1",
+                    tablePrefix
+            );
+            LOG.info(avoidDuplicateDescriptionSql);
+            int duplicatesAvoided = statement.executeUpdate(avoidDuplicateDescriptionSql);
+            LOG.info("Updated duplicate descriptions for {} calendars", duplicatesAvoided);
         }
         if (Table.TRIPS.name.equals(table.name)) {
             // Update use_frequency field for patterns. This sets all patterns that have a frequency trip to use

src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java

@@ -3,6 +3,7 @@
 import com.conveyal.gtfs.model.Entity;
 import com.conveyal.gtfs.model.Location;
 import com.conveyal.gtfs.model.LocationShape;
+import com.conveyal.gtfs.model.PatternStop;
 import com.conveyal.gtfs.model.Stop;
 import com.conveyal.gtfs.model.ScheduleException.ExemplarServiceDescriptor;
 import com.conveyal.gtfs.model.Shape;
@@ -39,8 +40,10 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 import static com.conveyal.gtfs.loader.JdbcGtfsLoader.INSERT_BATCH_SIZE;
 import static com.conveyal.gtfs.util.Util.ensureValidNamespace;
@@ -317,6 +320,23 @@ public int normalizeStopTimesForPattern(int id, int beginWithSequence) throws SQ
         }
     }
 
+    /**
+     * For a given pattern id and starting stop sequence (inclusive), normalize all stop times to match the pattern
+     * stops' travel times.
+     *
+     * @return number of stop times updated
+     */
+    public int normalizeStopTimesForPattern(int id, int beginWithSequence, boolean interpolateStopTimes) throws SQLException {
+        if (!interpolateStopTimes) {
+            // Use the newer approach which can handle flex.
+            return normalizeStopTimesForPattern(id, beginWithSequence);
+        } else {
+            // Use the legacy approach which can handle only pattern stops (not flex), but can do interpolating.
+            StopTimeNormalization stopTimeNormalization = new StopTimeNormalization(dataSource, connection, tablePrefix);
+            return stopTimeNormalization.normalizeStopTimesForPattern(id, beginWithSequence, true);
+        }
+    }
+
     /**
      * Updates linked fields with values from entity being updated. This is used to update identical fields in related
      * tables (for now just fields in trips and stop_times) where the reference table's value should take precedence over
@@ -1149,7 +1169,7 @@ private void ensureReferentialIntegrity(
             // Special case for schedule_exceptions where for exception type 10 and service_id is also a key.
             String calendarDateServiceKey = "custom_schedule";
             Field calendarDateServiceKeyField = table.getFieldForName(calendarDateServiceKey);
-            String calendarDateServiceKeyVal = jsonObject.get(calendarDateServiceKey).asText();
+            String calendarDateServiceKeyVal = jsonObject.get(calendarDateServiceKey).get(0).asText();
             TIntSet calendarDateServiceUniqueIds = getIdsForCondition(tableName, calendarDateServiceKey, calendarDateServiceKeyVal, connection);
             checkUniqueIdsAndUpdateReferencingTables(
                 calendarDateServiceUniqueIds,
@@ -1239,7 +1259,7 @@ private static Set<Table> getReferencingTables(Table table) {
     /**
      * For a given integer ID, return the value for the specified field name for that entity.
      */
-    private static String getValueForId(int id, String fieldName, String namespace, Table table, Connection connection) throws SQLException {
+    public static String getValueForId(int id, String fieldName, String namespace, Table table, Connection connection) throws SQLException {
         String tableName = String.join(".", namespace, table.name);
         String selectIdSql = String.format("select %s from %s where id = %d", fieldName, tableName, id);
         LOG.info(selectIdSql);
@@ -1372,7 +1392,7 @@ private void updateReferencingTables(
                                             connection.rollback();
                                             if (entityClass.getSimpleName().equals("Stop")) {
                                                 String patternStopLookup = String.format(
-                                                    "select distinct p.id, r.id " +
+                                                    "select distinct p.id, r.id, r.route_short_name, r.route_id  " +
                                                         "from %s.pattern_stops ps " +
                                                         "inner join " +
                                                         "%s.patterns p " +

src/main/java/com/conveyal/gtfs/loader/StopTimeNormalization.java

@@ -1,10 +1,12 @@
 package com.conveyal.gtfs.loader;
 
+import com.conveyal.gtfs.model.Entity;
 import com.conveyal.gtfs.model.PatternHalt;
 import com.conveyal.gtfs.model.PatternLocation;
 import com.conveyal.gtfs.model.PatternStop;
 import com.conveyal.gtfs.model.PatternStopArea;
 import com.google.common.collect.Iterators;
+import org.apache.commons.dbutils.DbUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -21,6 +23,8 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static com.conveyal.gtfs.loader.JdbcTableWriter.getValueForId;
+
 public class StopTimeNormalization {
     private static final Logger LOG = LoggerFactory.getLogger(StopTimeNormalization.class);
     final DataSource dataSource;
@@ -278,4 +282,154 @@ public int executeRemaining() throws SQLException{
             return singleTripBatchTracker.executeRemaining() + allTripsBatchTracker.executeRemaining();
         }
     }
+
+    /**
+     * For a given pattern id and starting stop sequence (inclusive), normalize all stop times to match the pattern
+     * stops' travel times.
+     *
+     * @return number of stop times updated
+     */
+    public int normalizeStopTimesForPattern(int id, int beginWithSequence, boolean interpolateStopTimes) throws SQLException {
+        try {
+            JDBCTableReader<PatternStop> patternStops = new JDBCTableReader(
+                Table.PATTERN_STOP,
+                dataSource,
+                tablePrefix + ".",
+                EntityPopulator.PATTERN_STOP
+            );
+            String patternId = getValueForId(id, "pattern_id", tablePrefix, Table.PATTERNS, connection);
+            List<PatternStop> patternStopsToNormalize = new ArrayList<>();
+            for (PatternStop patternStop : patternStops.getOrdered(patternId)) {
+                // Update stop times for any pattern stop with matching stop sequence (or for all pattern stops if the list
+                // is null).
+                if (patternStop.stop_sequence >= beginWithSequence) {
+                    patternStopsToNormalize.add(patternStop);
+                }
+            }
+            int stopTimesUpdated = updateStopTimesForPatternStops(patternStopsToNormalize, interpolateStopTimes);
+            connection.commit();
+            return stopTimesUpdated;
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw e;
+        } finally {
+            DbUtils.closeQuietly(connection);
+        }
+    }
+
+    /**
+     * Normalizes all stop times' arrivals and departures for an ordered set of pattern stops. This set can be the full
+     * set of stops for a pattern or just a subset. Typical usage for this method would be to overwrite the arrival and
+     * departure times for existing trips after a pattern stop has been added or inserted into a pattern or if a
+     * pattern stop's default travel or dwell time were updated and the stop times need to reflect this update.
+     *
+     * @param patternStops list of pattern stops for which to update stop times (ordered by increasing stop_sequence)
+     * @throws SQLException
+     *
+     */
+    private int updateStopTimesForPatternStops(List<PatternStop> patternStops, boolean interpolateStopTimes) throws SQLException {
+        PatternStop firstPatternStop = patternStops.iterator().next();
+        List<PatternStop> timepoints = patternStops.stream().filter(ps -> ps.timepoint == 1).collect(Collectors.toList());
+        int firstStopSequence = firstPatternStop.stop_sequence;
+        // Prepare SQL query to determine the time that should form the basis for adding the travel time values.
+        int previousStopSequence = firstStopSequence > 0 ? firstStopSequence - 1 : 0;
+        String timeField = firstStopSequence > 0 ? "departure_time" : "arrival_time";
+        String getFirstTravelTimeSql = String.format(
+            "select t.trip_id, %s from %s.stop_times st, %s.trips t where stop_sequence = ? " +
+                "and t.pattern_id = ? " +
+                "and t.trip_id = st.trip_id",
+            timeField,
+            tablePrefix,
+            tablePrefix
+        );
+        PreparedStatement statement = connection.prepareStatement(getFirstTravelTimeSql);
+        statement.setInt(1, previousStopSequence);
+        statement.setString(2, firstPatternStop.pattern_id);
+        LOG.info(statement.toString());
+        ResultSet resultSet = statement.executeQuery();
+        Map<String, Integer> timesForTripIds = new HashMap<>();
+        while (resultSet.next()) {
+            timesForTripIds.put(resultSet.getString(1), resultSet.getInt(2));
+        }
+        // Update stop times for individual trips with normalized travel times.
+        String updateTravelTimeSql = String.format(
+            "update %s.stop_times set arrival_time = ?, departure_time = ? where trip_id = ? and stop_sequence = ?",
+            tablePrefix
+        );
+        PreparedStatement updateStopTimeStatement = connection.prepareStatement(updateTravelTimeSql);
+        LOG.info(updateStopTimeStatement.toString());
+        final BatchTracker stopTimesTracker = new BatchTracker("stop_times", updateStopTimeStatement);
+        for (String tripId : timesForTripIds.keySet()) {
+            // Initialize travel time with previous stop time value.
+            int cumulativeTravelTime = timesForTripIds.get(tripId);
+            int cumulativeInterpolatedTime = cumulativeTravelTime;
+            int timepointNumber = 0;
+            double previousShapeDistTraveled = 0; // Used for calculating timepoint speed for interpolation
+            for (PatternStop patternStop : patternStops) {
+                boolean isTimepoint = patternStop.timepoint == 1;
+                if (isTimepoint) timepointNumber++;
+                // Gather travel/dwell time for pattern stop (being sure to check for missing values).
+                int travelTime = patternStop.default_travel_time == Entity.INT_MISSING ? 0 : patternStop.default_travel_time;
+                if (interpolateStopTimes) {
+                    if (patternStop.shape_dist_traveled == Entity.DOUBLE_MISSING) {
+                        throw new IllegalStateException("Shape_dist_traveled must be defined for all stops in order to perform interpolation");
+                    }
+                    // Override travel time if we're interpolating between timepoints.
+                    if (!isTimepoint) travelTime = interpolateTimesFromTimepoints(patternStop, timepoints, timepointNumber, previousShapeDistTraveled);
+                    previousShapeDistTraveled += patternStop.shape_dist_traveled;
+                }
+                int dwellTime = patternStop.default_dwell_time == Entity.INT_MISSING ? 0 : patternStop.default_dwell_time;
+                int oneBasedIndex = 1;
+                // Increase travel time by current pattern stop's travel and dwell times (and set values for update).
+                if (!isTimepoint && interpolateStopTimes) {
+                    // We don't want to increment the true cumulative travel time because that adjusts the timepoint
+                    // times later in the pattern.
+                    // Dwell times are ignored right now as they do not fit the typical use case for interpolation.
+                    // They may be incorporated by accounting for all dwell times in intermediate stops when calculating
+                    // the timepoint speed.
+                    cumulativeInterpolatedTime += travelTime;
+                    updateStopTimeStatement.setInt(oneBasedIndex++, cumulativeInterpolatedTime);
+                    updateStopTimeStatement.setInt(oneBasedIndex++, cumulativeInterpolatedTime);
+                } else {
+                    cumulativeTravelTime += travelTime;
+                    updateStopTimeStatement.setInt(oneBasedIndex++, cumulativeTravelTime);
+                    cumulativeTravelTime += dwellTime;
+                    updateStopTimeStatement.setInt(oneBasedIndex++, cumulativeTravelTime);
+                }
+                updateStopTimeStatement.setString(oneBasedIndex++, tripId);
+                updateStopTimeStatement.setInt(oneBasedIndex, patternStop.stop_sequence);
+                stopTimesTracker.addBatch();
+            }
+        }
+        return stopTimesTracker.executeRemaining();
+    }
+
+    /**
+     * Updates the non-timepoint stop times between two timepoints using the speed implied  by
+     * the travel time between them. Ignores any existing default_travel_time or default_dwell_time
+     * entered for the non-timepoint stops.
+     */
+    private int interpolateTimesFromTimepoints(
+        PatternStop patternStop,
+        List<PatternStop> timepoints,
+        Integer timepointNumber,
+        double previousShapeDistTraveled
+    ) {
+        if (timepointNumber == 0 || timepoints.size() == 1 || timepointNumber >= timepoints.size()) {
+            throw new IllegalStateException("Issue in pattern stops which prevents interpolation (e.g. less than 2 timepoints)");
+        }
+        PatternStop nextTimepoint = timepoints.get(timepointNumber);
+        PatternStop lastTimepoint = timepoints.get(timepointNumber-1);
+
+        if (nextTimepoint == null ||
+            nextTimepoint.default_travel_time == Entity.INT_MISSING ||
+            nextTimepoint.shape_dist_traveled == Entity.DOUBLE_MISSING ||
+            lastTimepoint.shape_dist_traveled == Entity.DOUBLE_MISSING
+        ) {
+            throw new IllegalStateException("Error with stop time interpolation: timepoint or shape_dist_traveled is null");
+        }
+
+        double timepointSpeed = (nextTimepoint.shape_dist_traveled - lastTimepoint.shape_dist_traveled) / nextTimepoint.default_travel_time;
+        return (int) Math.round((patternStop.shape_dist_traveled - previousShapeDistTraveled) / timepointSpeed);
+    }
 }

src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java

@@ -34,4 +34,12 @@ public PatternStopDTO (String patternId, String stopId, int stopSequence, int de
         default_travel_time = defaultTravelTime;
         default_dwell_time = defaultDwellTime;
     }
+
+    public PatternStopDTO (String patternId, String stopId, int stopSequence, int timepointValue, double shape_dist_traveledValue) {
+        timepoint = timepointValue;
+        pattern_id = patternId;
+        stop_id = stopId;
+        stop_sequence = stopSequence;
+        shape_dist_traveled = shape_dist_traveledValue;
+    }
 }