Merge pull request #1 from ibi-group/flex-merge-feeds

Flex merge feeds

src/main/java/com/conveyal/gtfs/error/GeoJsonParseError.java

@@ -2,7 +2,7 @@
 
 import java.io.Serializable;
 
-/** Represents a problem parsing location GeoJson from a GTFS feed. */
+/** Represents a problem parsing location GeoJSON from a GTFS feed. */
 public class GeoJsonParseError extends GTFSError implements Serializable {
     public static final long serialVersionUID = 1L;
 

src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java

@@ -54,7 +54,7 @@ public enum NewGTFSErrorType {
     FLEX_MISSING_FARE_RULE(Priority.HIGH, "A location zone id must reference a fare rule. One of contains id, destination id or origin id."),
     FLOATING_FORMAT(Priority.MEDIUM, "Incorrect floating point number format."),
     FREQUENCY_PERIOD_OVERLAP(Priority.MEDIUM, "A frequency for a trip overlaps with another frequency defined for the same trip."),
-    GEO_JSON_PARSING(Priority.HIGH, "Unable to parse the locations.geojson file. Make sure the file conforms to the GeoJson standard and supported geometry types are used."),
+    GEO_JSON_PARSING(Priority.HIGH, "Unable to parse the locations.geojson file. Make sure the file conforms to the GeoJSON standard and supported geometry types are used."),
     ILLEGAL_FIELD_VALUE(Priority.MEDIUM, "Fields may not contain tabs, carriage returns or new lines."),
     INTEGER_FORMAT(Priority.MEDIUM, "Incorrect integer format."),
     LANGUAGE_FORMAT(Priority.LOW, "Language should be specified with a valid BCP47 tag."),

src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java

@@ -168,7 +168,7 @@ public FeedLoadResult loadTables() {
             result.transfers = load(Table.TRANSFERS); // refs trips.
             result.frequencies = load(Table.FREQUENCIES); // refs trips
             result.locations = load(Table.LOCATIONS);
-            result.stopAreas = load(Table.STOP_AREAS);
+            result.stopAreas = load(Table.STOP_AREAS);  // refs areas.
             result.stopTimes = load(Table.STOP_TIMES); // refs stop areas, locations and stops
             result.translations = load(Table.TRANSLATIONS);
             result.attributions = load(Table.ATTRIBUTIONS);

src/main/java/com/conveyal/gtfs/loader/Table.java

@@ -367,19 +367,26 @@ public Table (String name, Class<? extends Entity> entityClass, Requirement requ
         new StringField("zone_id", OPTIONAL),
         new URLField("stop_url", OPTIONAL),
         new StringField("geometry_type", REQUIRED)
-    ).addPrimaryKey();
+    )
+    .addPrimaryKey()
+    .addPrimaryKeyNames("location_id");
+
+    // https://github.com/MobilityData/gtfs-flex/blob/master/spec/reference.md#areastxt-no-change
+    public static final Table AREA = new Table("areas", Area.class, OPTIONAL,
+        new StringField("area_id", REQUIRED),
+        new StringField("area_name", OPTIONAL)
+    )
+    .addPrimaryKeyNames("area_id");
+
 
     // https://github.com/MobilityData/gtfs-flex/blob/master/spec/reference.md#stop_areastxt-file-modified
     public static final Table STOP_AREAS = new Table("stop_areas", StopArea.class, OPTIONAL,
-        new StringField("area_id", REQUIRED),
+        new StringField("area_id", REQUIRED).isReferenceTo(AREA),
         new StringField("stop_id", REQUIRED).isReferenceTo(STOPS).isReferenceTo(LOCATIONS)
-    ).keyFieldIsNotUnique();
+    )
+    .keyFieldIsNotUnique()
+    .addPrimaryKeyNames("area_id", "stop_id");
 
-    // https://github.com/MobilityData/gtfs-flex/blob/master/spec/reference.md#areastxt-no-change
-    public static final Table AREA = new Table("areas", Area.class, OPTIONAL,
-        new StringField("area_id", REQUIRED).isReferenceTo(STOP_AREAS),
-        new StringField("area_name", OPTIONAL)
-    );
     // Must come after TRIPS table to which it has references.
     public static final Table TRANSFERS = new Table("transfers", Transfer.class, OPTIONAL,
         // Conditionally required fields (from_stop_id, to_stop_id, from_trip_id and to_trip_id) are defined here as
@@ -503,7 +510,8 @@ public Table (String name, Class<? extends Entity> entityClass, Requirement requ
             new StringField("phone_number", OPTIONAL),
             new URLField("info_url", OPTIONAL),
             new URLField("booking_url", OPTIONAL)
-    );
+    )
+    .addPrimaryKeyNames("booking_rule_id");
 
     // https://github.com/MobilityData/gtfs-flex/blob/master/spec/reference.md#locationsgeojson-file-added
     public static final Table LOCATION_SHAPES = new Table("location_shapes", LocationShape.class, OPTIONAL,
@@ -513,7 +521,8 @@ public Table (String name, Class<? extends Entity> entityClass, Requirement requ
         new DoubleField("geometry_pt_lon", REQUIRED, -180, 180, 6)
     )
     .keyFieldIsNotUnique()
-    .withParentTable(LOCATIONS);
+    .withParentTable(LOCATIONS)
+    .addPrimaryKeyNames("location_id", "geometry_id", "geometry_pt_lat", "geometry_pt_lon");
 
     public static final Table PATTERN_LOCATION = new Table("pattern_locations", PatternLocation.class, OPTIONAL,
             new StringField("pattern_id", REQUIRED).isReferenceTo(PATTERNS),
@@ -578,6 +587,7 @@ public Table (String name, Class<? extends Entity> entityClass, Requirement requ
         PATTERNS,
         SHAPES,
         STOPS,
+        LOCATIONS,
         STOP_AREAS,
         FARE_RULES,
         PATTERN_STOP,
@@ -590,8 +600,7 @@ public Table (String name, Class<? extends Entity> entityClass, Requirement requ
         TRANSLATIONS,
         ATTRIBUTIONS,
         BOOKING_RULES,
-        LOCATION_SHAPES,
-        LOCATIONS
+        LOCATION_SHAPES
     };
 
     /**
@@ -844,7 +853,7 @@ public CsvReader getCsvReader(ZipFile zipFile, SQLErrorStorage sqlErrorStorage)
     }
 
     /**
-     * Create a CSV reader depending on the table to be loaded. If the table is "locations.geojson" unpack the GeoJson
+     * Create a CSV reader depending on the table to be loaded. If the table is "locations.geojson" unpack the GeoJSON
      * data first and load into a CSV reader, else, read the table contents directly into the CSV reader.
      */
     public static CsvReader getCsvReader(

src/main/java/com/conveyal/gtfs/model/Location.java

@@ -45,15 +45,15 @@ public void setStatementParameters(PreparedStatement statement, boolean setDefau
 
     /**
      * Required by {@link com.conveyal.gtfs.util.GeoJsonUtil#getCsvReaderFromGeoJson(String, ZipFile, ZipEntry, List)} as part
-     * of the unpacking of GeoJson data to CSV.
+     * of the unpacking of GeoJSON data to CSV.
      */
     public static String header() {
         return "location_id,stop_name,stop_desc,zone_id,stop_url,geometry_type\n";
     }
 
     /**
      * Required by {@link com.conveyal.gtfs.util.GeoJsonUtil#getCsvReaderFromGeoJson(String, ZipFile, ZipEntry, List)} as part
-     * of the unpacking of GeoJson data to CSV.
+     * of the unpacking of GeoJSON data to CSV.
      */
     public String toCsvRow() {
         String stopName = "", stopDesc = "";
@@ -112,12 +112,12 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         Location that = (Location) o;
-        return stop_name == that.stop_name &&
-                zone_id == that.zone_id &&
-                stop_desc == that.stop_desc &&
+        return Objects.equals(stop_name, that.stop_name) &&
+            Objects.equals(zone_id, that.zone_id) &&
+            Objects.equals(stop_desc, that.stop_desc) &&
                 Objects.equals(stop_url, that.stop_url) &&
                 Objects.equals(location_id, that.location_id) &&
-                geometry_type == that.geometry_type;
+            Objects.equals(geometry_type, that.geometry_type);
     }
 
     @Override

src/main/java/com/conveyal/gtfs/model/LocationShape.java

@@ -86,15 +86,15 @@ public void loadOneRow() throws IOException {
 
     /**
      * Required by {@link com.conveyal.gtfs.util.GeoJsonUtil#getCsvReaderFromGeoJson(String, ZipFile, ZipEntry, List)}
-     * as part of the unpacking of GeoJson data to CSV.
+     * as part of the unpacking of GeoJSON data to CSV.
      */
     public static String header() {
         return "location_id,geometry_id,geometry_pt_lat,geometry_pt_lon\n";
     }
 
     /**
      * Required by {@link com.conveyal.gtfs.util.GeoJsonUtil#getCsvReaderFromGeoJson(String, ZipFile, ZipEntry, List)}
-     * as part of the unpacking of GeoJson data to CSV.
+     * as part of the unpacking of GeoJSON data to CSV.
      */
     public String toCsvRow() {
         return String.join(

src/main/java/com/conveyal/gtfs/model/StopArea.java

@@ -44,7 +44,7 @@ public StopArea(String areaId, String stopId) {
 
     @Override
     public String getId() {
-        return area_id;
+        return createId(area_id, stop_id);
     }
 
     /**
@@ -77,8 +77,8 @@ public void loadOneRow() throws IOException {
             stopArea.area_id = getStringField("area_id", true);
             stopArea.stop_id = getStringField("stop_id", true);
             // Attempting to put a null key or value will cause an NPE in BTreeMap
-            if (stopArea.area_id != null) {
-                feed.stopAreas.put(stopArea.area_id, stopArea);
+            if (stopArea.area_id != null && stopArea.stop_id != null) {
+                feed.stopAreas.put(createId(stopArea.area_id, stopArea.stop_id), stopArea);
             }
         }
     }
@@ -218,6 +218,11 @@ public static String packStopAreas(List<StopArea> stopAreas) {
         return csvContent.toString();
     }
 
+    private static String createId(String areaId, String stopId) {
+        return String.format("%s_%s", areaId, stopId);
+    }
+
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

src/main/java/com/conveyal/gtfs/model/StopTime.java

@@ -1,9 +1,7 @@
 package com.conveyal.gtfs.model;
 
 import com.conveyal.gtfs.GTFSFeed;
-
 import org.mapdb.Fun;
-
 import java.io.IOException;
 import java.io.Serializable;
 import java.sql.Connection;

src/main/java/com/conveyal/gtfs/util/GeoJsonUtil.java

@@ -25,6 +25,7 @@
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -38,9 +39,9 @@
 
 /**
  * With the aid of this third party library: https://ngageoint.github.io/simple-features-geojson-java/, this util class
- * handles the unpacking and packing of GeoJson data. Unpacking flattens the location data into two classes
+ * handles the unpacking and packing of GeoJSON data. Unpacking flattens the location data into two classes
  * {@link Location} and {@link LocationShape}. Packing does the opposite by using these two classes to convert
- * the data back into validate GeoJson.
+ * the data back into validate GeoJSON.
  */
 public class GeoJsonUtil {
 
@@ -54,27 +55,25 @@ public class GeoJsonUtil {
     private static final String STOP_URL = "stop_url";
 
     private GeoJsonUtil() {
-        throw new IllegalStateException("GeoJson utility class.");
+        throw new IllegalStateException("GeoJSON utility class.");
     }
 
     /**
      * Takes the content of a zip file entry and converts it into a {@link FeatureCollection} which is a class
      * representation of features held in the locations file.
      */
-    public static FeatureCollection getLocations(ZipFile zipFile, ZipEntry entry) {
+    public static FeatureCollection getFeatureCollection(ZipFile zipFile, ZipEntry entry) {
         try (InputStream zipInputStream = zipFile.getInputStream(entry)) {
             String content;
             try (InputStream bomInputStream = new BOMInputStream(zipInputStream)) {
                 content = new BufferedReader(
                     new InputStreamReader(bomInputStream, StandardCharsets.UTF_8))
                     .lines()
-                    .collect(Collectors.joining("\n")
-                    );
+                    .collect(Collectors.joining("\n"));
             }
             return FeatureConverter.toFeatureCollection(content);
         } catch (IOException e) {
             LOG.error("Exception while opening zip entry: ", e);
-            e.printStackTrace();
             return null;
         }
     }
@@ -195,9 +194,9 @@ private static String getPropertyValue(Object propertyValue) {
 
     /**
      * Extract from a list of features the different geometry types and produce the appropriate {@link LocationShape}
-     * representing this geometry type so that enough information is available to revert it back to GeoJson.
+     * representing this geometry type so that enough information is available to revert it back to GeoJSON.
      *
-     * GeoJson format reference: https://datatracker.ietf.org/doc/html/rfc7946#section-3.1.4
+     * GeoJSON format reference: https://datatracker.ietf.org/doc/html/rfc7946#section-3.1.4
      */
     private static List<LocationShape> unpackLocationShapes(
         FeatureCollection featureCollection,
@@ -275,11 +274,8 @@ public static CsvReader getCsvReaderFromGeoJson(
         ZipEntry entry,
         List<String> errors
     ) {
-        FeatureCollection features = GeoJsonUtil.getLocations(zipFile, entry);
-        if (features == null || features.numFeatures() == 0) {
-            String message = "Unable to extract GeoJson features (or none are available) from " + entry.getName();
-            LOG.warn(message);
-            if (errors != null) errors.add(message);
+        FeatureCollection features = getFeaturesFromGeoJson(zipFile, entry, errors);
+        if (features == null) {
             return null;
         }
         StringBuilder csvContent = new StringBuilder();
@@ -296,7 +292,37 @@ public static CsvReader getCsvReaderFromGeoJson(
     }
 
     /**
-     * Convert {@link Location} and {@link LocationShape} lists to a serialized String conforming to the GeoJson
+     * Extract the locations from GeoJSON.
+     */
+    public static List<Location> getLocationsFromGeoJson(ZipFile zipFile, ZipEntry entry, List<String> errors) {
+        FeatureCollection features = getFeaturesFromGeoJson(zipFile, entry, errors);
+        return (features == null) ? Collections.emptyList() : GeoJsonUtil.unpackLocations(features, errors);
+    }
+
+    /**
+     * Extract the location shapes from GeoJSON.
+     */
+    public static List<LocationShape> getLocationShapesFromGeoJson(ZipFile zipFile, ZipEntry entry, List<String> errors) {
+        FeatureCollection features = getFeaturesFromGeoJson(zipFile, entry, errors);
+        return (features == null) ? Collections.emptyList() : GeoJsonUtil.unpackLocationShapes(features, errors);
+    }
+
+    /**
+     * Extract the GeoJSON features from file.
+     */
+    private static FeatureCollection getFeaturesFromGeoJson(ZipFile zipFile, ZipEntry entry, List<String> errors) {
+        FeatureCollection features = GeoJsonUtil.getFeatureCollection(zipFile, entry);
+        if (features == null || features.numFeatures() == 0) {
+            String message = "Unable to extract GeoJSON features (or none are available) from " + entry.getName();
+            LOG.warn(message);
+            if (errors != null) errors.add(message);
+            return null;
+        }
+        return features;
+    }
+
+    /**
+     * Convert {@link Location} and {@link LocationShape} lists to a serialized String conforming to the GeoJSON
      * standard.
      */
     public static String packLocations(

src/main/java/com/conveyal/gtfs/validator/FlexValidator.java

@@ -14,6 +14,7 @@
 import com.google.common.collect.Lists;
 
 import javax.sql.DataSource;
+import java.sql.Connection;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
@@ -52,8 +53,8 @@ public void validate() {
 
         if (isFlexFeed(bookingRules, stopAreas, locations)) {
             List<NewGTFSError> errors = new ArrayList<>();
-            try {
-                List<StopTime> stopTimes = getFlexStopTimesForValidation(dataSource.getConnection(), feed.databaseSchemaPrefix);
+            try (Connection connection = dataSource.getConnection()) {
+                List<StopTime> stopTimes = getFlexStopTimesForValidation(connection, feed.databaseSchemaPrefix);
                 stopTimes.forEach(stopTime -> errors.addAll(validateStopTime(stopTime, stopAreas, locations)));
                 feed.trips.forEach(trip -> errors.addAll(validateTrip(trip, stopTimes, stopAreas, locations)));
             } catch (SQLException e) {