From cd0d0f85de9d12bce3ef06ce826f9b2615ff628b Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 14 Apr 2023 17:12:48 +0100 Subject: [PATCH 1/8] refactor(Check for pinned deployment in mongo db): Check if a project has a pinned deployment and re --- .../datatools/manager/models/FeedSource.java | 4 +- .../datatools/manager/models/Project.java | 43 ++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 555414f71..f90ad15a7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -49,6 +49,7 @@ import java.util.stream.Collectors; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; +import static com.conveyal.datatools.manager.models.Project.hasPinnedDeployment; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -516,8 +517,7 @@ public FeedVersionDeployed retrieveDeployedFeedVersion() { if (deployedFeedVersionDefined) { return deployedFeedVersion; } - Project project = Persistence.projects.getById(projectId); - deployedFeedVersion = (project.pinnedDeploymentId != null && !project.pinnedDeploymentId.isEmpty()) + deployedFeedVersion = hasPinnedDeployment(projectId) ? FeedVersionDeployed.getFeedVersionFromPinnedDeployment(projectId, id) : FeedVersionDeployed.getFeedVersionFromLatestDeployment(projectId, id); deployedFeedVersionDefined = true; diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 110618cb5..ddd2298a2 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -2,13 +2,17 @@ import com.conveyal.datatools.manager.jobs.AutoDeployType; import com.conveyal.datatools.manager.persistence.Persistence; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; +import com.mongodb.client.model.Projections; +import org.bson.Document; import org.bson.codecs.pojo.annotations.BsonIgnore; +import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.HashSet; @@ -16,8 +20,11 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Aggregates.project; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.client.model.Filters.in; import static com.mongodb.client.model.Filters.or; /** @@ -191,4 +198,38 @@ public abstract static class ProjectWithOtpServers { @JsonProperty("otpServers") public abstract Collection availableOtpServers (); } + + @BsonIgnore + @JsonIgnore + public static boolean hasPinnedDeployment(String projectId) { + /* + db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "project-with-latest-deployment" + } + }, + { + $project: { + _id: 0, + pinnedDeploymentId: 1 + } + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("_id", projectId) + ), + project(Projections.excludeId()), + project(Projections.include("pinnedDeploymentId")) + ); + Document project = Persistence + .getMongoDatabase() + .getCollection("Project") + .aggregate(stages) + .first(); + return !project.isEmpty(); + } } From 438a3c04addf017cedb32039afa26a288dbc7875 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 3 May 2023 15:55:53 +0100 Subject: [PATCH 2/8] refactor(Added feed source summary): All feed source summaries are available via a new end point --- .../controllers/api/FeedSourceController.java | 17 + .../datatools/manager/models/FeedSource.java | 66 --- .../manager/models/FeedSourceSummary.java | 527 ++++++++++++++++++ .../manager/models/FeedVersionDeployed.java | 239 -------- .../datatools/manager/models/Project.java | 19 + .../manager/persistence/Persistence.java | 14 +- .../api/FeedSourceControllerTest.java | 51 +- 7 files changed, 605 insertions(+), 328 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java delete mode 100644 src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java index cc8807c87..badedda76 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java @@ -9,9 +9,11 @@ import com.conveyal.datatools.manager.extensions.ExternalFeedResource; import com.conveyal.datatools.manager.jobs.FetchSingleFeedJob; import com.conveyal.datatools.manager.jobs.NotifyUsersForSubscriptionJob; +import com.conveyal.datatools.manager.models.DeploymentSummary; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedSourceSummary; import com.conveyal.datatools.manager.models.JsonViews; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation; @@ -395,6 +397,20 @@ protected static FeedSource cleanFeedSourceForNonAdmins(FeedSource feedSource, b return feedSource; } + private static Collection getAllFeedSourceSummaries(Request req, Response res) { + Auth0UserProfile userProfile = req.attribute("user"); + String projectId = req.queryParams("projectId"); + Project project = Persistence.projects.getById(projectId); + if (project == null) { + logMessageAndHalt(req, 400, "Must provide valid projectId value."); + } + if (!userProfile.canAdministerProject(project)) { + logMessageAndHalt(req, 401, "User not authorized to view project feed sources."); + } + return project.retrieveFeedSourceSummaries(); + } + + // FIXME: use generic API controller and return JSON documents via BSON/Mongo public static void register (String apiPrefix) { get(apiPrefix + "secure/feedsource/:id", FeedSourceController::getFeedSource, json::write); @@ -404,5 +420,6 @@ public static void register (String apiPrefix) { put(apiPrefix + "secure/feedsource/:id/updateExternal", FeedSourceController::updateExternalFeedResource, json::write); delete(apiPrefix + "secure/feedsource/:id", FeedSourceController::deleteFeedSource, json::write); post(apiPrefix + "secure/feedsource/:id/fetch", FeedSourceController::fetch, json::write); + get(apiPrefix + "secure/feedsourceSummaries", FeedSourceController::getAllFeedSourceSummaries, json::write); } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index f90ad15a7..db7eea7a9 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -38,7 +38,6 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -49,7 +48,6 @@ import java.util.stream.Collectors; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; -import static com.conveyal.datatools.manager.models.Project.hasPinnedDeployment; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -460,70 +458,6 @@ public String latestVersionId() { return latest != null ? latest.id : null; } - /** - * The deployed feed version. - * This cannot be returned because of a circular reference between feed source and feed version. Instead, individual - * parameters (version id, start date and end date) are returned. - */ - @JsonIgnore - @BsonIgnore - private FeedVersionDeployed deployedFeedVersion; - - /** - * This value is set to true once an attempt has been made to get the deployed feed version. This prevents subsequent - * attempts by Json annotated properties to get a deployed feed version that is not available. - */ - @JsonIgnore - @BsonIgnore - private boolean deployedFeedVersionDefined; - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("deployedFeedVersionId") - @BsonIgnore - public String getDeployedFeedVersionId() { - deployedFeedVersion = retrieveDeployedFeedVersion(); - return deployedFeedVersion != null ? deployedFeedVersion.id : null; - } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("deployedFeedVersionStartDate") - @BsonIgnore - public LocalDate getDeployedFeedVersionStartDate() { - deployedFeedVersion = retrieveDeployedFeedVersion(); - return deployedFeedVersion != null ? deployedFeedVersion.startDate : null; - } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("deployedFeedVersionEndDate") - @BsonIgnore - public LocalDate getDeployedFeedVersionEndDate() { - deployedFeedVersion = retrieveDeployedFeedVersion(); - return deployedFeedVersion != null ? deployedFeedVersion.endDate : null; - } - - /** - * Get deployed feed version for this feed source. - * - * If a project has a "pinned" deployment, return the feed version from this pinned deployment. If it is not - * available return null and don't attempt to get the feed version from the latest deployment. - * - * If a project does not have a "pinned" deployment, return the latest deployment's feed versions for this feed - * source, if available. - */ - public FeedVersionDeployed retrieveDeployedFeedVersion() { - if (deployedFeedVersionDefined) { - return deployedFeedVersion; - } - deployedFeedVersion = hasPinnedDeployment(projectId) - ? FeedVersionDeployed.getFeedVersionFromPinnedDeployment(projectId, id) - : FeedVersionDeployed.getFeedVersionFromLatestDeployment(projectId, id); - deployedFeedVersionDefined = true; - return deployedFeedVersion; - } - /** * Number of {@link FeedVersion}s that exist for the feed source. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java new file mode 100644 index 000000000..484760c32 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -0,0 +1,527 @@ +package com.conveyal.datatools.manager.models; + +import com.conveyal.datatools.editor.utils.JacksonSerializers; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.validator.ValidationResult; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.collect.Lists; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.Sorts; +import org.bson.Document; +import org.bson.conversions.Bson; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.mongodb.client.model.Aggregates.group; +import static com.mongodb.client.model.Aggregates.limit; +import static com.mongodb.client.model.Aggregates.lookup; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Aggregates.project; +import static com.mongodb.client.model.Aggregates.replaceRoot; +import static com.mongodb.client.model.Aggregates.sort; +import static com.mongodb.client.model.Aggregates.unwind; +import static com.mongodb.client.model.Filters.in; + +public class FeedSourceSummary { + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + + public String id; + public String name; + public boolean deployable; + public boolean isPublic; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate lastUpdated; + + public String deployedFeedVersionId; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate deployedFeedVersionStartDate; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate deployedFeedVersionEndDate; + + public Integer deployedFeedVersionIssues; + + public String latestFeedVersionId; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate latestFeedVersionStartDate; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate latestFeedVersionEndDate; + + public Integer latestFeedVersionIssues; + + public FeedSourceSummary() { + } + + public FeedSourceSummary(Document feedSourceDocument) { + this.id = feedSourceDocument.getString("_id"); + this.name = feedSourceDocument.getString("name"); + this.deployable = feedSourceDocument.getBoolean("deployable"); + this.isPublic = feedSourceDocument.getBoolean("isPublic"); + // Convert to local date type for consistency. + this.lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); + } + + /** + * Set the appropriate feed version. For consistency, if no error count is available set the related number of + * issues to null. + */ + public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDeployed) { + if (feedVersionSummary != null) { + if (isDeployed) { + this.deployedFeedVersionId = feedVersionSummary.id; + this.deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; + this.deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; + this.deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + ? null + : feedVersionSummary.validationResult.errorCount; + } else { + this.latestFeedVersionId = feedVersionSummary.id; + this.latestFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; + this.latestFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; + this.latestFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + ? null + : feedVersionSummary.validationResult.errorCount; + } + } + } + + /** + * Get all feed source summaries matching the project id. + */ + public static List getFeedSourceSummaries(String projectId) { + /* + db.getCollection('FeedSource').aggregate([ + { + // Match provided project id. + $match: { + projectId: "" + } + }, + { + $project: { + "_id": 1, + "name": 1, + "deployable": 1, + "isPublic": 1, + "lastUpdated": 1 + } + }, + { + $sort: { + "name": 1 + } + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("projectId", projectId) + ), + project( + Projections.fields(Projections.include( + "_id", + "name", + "deployable", + "isPublic", + "lastUpdated") + ) + ), + sort(Sorts.ascending("name")) + ); + return extractFeedSourceSummaries(stages); + } + + /** + * Get the latest feed version from all feed sources for this project. + */ + public static Map getLatestFeedVersionForFeedSources(String projectId) { + /* + Note: To test this script: + 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 2) Run FeedSourceControllerTest to created required objects referenced here. + 3) Once complete, delete documents via MongoDB. + 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 5) Re-run FeedSourceControllerTest to confirm deletion of objects. + + db.getCollection('FeedSource').aggregate([ + { + // Match provided project id. + $match: { + projectId: "project-with-latest-deployment" + } + }, + { + $lookup: { + from: "FeedVersion", + localField: "_id", + foreignField: "feedSourceId", + as: "feedVersions" + } + }, + { + $unwind: "$feedVersions" + }, + { + $group: { + _id: "$_id", + doc: { + $max: { + version: "$feedVersions.version", + feedVersionId: "$feedVersions._id", + firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", + lastCalendarDate: "$feedVersions.validationResult.lastCalendarDate", + issues: "$feedVersions.validationResult.errorCount" + } + } + } + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("projectId", projectId) + ), + lookup("FeedVersion", "_id", "feedSourceId", "feedVersions"), + unwind("$feedVersions"), + group( + "$_id", + Accumulators.first("feedVersionId", "$feedVersions._id"), + Accumulators.first("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), + Accumulators.first("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), + Accumulators.first("errorCount", "$feedVersions.validationResult.errorCount") + ) + ); + return extractFeedVersionSummaries( + "FeedSource", + "feedVersionId", + "_id", + false, + stages); + } + + /** + * Get the deployed feed versions from the latest deployment for this project. + */ + public static Map getFeedVersionsFromLatestDeployment(String projectId) { + /* + Note: To test this script: + 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 2) Run FeedSourceControllerTest to created required objects referenced here. + 3) Once complete, delete documents via MongoDB. + 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 5) Re-run FeedSourceControllerTest to confirm deletion of objects. + + db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "project-with-latest-deployment" + } + }, + { + // Get all deployments for this project. + $lookup:{ + from:"Deployment", + localField:"_id", + foreignField:"projectId", + as:"deployment" + } + }, + { + // Deconstruct deployments array to a document for each element. + $unwind: "$deployment" + }, + { + // Make the deployment documents the input/root document. + "$replaceRoot": { + "newRoot": "$deployment" + } + }, + { + // Sort descending. + $sort: { + lastUpdated : -1 + } + }, + { + // At this point we will have the latest deployment for a project. + $limit: 1 + }, + { + $lookup:{ + from:"FeedVersion", + localField:"feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + // Deconstruct feedVersions array to a document for each element. + $unwind: "$feedVersions" + }, + { + // Make the feed version documents the input/root document. + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $project: { + "_id": 1, + "feedSourceId": 1, + "validationResult.firstCalendarDate": 1, + "validationResult.lastCalendarDate": 1, + "validationResult.errorCount": 1 + } + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("_id", projectId) + ), + lookup("Deployment", "_id", "projectId", "deployments"), + unwind("$deployments"), + replaceRoot("$deployments"), + sort(Sorts.descending("lastUpdated")), + limit(1), + lookup("FeedVersion", "feedVersionIds", "_id", "feedVersions"), + unwind("$feedVersions"), + replaceRoot("$feedVersions"), + project( + Projections.fields(Projections.include( + "feedSourceId", + "validationResult.firstCalendarDate", + "validationResult.lastCalendarDate", + "validationResult.errorCount") + ) + ) + ); + return extractFeedVersionSummaries( + "Project", + "_id", + "feedSourceId", + true, + stages); + } + + /** + * Get the deployed feed version from the pinned deployment for this feed source. + */ + public static Map getFeedVersionsFromPinnedDeployment(String projectId) { + /* + Note: To test this script: + 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 2) Run FeedSourceControllerTest to created required objects referenced here. + 3) Once complete, delete documents via MongoDB. + 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 5) Re-run FeedSourceControllerTest to confirm deletion of objects. + + db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "project-with-pinned-deployment" + } + }, + { + $project: { + pinnedDeploymentId: 1 + } + }, + { + $lookup:{ + from:"Deployment", + localField:"pinnedDeploymentId", + foreignField:"_id", + as:"deployment" + } + }, + { + $unwind: "$deployment" + }, + { + $lookup:{ + from:"FeedVersion", + localField:"deployment.feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + // Deconstruct feedVersions array to a document for each element. + $unwind: "$feedVersions" + }, + { + // Make the feed version documents the input/root document. + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $project: { + "_id": 1, + "feedSourceId": 1, + "validationResult.firstCalendarDate": 1, + "validationResult.lastCalendarDate": 1, + "validationResult.errorCount": 1 + } + } + ]) + */ + + List stages = Lists.newArrayList( + match( + in("_id", projectId) + ), + project( + Projections.fields(Projections.include("pinnedDeploymentId")) + ), + lookup("Deployment", "pinnedDeploymentId", "_id", "deployment"), + unwind("$deployment"), + lookup("FeedVersion", "deployment.feedVersionIds", "_id", "feedVersions"), + unwind("$feedVersions"), + replaceRoot("$feedVersions"), + project( + Projections.fields(Projections.include( + "feedSourceId", + "validationResult.firstCalendarDate", + "validationResult.lastCalendarDate", + "validationResult.errorCount") + ) + ) + ); + return extractFeedVersionSummaries( + "Project", + "_id", + "feedSourceId", + true, + stages); + } + + + /** + * Produce a list of all feed source summaries for a project. + */ + private static List extractFeedSourceSummaries(List stages) { + List feedSourceSummaries = new ArrayList<>(); + for (Document feedSourceDocument : Persistence.getDocuments("FeedSource", stages)) { + feedSourceSummaries.add(new FeedSourceSummary(feedSourceDocument)); + } + return feedSourceSummaries; + } + + /** + * Extract feed version summaries from feed version documents. Each feed version is held against the matching feed + * source. + */ + private static Map extractFeedVersionSummaries( + String collection, + String feedVersionKey, + String feedSourceKey, + boolean hasChildValidationResultDocument, + List stages + ) { + Map feedVersionSummaries = new HashMap<>(); + for (Document feedVersionDocument : Persistence.getDocuments(collection, stages)) { + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); + feedVersionSummary.id = feedVersionDocument.getString(feedVersionKey); + feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); + feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); + } + return feedVersionSummaries; + } + + /** + * Build validation result from feed version document. + */ + private static ValidationResult getValidationResult(boolean hasChildValidationResultDocument, Document feedVersionDocument) { + ValidationResult validationResult = new ValidationResult(); + validationResult.errorCount = getValidationResultErrorCount(hasChildValidationResultDocument, feedVersionDocument); + validationResult.firstCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "firstCalendarDate"); + validationResult.lastCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "lastCalendarDate"); + return validationResult; + } + + private static LocalDate getValidationResultDate( + boolean hasChildValidationResultDocument, + Document feedVersionDocument, + String key + ) { + return (hasChildValidationResultDocument) + ? getDateFieldFromDocument(feedVersionDocument, key) + : getDateFromString(feedVersionDocument.getString(key)); + } + + /** + * Extract date value from validation result document. + */ + private static LocalDate getDateFieldFromDocument(Document document, String dateKey) { + Document validationResult = getDocumentChild(document, "validationResult"); + return (validationResult != null) + ? getDateFromString(validationResult.getString(dateKey)) + : null; + } + + /** + * Extract the error count from the parent document or child validation result document. If the error count is not + * available, return -1. + */ + private static int getValidationResultErrorCount(boolean hasChildValidationResultDocument, Document feedVersionDocument) { + int errorCount; + try { + errorCount = (hasChildValidationResultDocument) + ? getErrorCount(feedVersionDocument) + : feedVersionDocument.getInteger("errorCount"); + } catch (NullPointerException e) { + errorCount = -1; + } + return errorCount; + } + + /** + * Get the child validation result document and extract the error count from this. + */ + private static int getErrorCount(Document document) { + return getDocumentChild(document, "validationResult").getInteger("errorCount"); + } + + /** + * Extract child document matching provided name. + */ + private static Document getDocumentChild(Document document, String name) { + return (Document) document.get(name); + } + + /** + * Convert String date (if not null) into LocalDate. + */ + private static LocalDate getDateFromString(String date) { + return (date == null) ? null : LocalDate.parse(date, formatter); + } + + /** + * Convert Date object into LocalDate object. + */ + private static LocalDate getLocalDateFromDate(Date date) { + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java deleted file mode 100644 index 4fafc888d..000000000 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java +++ /dev/null @@ -1,239 +0,0 @@ -package com.conveyal.datatools.manager.models; - -import com.conveyal.datatools.editor.utils.JacksonSerializers; -import com.conveyal.datatools.manager.persistence.Persistence; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.google.common.collect.Lists; -import com.mongodb.client.model.Sorts; -import org.bson.Document; -import org.bson.conversions.Bson; - -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.List; - -import static com.mongodb.client.model.Aggregates.limit; -import static com.mongodb.client.model.Aggregates.lookup; -import static com.mongodb.client.model.Aggregates.match; -import static com.mongodb.client.model.Aggregates.replaceRoot; -import static com.mongodb.client.model.Aggregates.sort; -import static com.mongodb.client.model.Aggregates.unwind; -import static com.mongodb.client.model.Filters.in; - -public class FeedVersionDeployed { - public String id; - - @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) - @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate startDate; - - @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) - @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate endDate; - - public FeedVersionDeployed() { - } - - public FeedVersionDeployed(Document feedVersionDocument) { - this.id = feedVersionDocument.getString("_id"); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); - Document validationResult = (Document) feedVersionDocument.get("validationResult"); - if (validationResult != null) { - String first = validationResult.getString("firstCalendarDate"); - String last = validationResult.getString("lastCalendarDate"); - this.startDate = (first == null) ? null : LocalDate.parse(first, formatter); - this.endDate = (last == null) ? null : LocalDate.parse(last, formatter); - } - } - - /** - * Get the deployed feed version from the pinned deployment for this feed source. - */ - public static FeedVersionDeployed getFeedVersionFromPinnedDeployment(String projectId, String feedSourceId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('Project').aggregate([ - { - $match: { - _id: "project-with-pinned-deployment" - } - }, - { - $lookup:{ - from:"Deployment", - localField:"pinnedDeploymentId", - foreignField:"_id", - as:"deployment" - } - }, - { - $lookup:{ - from:"FeedVersion", - localField:"deployment.feedVersionIds", - foreignField:"_id", - as:"feedVersions" - } - }, - { - $unwind: "$feedVersions" - }, - { - "$replaceRoot": { - "newRoot": "$feedVersions" - } - }, - { - $match: { - feedSourceId: "feed-source-with-pinned-deployment-feed-version" - } - }, - { - $sort: { - lastUpdated : -1 - } - }, - { - $limit: 1 - } - ]) - */ - List stages = Lists.newArrayList( - match( - in("_id", projectId) - ), - lookup("Deployment", "pinnedDeploymentId", "_id", "deployment"), - lookup("FeedVersion", "deployment.feedVersionIds", "_id", "feedVersions"), - unwind("$feedVersions"), - replaceRoot("$feedVersions"), - match( - in("feedSourceId", feedSourceId) - ), - // If more than one feed version for a feed source is held against a deployment the latest is used. - sort(Sorts.descending("lastUpdated")), - limit(1) - ); - return getFeedVersionDeployed(stages); - } - - /** - * Get the deployed feed version from the latest deployment for this feed source. - */ - public static FeedVersionDeployed getFeedVersionFromLatestDeployment(String projectId, String feedSourceId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('Project').aggregate([ - { - // Match provided project id. - $match: { - _id: "project-with-latest-deployment" - } - }, - { - // Get all deployments for this project. - $lookup:{ - from:"Deployment", - localField:"_id", - foreignField:"projectId", - as:"deployments" - } - }, - { - // Deconstruct deployments array to a document for each element. - $unwind: "$deployments" - }, - { - // Make the deployment documents the input/root document. - "$replaceRoot": { - "newRoot": "$deployments" - } - }, - { - // Sort descending. - $sort: { - lastUpdated : -1 - } - }, - { - // At this point we will have the latest deployment for a project. - $limit: 1 - }, - { - // Get all feed versions that have been deployed as part of the latest deployment. - $lookup:{ - from:"FeedVersion", - localField:"feedVersionIds", - foreignField:"_id", - as:"feedVersions" - } - }, - { - // Deconstruct feedVersions array to a document for each element. - $unwind: "$feedVersions" - }, - { - // Make the feed version documents the input/root document. - "$replaceRoot": { - "newRoot": "$feedVersions" - } - }, - { - // Match the required feed source. - $match: { - feedSourceId: "feed-source-with-latest-deployment-feed-version" - } - }, - { - $sort: { - lastUpdated : -1 - } - }, - { - // At this point we will have the latest feed version from the latest deployment for a feed source. - $limit: 1 - } - ]) - */ - List stages = Lists.newArrayList( - match( - in("_id", projectId) - ), - lookup("Deployment", "_id", "projectId", "deployments"), - unwind("$deployments"), - replaceRoot("$deployments"), - sort(Sorts.descending("lastUpdated")), - limit(1), - lookup("FeedVersion", "feedVersionIds", "_id", "feedVersions"), - unwind("$feedVersions"), - replaceRoot("$feedVersions"), - match( - in("feedSourceId", feedSourceId) - ), - // If more than one feed version for a feed source is held against a deployment the latest is used. - sort(Sorts.descending("lastUpdated")), - limit(1) - ); - return getFeedVersionDeployed(stages); - } - - private static FeedVersionDeployed getFeedVersionDeployed(List stages) { - Document feedVersionDocument = Persistence - .getMongoDatabase() - .getCollection("Project") - .aggregate(stages) - .first(); - return (feedVersionDocument == null) ? null : new FeedVersionDeployed(feedVersionDocument); - } -} diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index ddd2298a2..b868c6b45 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -17,6 +17,7 @@ import java.util.Date; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -164,6 +165,24 @@ public Collection retrieveDeploymentSummaries() { .collect(Collectors.toList()); } + /** + * Get all feed source summaries for this project. + */ + public Collection retrieveFeedSourceSummaries() { + List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id); + Map latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id); + Map deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); + if (deployedFeedVersions.isEmpty()) { + // No pinned deployments, instead, get the deployed feed versions from the latest deployment. + deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); + } + for (FeedSourceSummary feedSourceSummary : feedSourceSummaries) { + feedSourceSummary.setFeedVersion(latestFeedVersionForFeedSources.get(feedSourceSummary.id), false); + feedSourceSummary.setFeedVersion(deployedFeedVersions.get(feedSourceSummary.id), true); + } + return feedSourceSummaries; + } + // TODO: Does this need to be returned with JSON API response public Organization retrieveOrganization() { if (organizationId != null) { diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java index 2bd79ef6f..e215c6742 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java @@ -18,15 +18,20 @@ import com.conveyal.datatools.manager.models.Snapshot; import com.mongodb.ConnectionString; import com.mongodb.MongoClientSettings; +import com.mongodb.client.AggregateIterable; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; +import org.bson.Document; import org.bson.codecs.configuration.CodecRegistries; import org.bson.codecs.configuration.CodecRegistry; import org.bson.codecs.pojo.PojoCodecProvider; +import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; /** @@ -135,5 +140,12 @@ public static void initialize () { public static MongoDatabase getMongoDatabase() { return mongoDatabase; } - + + /** + * Get all bespoke documents matching query. These documents are tailored to the query response and are not tied + * directly to a persistence type. + */ + public static AggregateIterable getDocuments(String collection, List stages) { + return getMongoDatabase().getCollection(collection).aggregate(stages); + } } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index e93b9baaf..a32474337 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -7,6 +7,7 @@ import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedSourceSummary; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; @@ -323,46 +324,51 @@ public void createFeedSourceWithLabels() { void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( String.format( - "/api/manager/secure/feedsource/%s", - feedSourceWithLatestDeploymentFeedVersion.id + "/api/manager/secure/feedsourceSummaries?projectId=%s", + feedSourceWithLatestDeploymentFeedVersion.projectId ), null, HttpUtils.REQUEST_METHOD.GET ); assertEquals(OK_200, response.status); - FeedSource feedSource = - JsonUtil.getPOJOFromResponse( - response, - FeedSource.class + + List feedSourceSummaries = + JsonUtil.getPOJOFromJSONAsList( + JsonUtil.getJsonNodeFromResponse(response), + FeedSourceSummary.class ); - assertNotNull(feedSource); - assertEquals(feedSourceWithLatestDeploymentFeedVersion.id, feedSource.id); - assertEquals(feedVersionFromLatestDeployment.id, feedSource.getDeployedFeedVersionId()); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSource.getDeployedFeedVersionStartDate()); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSource.getDeployedFeedVersionEndDate()); + + assertNotNull(feedSourceSummaries); + assertEquals(feedSourceWithLatestDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); + assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); } @Test void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( String.format( - "/api/manager/secure/feedsource/%s", - feedSourceWithPinnedDeploymentFeedVersion.id + "/api/manager/secure/feedsourceSummaries?projectId=%s", + feedSourceWithPinnedDeploymentFeedVersion.projectId ), null, HttpUtils.REQUEST_METHOD.GET ); assertEquals(OK_200, response.status); - FeedSource feedSource = - JsonUtil.getPOJOFromResponse( - response, - FeedSource.class + + List feedSourceSummaries = + JsonUtil.getPOJOFromJSONAsList( + JsonUtil.getJsonNodeFromResponse(response), + FeedSourceSummary.class ); - assertNotNull(feedSource); - assertEquals(feedSourceWithPinnedDeploymentFeedVersion.id, feedSource.id); - assertEquals(feedVersionFromPinnedDeployment.id, feedSource.getDeployedFeedVersionId()); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSource.getDeployedFeedVersionEndDate()); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSource.getDeployedFeedVersionStartDate()); + assertNotNull(feedSourceSummaries); + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -421,6 +427,7 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc ValidationResult validationResult = new ValidationResult(); validationResult.firstCalendarDate = startDate; validationResult.lastCalendarDate = endDate; + validationResult.errorCount = 5 + (int)(Math.random() * ((1000 - 5) + 1)); feedVersion.validationResult = validationResult; Persistence.feedVersions.create(feedVersion); return feedVersion; From 62eb590cc32e95baa65cac54a515ebb30b2d49c9 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 20 Jun 2023 11:52:09 +0100 Subject: [PATCH 3/8] refactor(Update to include label ids): Updated to get label ids from DB for a feed source --- .../manager/models/FeedSourceSummary.java | 9 +++++-- .../api/FeedSourceControllerTest.java | 27 +++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 484760c32..ccce4a0da 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -43,6 +43,8 @@ public class FeedSourceSummary { @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) public LocalDate lastUpdated; + public List labelIds = new ArrayList<>(); + public String deployedFeedVersionId; @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) @@ -75,6 +77,7 @@ public FeedSourceSummary(Document feedSourceDocument) { this.name = feedSourceDocument.getString("name"); this.deployable = feedSourceDocument.getBoolean("deployable"); this.isPublic = feedSourceDocument.getBoolean("isPublic"); + this.labelIds = feedSourceDocument.getList("labelIds", String.class); // Convert to local date type for consistency. this.lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); } @@ -121,7 +124,8 @@ public static List getFeedSourceSummaries(String projectId) { "name": 1, "deployable": 1, "isPublic": 1, - "lastUpdated": 1 + "lastUpdated": 1, + "labelIds": 1 } }, { @@ -141,7 +145,8 @@ public static List getFeedSourceSummaries(String projectId) { "name", "deployable", "isPublic", - "lastUpdated") + "lastUpdated", + "labelIds") ) ), sort(Sorts.ascending("name")) diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index a32474337..217b05586 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -91,7 +91,8 @@ public static void setUp() throws IOException { "FeedSource", null, projectWithLatestDeployment, - true + true, + List.of("labelOne", "labelTwo") ); LocalDate deployedSuperseded = LocalDate.of(2020, Month.MARCH, 12); LocalDate deployedEndDate = LocalDate.of(2021, Month.MARCH, 12); @@ -124,7 +125,8 @@ public static void setUp() throws IOException { "FeedSourceWithPinnedFeedVersion", null, projectWithPinnedDeployment, - true + true, + List.of("labelOne", "labelTwo") ); feedVersionFromPinnedDeployment = createFeedVersion( "feed-version-from-pinned-deployment", @@ -340,10 +342,15 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertNotNull(feedSourceSummaries); assertEquals(feedSourceWithLatestDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedSourceWithLatestDeploymentFeedVersion.labelIds, feedSourceSummaries.get(0).labelIds); assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); + assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).latestFeedVersionId); + assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestFeedVersionStartDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestFeedVersionEndDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestFeedVersionIssues); } @Test @@ -365,10 +372,15 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { ); assertNotNull(feedSourceSummaries); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.labelIds, feedSourceSummaries.get(0).labelIds); assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); + assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).latestFeedVersionId); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestFeedVersionStartDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestFeedVersionEndDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestFeedVersionIssues); } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -379,6 +391,16 @@ private static FeedSource createFeedSource(String name, URL url, Project project * Helper method to create feed source. */ private static FeedSource createFeedSource(String id, String name, URL url, Project project, boolean persist) { + return createFeedSource(id, name, url, project, persist, null); + } + private static FeedSource createFeedSource( + String id, + String name, + URL url, + Project project, + boolean persist, + List labels + ) { FeedSource feedSource = new FeedSource(); if (id != null) feedSource.id = id; feedSource.fetchFrequency = FetchFrequency.MINUTES; @@ -388,6 +410,7 @@ private static FeedSource createFeedSource(String id, String name, URL url, Proj feedSource.projectId = project.id; feedSource.retrievalMethod = FeedRetrievalMethod.FETCHED_AUTOMATICALLY; feedSource.url = url; + if (labels != null) feedSource.labelIds = labels; if (persist) Persistence.feedSources.create(feedSource); return feedSource; } From 268b949ba5a8a22b43dc9d59795848696ec9f898 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 14 Aug 2023 12:08:54 +0100 Subject: [PATCH 4/8] refactor(Added project id to feed source summary): --- .../datatools/manager/models/FeedSourceSummary.java | 12 ++++++++---- .../controllers/api/FeedSourceControllerTest.java | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index ccce4a0da..1f380d541 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -34,7 +34,10 @@ public class FeedSourceSummary { private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + public String projectId; + public String id; + public String name; public boolean deployable; public boolean isPublic; @@ -72,7 +75,8 @@ public class FeedSourceSummary { public FeedSourceSummary() { } - public FeedSourceSummary(Document feedSourceDocument) { + public FeedSourceSummary(String projectId, Document feedSourceDocument) { + this.projectId = projectId; this.id = feedSourceDocument.getString("_id"); this.name = feedSourceDocument.getString("name"); this.deployable = feedSourceDocument.getBoolean("deployable"); @@ -151,7 +155,7 @@ public static List getFeedSourceSummaries(String projectId) { ), sort(Sorts.ascending("name")) ); - return extractFeedSourceSummaries(stages); + return extractFeedSourceSummaries(projectId, stages); } /** @@ -426,10 +430,10 @@ public static Map getFeedVersionsFromPinnedDeploymen /** * Produce a list of all feed source summaries for a project. */ - private static List extractFeedSourceSummaries(List stages) { + private static List extractFeedSourceSummaries(String projectId, List stages) { List feedSourceSummaries = new ArrayList<>(); for (Document feedSourceDocument : Persistence.getDocuments("FeedSource", stages)) { - feedSourceSummaries.add(new FeedSourceSummary(feedSourceDocument)); + feedSourceSummaries.add(new FeedSourceSummary(projectId, feedSourceDocument)); } return feedSourceSummaries; } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 217b05586..1f58ea1ce 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -342,6 +342,7 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertNotNull(feedSourceSummaries); assertEquals(feedSourceWithLatestDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedSourceWithLatestDeploymentFeedVersion.projectId, feedSourceSummaries.get(0).projectId); assertEquals(feedSourceWithLatestDeploymentFeedVersion.labelIds, feedSourceSummaries.get(0).labelIds); assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); @@ -372,6 +373,7 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { ); assertNotNull(feedSourceSummaries); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.projectId, feedSourceSummaries.get(0).projectId); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.labelIds, feedSourceSummaries.get(0).labelIds); assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); From 1f580545d80a57f456ee337492f5b5fb6a3d05d5 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 14 Aug 2023 12:19:27 +0100 Subject: [PATCH 5/8] refactor(pom.xml): Updated gtfs-lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a265b5442..1566ccefa 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - f2ceb59 + f2ceb59027 From be6f44030750a5734372436e775f174dd0dcee52 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 18 Aug 2023 09:59:15 +0100 Subject: [PATCH 6/8] refactor(Update to include a new private class to hold latest validation result): This new class ali --- .../manager/models/FeedSourceSummary.java | 46 ++++++++++++------- .../api/FeedSourceControllerTest.java | 16 +++---- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 1f380d541..931fb49f3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -60,17 +60,7 @@ public class FeedSourceSummary { public Integer deployedFeedVersionIssues; - public String latestFeedVersionId; - - @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) - @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate latestFeedVersionStartDate; - - @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) - @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate latestFeedVersionEndDate; - - public Integer latestFeedVersionIssues; + public LatestValidationResult latestValidation; public FeedSourceSummary() { } @@ -100,12 +90,7 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl ? null : feedVersionSummary.validationResult.errorCount; } else { - this.latestFeedVersionId = feedVersionSummary.id; - this.latestFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; - this.latestFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; - this.latestFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) - ? null - : feedVersionSummary.validationResult.errorCount; + this.latestValidation = new LatestValidationResult(feedVersionSummary); } } } @@ -533,4 +518,31 @@ private static LocalDate getDateFromString(String date) { private static LocalDate getLocalDateFromDate(Date date) { return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); } + + public static class LatestValidationResult { + + public String feedVersionId; + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate feedVersionStartDate; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate feedVersionEndDate; + + public Integer feedVersionIssues; + + /** Required for JSON de/serializing. **/ + public LatestValidationResult() {} + + LatestValidationResult(FeedVersionSummary feedVersionSummary) { + this.feedVersionId = feedVersionSummary.id; + this.feedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; + this.feedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; + this.feedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + ? null + : feedVersionSummary.validationResult.errorCount; + } + } + } \ No newline at end of file diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 1f58ea1ce..0026e15fc 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -348,10 +348,10 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); - assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).latestFeedVersionId); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestFeedVersionStartDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestFeedVersionEndDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestFeedVersionIssues); + assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); + assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.feedVersionStartDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.feedVersionEndDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.feedVersionIssues); } @Test @@ -379,10 +379,10 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); - assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).latestFeedVersionId); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestFeedVersionStartDate); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestFeedVersionEndDate); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestFeedVersionIssues); + assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.feedVersionStartDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.feedVersionEndDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.feedVersionIssues); } private static FeedSource createFeedSource(String name, URL url, Project project) { From 15d43e7cc8aac9cec6b831a8ac507332b7813200 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 18 Aug 2023 15:10:22 -0400 Subject: [PATCH 7/8] refactor(feedsourceSummary): rename fields to agree with old format --- .../datatools/manager/models/FeedSourceSummary.java | 12 ++++++------ .../controllers/api/FeedSourceControllerTest.java | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 931fb49f3..d472b05c4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -524,22 +524,22 @@ public static class LatestValidationResult { public String feedVersionId; @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate feedVersionStartDate; + public LocalDate startDate; @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate feedVersionEndDate; + public LocalDate endDate; - public Integer feedVersionIssues; + public Integer errorCount; /** Required for JSON de/serializing. **/ public LatestValidationResult() {} LatestValidationResult(FeedVersionSummary feedVersionSummary) { this.feedVersionId = feedVersionSummary.id; - this.feedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; - this.feedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; - this.feedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + this.startDate = feedVersionSummary.validationResult.firstCalendarDate; + this.endDate = feedVersionSummary.validationResult.lastCalendarDate; + this.errorCount = (feedVersionSummary.validationResult.errorCount == -1) ? null : feedVersionSummary.validationResult.errorCount; } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 0026e15fc..21faa0dd9 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -349,9 +349,9 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.feedVersionStartDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.feedVersionEndDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.feedVersionIssues); + assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); } @Test @@ -380,9 +380,9 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.feedVersionStartDate); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.feedVersionEndDate); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.feedVersionIssues); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); } private static FeedSource createFeedSource(String name, URL url, Project project) { From ab782d57e8e8178fcfd6e3262d4141a80c8c7944 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 24 Aug 2023 09:09:59 +0100 Subject: [PATCH 8/8] refactor(FeedSourceSummary.java): Addressed PR feedback --- .../manager/models/FeedSourceSummary.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index d472b05c4..b9917e94a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -71,7 +71,10 @@ public FeedSourceSummary(String projectId, Document feedSourceDocument) { this.name = feedSourceDocument.getString("name"); this.deployable = feedSourceDocument.getBoolean("deployable"); this.isPublic = feedSourceDocument.getBoolean("isPublic"); - this.labelIds = feedSourceDocument.getList("labelIds", String.class); + List documentLabelIds = feedSourceDocument.getList("labelIds", String.class); + if (documentLabelIds != null) { + this.labelIds = documentLabelIds; + } // Convert to local date type for consistency. this.lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); } @@ -87,7 +90,7 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl this.deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; this.deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; this.deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) - ? null + ? 0 : feedVersionSummary.validationResult.errorCount; } else { this.latestValidation = new LatestValidationResult(feedVersionSummary); @@ -341,25 +344,25 @@ public static Map getFeedVersionsFromPinnedDeploymen pinnedDeploymentId: 1 } }, - { + { $lookup:{ from:"Deployment", localField:"pinnedDeploymentId", foreignField:"_id", as:"deployment" } - }, - { - $unwind: "$deployment" - }, - { + }, + { + $unwind: "$deployment" + }, + { $lookup:{ from:"FeedVersion", localField:"deployment.feedVersionIds", foreignField:"_id", as:"feedVersions" } - }, + }, { // Deconstruct feedVersions array to a document for each element. $unwind: "$feedVersions"