Merge pull request #303 from VEuPathDB/issue-222

Remote file download patches.

.gitattributes

@@ -1,2 +1,2 @@
-*.html linguist-generated
-modules/rest-service/schema/library.raml linguist-generated
+*.html linguist-generated=true
+modules/rest-service/schema/library.raml linguist-generated=true

platform/build.gradle.kts

@@ -5,7 +5,7 @@ plugins {
 dependencies {
   constraints {
     api("org.gusdb:fgputil-db:2.13.1")
-    api("org.veupathdb.lib:jaxrs-container-core:7.0.9")
+    api("org.veupathdb.lib:jaxrs-container-core:7.1.0")
     api("org.veupathdb.lib:multipart-jackson-pojo:1.1.7")
 
     // VDI
@@ -29,11 +29,7 @@ dependencies {
 
     // HTTP
     api("io.foxcapades.lib:k-multipart:1.2.1")
-    api("org.glassfish.jersey.core:jersey-server:3.1.5")
-
-    // JSON
-    api("com.fasterxml.jackson.core:jackson-databind:2.17.0")
-    api("com.fasterxml.jackson.core:jackson-annotations:2.17.0")
+    api("org.glassfish.jersey.core:jersey-server")
 
     // Messaging
     api("org.apache.kafka:kafka-clients:3.7.0")

service/rest-service/api.raml

@@ -3,7 +3,7 @@
 extends: https://raw.githubusercontent.com/VEuPathDB/docs-api-schema/v2.2.0/libraries/base-service.raml
 
 title: VEuPathDB Dataset Installer
-version: 2.0.0
+version: 2.1.0
 mediaType: application/json
 
 traits:
@@ -216,6 +216,17 @@ uses:
         body:
           application/json:
             type: lib.UnprocessableEntityError
+      424:
+        description: |
+          Failed Dependency.
+          
+          Returned when the dataset data source was a URL and the VDI service
+          encountered a non-success HTTP status code from the target URL.  This
+          could be, for example, a 403 error from an expired AWS S3 URL, or a
+          404 for a file that no longer exists on the remote server.
+        body:
+          application/json:
+            type: lib.FailedDependencyError
       500:
         description: |
           Internal Server Error.

service/rest-service/build.gradle.kts

@@ -70,7 +70,6 @@ dependencies {
   implementation(kotlin("stdlib-jdk8"))
   implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core")
 
-
   // Jersey
   implementation("org.glassfish.jersey.core:jersey-server")
 

service/rest-service/schema/common.raml

@@ -1,7 +1,7 @@
 #%RAML 1.0 Library
 
 uses:
-  error: https://raw.githubusercontent.com/VEuPathDB/docs-api-schema/v2.2.1/libraries/errors.raml
+  error: https://raw.githubusercontent.com/VEuPathDB/docs-api-schema/v2.3.0/libraries/errors.raml
 
 types:
   VDI-ID:

service/rest-service/schema/library.raml

@@ -777,6 +777,7 @@ types:
       - conflict
       - gone
       - invalid-input
+      - failed-dependency
       - server-error
   BadRequestError:
     displayName: Bad Request
@@ -872,6 +873,16 @@ types:
         general: []
         byKey:
           id: [Given ID value does not point to an existing record.]
+  FailedDependencyError:
+    displayName: Failed Dependency Error
+    type: Error
+    discriminatorValue: failed-dependency
+    properties:
+      dependency: string
+    example:
+      status: failed-dependency
+      dependency: google.com
+      message: unexpected status code 403 from google.com
   ServerError:
     displayName: Internal Server Error
     type: Error

service/rest-service/src/main/java/org/veupathdb/service/vdi/generated/model/Error.java

@@ -13,13 +13,14 @@
 @JsonSubTypes({
     @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.ConflictError.class),
     @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.MethodNotAllowedError.class),
+    @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.FailedDependencyError.class),
     @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.ServerError.class),
     @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.BadRequestError.class),
     @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.UnauthorizedError.class),
-    @JsonSubTypes.Type(java.lang.String.class),
-    @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.UnprocessableEntityError.class),
     @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.NotFoundError.class),
     @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.ForbiddenError.class),
+    @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.UnprocessableEntityError.class),
+    @JsonSubTypes.Type(java.lang.String.class),
     @JsonSubTypes.Type(org.veupathdb.service.vdi.generated.model.Error.class)
 })
 @JsonDeserialize(

service/rest-service/src/main/java/org/veupathdb/service/vdi/generated/model/ErrorType.java

@@ -27,6 +27,9 @@ public enum ErrorType {
   @JsonProperty("invalid-input")
   INVALIDINPUT("invalid-input"),
 
+  @JsonProperty("failed-dependency")
+  FAILEDDEPENDENCY("failed-dependency"),
+
   @JsonProperty("server-error")
   SERVERERROR("server-error");
 

service/rest-service/src/main/java/org/veupathdb/service/vdi/generated/model/FailedDependencyError.java

@@ -0,0 +1,28 @@
+package org.veupathdb.service.vdi.generated.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+@JsonTypeName("failed-dependency")
+@JsonDeserialize(
+    as = FailedDependencyErrorImpl.class
+)
+public interface FailedDependencyError extends Error {
+  ErrorType _DISCRIMINATOR_TYPE_NAME = ErrorType.FAILEDDEPENDENCY;
+
+  @JsonProperty("status")
+  ErrorType getStatus();
+
+  @JsonProperty("message")
+  String getMessage();
+
+  @JsonProperty("message")
+  void setMessage(String message);
+
+  @JsonProperty("dependency")
+  String getDependency();
+
+  @JsonProperty("dependency")
+  void setDependency(String dependency);
+}

service/rest-service/src/main/java/org/veupathdb/service/vdi/generated/model/FailedDependencyErrorImpl.java

@@ -0,0 +1,49 @@
+package org.veupathdb.service.vdi.generated.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonTypeName("failed-dependency")
+@JsonPropertyOrder({
+    "status",
+    "message",
+    "dependency"
+})
+public class FailedDependencyErrorImpl implements FailedDependencyError {
+  @JsonProperty("status")
+  private final ErrorType status = _DISCRIMINATOR_TYPE_NAME;
+
+  @JsonProperty("message")
+  private String message;
+
+  @JsonProperty("dependency")
+  private String dependency;
+
+  @JsonProperty("status")
+  public ErrorType getStatus() {
+    return this.status;
+  }
+
+  @JsonProperty("message")
+  public String getMessage() {
+    return this.message;
+  }
+
+  @JsonProperty("message")
+  public void setMessage(String message) {
+    this.message = message;
+  }
+
+  @JsonProperty("dependency")
+  public String getDependency() {
+    return this.dependency;
+  }
+
+  @JsonProperty("dependency")
+  public void setDependency(String dependency) {
+    this.dependency = dependency;
+  }
+}

service/rest-service/src/main/java/org/veupathdb/service/vdi/generated/model/FailedDependencyErrorStream.java

@@ -0,0 +1,23 @@
+
+package org.veupathdb.service.vdi.generated.model;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.function.Consumer;
+
+import jakarta.ws.rs.WebApplicationException;
+import jakarta.ws.rs.core.StreamingOutput;
+
+public class FailedDependencyErrorStream extends FailedDependencyErrorImpl implements StreamingOutput {
+
+  private final Consumer<OutputStream> _streamer;
+
+  public FailedDependencyErrorStream(Consumer<OutputStream> streamer) {
+    _streamer = streamer;
+  }
+
+  @Override
+  public void write(OutputStream output) throws IOException, WebApplicationException {
+    _streamer.accept(output);
+  }
+}

service/rest-service/src/main/kotlin/org/veupathdb/service/vdi/generated/model/readme.adoc

@@ -0,0 +1,7 @@
+= Generated Java Model Extensions
+
+This package contains extension/mixin helper methods for working with the model
+Java classes generated by the RAML -> JaxRS code generator.
+
+The file names all follow the scheme `x-\{generated class name}` where the
+prefix `x` is short for 'extension'.

service/rest-service/src/main/kotlin/org/veupathdb/service/vdi/generated/model/x-DatasetPostMeta.kt

@@ -2,6 +2,25 @@ package org.veupathdb.service.vdi.generated.model
 
 import org.veupathdb.service.vdi.util.ValidationErrors
 
+internal fun DatasetPostMeta.cleanup() {
+  datasetType?.cleanup()
+
+  name = name?.trim()
+  summary = summary?.takeIf { it.isNotBlank() }
+    ?.trim()
+  description = description?.takeIf { it.isNotBlank() }
+    ?.trim()
+  origin = origin?.trim()
+
+  projects?.forEachIndexed { i, s -> projects[i] = s?.takeIf { it.isNotBlank() } ?.trim() }
+
+  dependencies?.forEach {
+    it.resourceVersion = it.resourceVersion?.takeIf { it.isNotBlank() } ?.trim()
+    it.resourceIdentifier = it.resourceIdentifier?.takeIf { it.isNotBlank() } ?.trim()
+    it.resourceDisplayName = it.resourceDisplayName?.takeIf { it.isNotBlank() } ?.trim()
+  }
+}
+
 internal fun DatasetPostMeta.validate(validationErrors: ValidationErrors) {
   if (datasetType == null)
     validationErrors.add("meta.type", "field is required")

service/rest-service/src/main/kotlin/org/veupathdb/service/vdi/generated/model/x-DatasetPostRequest.kt

@@ -1,8 +1,19 @@
 package org.veupathdb.service.vdi.generated.model
 
 import org.veupathdb.service.vdi.util.ValidationErrors
+import org.veupathdb.vdi.lib.common.field.UserID
+import org.veupathdb.vdi.lib.common.model.VDIDatasetDependencyImpl
+import org.veupathdb.vdi.lib.common.model.VDIDatasetMetaImpl
+import org.veupathdb.vdi.lib.common.model.VDIDatasetTypeImpl
+import org.veupathdb.vdi.lib.common.model.VDIDatasetVisibility
+import java.time.OffsetDateTime
 
-fun DatasetPostRequest.validate(): ValidationErrors {
+internal fun DatasetPostRequest.cleanup() {
+  meta?.cleanup()
+  url = url?.takeIf { it.isNotBlank() } ?.trim()
+}
+
+internal fun DatasetPostRequest.validate(): ValidationErrors {
   val validationErrors = ValidationErrors()
 
   if (meta == null)
@@ -18,3 +29,26 @@ fun DatasetPostRequest.validate(): ValidationErrors {
   return validationErrors
 }
 
+internal fun DatasetPostRequest.toDatasetMeta(userID: UserID) =
+  VDIDatasetMetaImpl(
+    type         = VDIDatasetTypeImpl(
+      name    = meta.datasetType.name.lowercase(),
+      version = meta.datasetType.version,
+    ),
+    projects     = meta.projects.toSet(),
+    owner        = userID,
+    name         = meta.name,
+    summary      = meta.summary,
+    description  = meta.description,
+    visibility   = meta.visibility?.toInternalVisibility() ?: VDIDatasetVisibility.Private,
+    origin       = meta.origin,
+    sourceURL    = url,
+    created      = meta.createdOn ?: OffsetDateTime.now(),
+    dependencies = (meta.dependencies ?: emptyList()).map {
+      VDIDatasetDependencyImpl(
+        identifier  = it.resourceIdentifier,
+        version     = it.resourceVersion,
+        displayName = it.resourceDisplayName
+      )
+    },
+  )

service/rest-service/src/main/kotlin/org/veupathdb/service/vdi/generated/model/x-DatasetPostType.kt

@@ -2,6 +2,11 @@ package org.veupathdb.service.vdi.generated.model
 
 import org.veupathdb.service.vdi.util.ValidationErrors
 
+internal fun DatasetPostType.cleanup() {
+  name = name?.trim()
+  version = version?.trim()
+}
+
 internal fun DatasetPostType.validate(validationErrors: ValidationErrors) {
   if (name.isNullOrBlank())
     validationErrors.add("meta.type.name", "field is required")