Merge pull request #13 from VEuPathDB/object-stream

Add object stream APIs that will support lazy pagination

build.gradle.kts

@@ -8,7 +8,7 @@ plugins {
 }
 
 group = "org.veupathdb.lib.s3"
-version = "0.8.0"
+version = "0.9.0"
 
 java {
   sourceCompatibility = JavaVersion.VERSION_1_8

src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectContainer.kt

@@ -527,6 +527,73 @@ interface ObjectContainer {
 
   // endregion Stat
 
+  // region Stream All
+
+  /**
+   * Produces a stream of all the objects in this container.
+   *
+   * @return An [ObjectStream] instance containing `0` or more entries for each
+   * key/path found in this container.
+   *
+   * @throws BucketNotFoundError If this bucket or the bucket in which this
+   * object container resides no longer exists.
+   *
+   * @throws S34KError If an implementation specific exception is thrown.
+   * The implementation specific exception will be set to the thrown exception's
+   * 'cause' value.
+   */
+  @Throws(BucketNotFoundError::class, S34KError::class)
+  fun streamAll(): ObjectStream
+
+  /**
+   * Produces a stream of all the objects in this container.
+   *
+   * @param action Action used to configure the backing S3 operation.
+   *
+   * @return An [ObjectStream] instance containing `0` or more entries for each
+   * key/path found in this container.
+   *
+   * @throws BucketNotFoundError If this bucket or the bucket in which this
+   * object container resides no longer exists.
+   *
+   * @throws S34KError If an implementation specific exception is thrown.
+   * The implementation specific exception will be set to the thrown exception's
+   * 'cause' value.
+   */
+  @Throws(BucketNotFoundError::class, S34KError::class)
+  fun streamAll(action: ObjectStreamAllParams.() -> Unit): ObjectStream
+
+  /**
+   * Produces a stream of all the objects in this container.
+   *
+   * @param params Parameters for the backing S3 operation.
+   *
+   * @return An [ObjectList] instance containing `0` or more entries for each
+   * key/path found in this container.
+   *
+   * @throws BucketNotFoundError If this bucket or the bucket in which this
+   * object container resides no longer exists.
+   *
+   * @throws S34KError If an implementation specific exception is thrown.
+   * The implementation specific exception will be set to the thrown exception's
+   * 'cause' value.
+   */
+  @Throws(BucketNotFoundError::class, S34KError::class)
+  fun streamAll(params: ObjectStreamAllParams): ObjectStream
+
+  // endregion Stream All
+
+  // region Stream
+
+  fun stream(prefix: String? = null): ObjectStream
+
+  fun stream(action: ObjectStreamParams.() -> Unit): ObjectStream
+
+  fun stream(params: ObjectStreamParams): ObjectStream
+
+  // endregion Stream
+
+
   // region List All
 
   /**

src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectStream.kt

@@ -0,0 +1,12 @@
+package org.veupathdb.lib.s3.s34k.objects
+
+import java.util.stream.Stream
+
+// TODO: Document me
+interface ObjectStream {
+
+  // TODO: Document me
+  fun stream(): Stream<S3Object>
+
+  fun toObjectList(): ObjectList
+}

src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamAllParams.kt

@@ -0,0 +1,15 @@
+package org.veupathdb.lib.s3.s34k.params.`object`
+
+import org.veupathdb.lib.s3.s34k.objects.ObjectStream
+import org.veupathdb.lib.s3.s34k.params.RegionRequestParams
+
+interface ObjectStreamAllParams : RegionRequestParams {
+
+  /**
+   * Callback that will be executed on successful completion of the S3
+   * operation.
+   *
+   * The callback will be passed an [ObjectStream] value.
+   */
+  var callback: ((objects: ObjectStream) -> Unit)?
+}

src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt

@@ -0,0 +1,24 @@
+package org.veupathdb.lib.s3.s34k.params.`object`
+
+import org.veupathdb.lib.s3.s34k.objects.ObjectContainer
+import org.veupathdb.lib.s3.s34k.objects.ObjectStream
+import org.veupathdb.lib.s3.s34k.params.RegionRequestParams
+
+interface ObjectStreamParams : RegionRequestParams {
+
+  /**
+   * Filtering prefix for the paths to return.
+   *
+   * If set to `null` or a blank string, this operation is the same as
+   * [ObjectContainer.streamAll].
+   */
+  var prefix: String?
+
+  /**
+   * Optional callback that will be executed on successful completion of the S3
+   * operation.
+   *
+   * This callback will be passed a stream of objects retrieved from the store.
+   */
+  var callback: ((ObjectStream) -> Unit)?
+}