From 3de343371f82e4f7b31d34d108e50941221cb0eb Mon Sep 17 00:00:00 2001 From: Dan Galdi Date: Fri, 14 Apr 2023 10:17:55 -0400 Subject: [PATCH 1/3] Add object stream APIs that will support lazy pagination --- build.gradle.kts | 2 +- .../lib/s3/s34k/objects/ObjectContainer.kt | 67 +++++++++++++++++++ .../lib/s3/s34k/objects/ObjectStream.kt | 12 ++++ .../params/object/ObjectStreamAllParams.kt | 15 +++++ .../s34k/params/object/ObjectStreamParams.kt | 24 +++++++ 5 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectStream.kt create mode 100644 src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamAllParams.kt create mode 100644 src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt diff --git a/build.gradle.kts b/build.gradle.kts index ef093f5..4e1deef 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { } group = "org.veupathdb.lib.s3" -version = "0.8.0" +version = "0.8.1" java { sourceCompatibility = JavaVersion.VERSION_1_8 diff --git a/src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectContainer.kt b/src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectContainer.kt index fa60f39..42982e7 100644 --- a/src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectContainer.kt +++ b/src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectContainer.kt @@ -527,6 +527,73 @@ interface ObjectContainer { // endregion Stat + // region Stream All + + /** + * Produces a stream of all the objects in this container. + * + * @return An [ObjectStream] instance containing `0` or more entries for each + * key/path found in this container. + * + * @throws BucketNotFoundError If this bucket or the bucket in which this + * object container resides no longer exists. + * + * @throws S34KError If an implementation specific exception is thrown. + * The implementation specific exception will be set to the thrown exception's + * 'cause' value. + */ + @Throws(BucketNotFoundError::class, S34KError::class) + fun streamAll(): ObjectStream + + /** + * Produces a stream of all the objects in this container. + * + * @param action Action used to configure the backing S3 operation. + * + * @return An [ObjectStream] instance containing `0` or more entries for each + * key/path found in this container. + * + * @throws BucketNotFoundError If this bucket or the bucket in which this + * object container resides no longer exists. + * + * @throws S34KError If an implementation specific exception is thrown. + * The implementation specific exception will be set to the thrown exception's + * 'cause' value. + */ + @Throws(BucketNotFoundError::class, S34KError::class) + fun streamAll(action: ObjectStreamAllParams.() -> Unit): ObjectStream + + /** + * Produces a stream of all the objects in this container. + * + * @param params Parameters for the backing S3 operation. + * + * @return An [ObjectList] instance containing `0` or more entries for each + * key/path found in this container. + * + * @throws BucketNotFoundError If this bucket or the bucket in which this + * object container resides no longer exists. + * + * @throws S34KError If an implementation specific exception is thrown. + * The implementation specific exception will be set to the thrown exception's + * 'cause' value. + */ + @Throws(BucketNotFoundError::class, S34KError::class) + fun streamAll(params: ObjectStreamAllParams): ObjectStream + + // endregion Stream All + + // region Stream + + fun stream(prefix: String? = null): ObjectStream + + fun stream(action: ObjectStreamParams.() -> Unit): ObjectStream + + fun stream(params: ObjectStreamParams): ObjectStream + + // endregion Stream + + // region List All /** diff --git a/src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectStream.kt b/src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectStream.kt new file mode 100644 index 0000000..a8f0c63 --- /dev/null +++ b/src/main/kotlin/org/veupathdb/lib/s3/s34k/objects/ObjectStream.kt @@ -0,0 +1,12 @@ +package org.veupathdb.lib.s3.s34k.objects + +import java.util.stream.Stream + +// TODO: Document me +interface ObjectStream { + + // TODO: Document me + fun stream(): Stream + + fun toObjectList(): ObjectList +} \ No newline at end of file diff --git a/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamAllParams.kt b/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamAllParams.kt new file mode 100644 index 0000000..45efb22 --- /dev/null +++ b/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamAllParams.kt @@ -0,0 +1,15 @@ +package org.veupathdb.lib.s3.s34k.params.`object` + +import org.veupathdb.lib.s3.s34k.objects.ObjectStream +import org.veupathdb.lib.s3.s34k.params.RegionRequestParams + +interface ObjectStreamAllParams : RegionRequestParams { + + /** + * Callback that will be executed on successful completion of the S3 + * operation. + * + * The callback will be passed an [ObjectList] value. + */ + var callback: ((objects: ObjectStream) -> Unit)? +} \ No newline at end of file diff --git a/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt b/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt new file mode 100644 index 0000000..3d9398c --- /dev/null +++ b/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt @@ -0,0 +1,24 @@ +package org.veupathdb.lib.s3.s34k.params.`object` + +import org.veupathdb.lib.s3.s34k.objects.ObjectContainer +import org.veupathdb.lib.s3.s34k.objects.ObjectStream +import org.veupathdb.lib.s3.s34k.params.RegionRequestParams + +interface ObjectStreamParams : RegionRequestParams { + + /** + * Filtering prefix for the paths to return. + * + * If set to `null` or a blank string, this operation is the same as + * [ObjectContainer.listAll]. + */ + var prefix: String? + + /** + * Optional callback that will be executed on successful completion of the S3 + * operation. + * + * This callback will be passed a stream of objects retrieved from the store. + */ + var callback: ((ObjectStream) -> Unit)? +} \ No newline at end of file From c2be6b2fe021330cc6bbc22a92ba4a7e1bbdd84f Mon Sep 17 00:00:00 2001 From: Dan Galdi Date: Fri, 14 Apr 2023 10:26:47 -0400 Subject: [PATCH 2/3] Upgrade minor version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4e1deef..ea82f04 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { } group = "org.veupathdb.lib.s3" -version = "0.8.1" +version = "0.9.0" java { sourceCompatibility = JavaVersion.VERSION_1_8 From bf59d43206c1177728641be46b20bdaeefbae8fe Mon Sep 17 00:00:00 2001 From: Dan Galdi Date: Fri, 14 Apr 2023 12:20:32 -0400 Subject: [PATCH 3/3] Update documentation --- .../lib/s3/s34k/params/object/ObjectStreamAllParams.kt | 2 +- .../veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamAllParams.kt b/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamAllParams.kt index 45efb22..d48e940 100644 --- a/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamAllParams.kt +++ b/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamAllParams.kt @@ -9,7 +9,7 @@ interface ObjectStreamAllParams : RegionRequestParams { * Callback that will be executed on successful completion of the S3 * operation. * - * The callback will be passed an [ObjectList] value. + * The callback will be passed an [ObjectStream] value. */ var callback: ((objects: ObjectStream) -> Unit)? } \ No newline at end of file diff --git a/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt b/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt index 3d9398c..ef9fa8d 100644 --- a/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt +++ b/src/main/kotlin/org/veupathdb/lib/s3/s34k/params/object/ObjectStreamParams.kt @@ -10,7 +10,7 @@ interface ObjectStreamParams : RegionRequestParams { * Filtering prefix for the paths to return. * * If set to `null` or a blank string, this operation is the same as - * [ObjectContainer.listAll]. + * [ObjectContainer.streamAll]. */ var prefix: String?