Merge pull request #60 from VEuPathDB/minio-bandaid

MinIO object deletion bandaid

lib/build.gradle.kts

@@ -6,7 +6,7 @@ plugins {
 }
 
 group = "org.veupathdb.lib"
-version = "1.8.1"
+version = "1.8.2"
 
 
 dependencies {

lib/src/main/kotlin/org/veupathdb/lib/compute/platform/intern/s3/S3.kt

@@ -10,11 +10,13 @@ import org.veupathdb.lib.s3.s34k.S3Api
 import org.veupathdb.lib.s3.s34k.S3Client
 import org.veupathdb.lib.s3.s34k.S3Config
 import org.veupathdb.lib.s3.s34k.fields.BucketName
+import org.veupathdb.lib.s3.s34k.objects.S3Object
 import org.veupathdb.lib.s3.workspaces.java.S3Workspace
 import org.veupathdb.lib.s3.workspaces.java.S3WorkspaceFactory
 import java.io.File
 import java.io.InputStream
 import java.nio.file.Path
+import kotlin.time.Duration.Companion.milliseconds
 
 /**
  * S3 Manager
@@ -150,7 +152,35 @@ internal object S3 {
     s3.buckets[BucketName(config.bucket)]!!
       .objects
       .list(jobID.toS3Prefix())
-      .forEach { it.delete() }
+      .forEach(::minioObjectDeleteHack)
+  }
+
+  // TODO: remove this if/when S34K is updated to allow suspend functions
+  private fun minioObjectDeleteHack(obj: S3Object) {
+    obj.delete()
+
+    val sleepMillis = 100L
+    val maxSleeps = 15  // 1.5 seconds
+
+    var sleepCounter = 0
+
+    // while MinIO is still reporting that the object exists, sleep on it.
+    while (obj.exists()) {
+      if (sleepCounter > maxSleeps) {
+        // DON'T THROW HERE, IT MAY LEAVE THE WORKSPACE IN A WONKY STATE, IF THE
+        // CALLER ATTEMPTS TO RECREATE THE WORKSPACE THEY WILL GET AN EXCEPTION
+        // AT THAT POINT
+        Log.error(
+          "waited {} seconds for MinIO to acknowledge the deletion of object {} but it never did",
+          (sleepMillis*maxSleeps).milliseconds,
+          obj.path
+        )
+        break
+      }
+
+      Thread.sleep(sleepMillis)
+      sleepCounter++
+    }
   }
 
   /**
@@ -350,4 +380,4 @@ internal object S3 {
 
   private fun String.getJobID() =
     substring(lastIndexOf('/', length - 2) + 1, length - 1)
-}
+}

readme.adoc

@@ -29,7 +29,7 @@ image::docs/assets/overview.png[]
 [source, kotlin]
 ----
 dependencies {
-  implementation("org.veupathdb.lib:compute-platform:1.8.1")
+  implementation("org.veupathdb.lib:compute-platform:1.8.2")
 }
 ----