misc fixes from testing

VEuPathDB · Mar 1, 2024 · 3ca4eb3 · 3ca4eb3
1 parent b012ef7
commit 3ca4eb3
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 6 deletions.
diff --git a/...nents/dataset-reinstaller/src/main/kotlin/vdi/component/reinstaller/DatasetReinstaller.kt b/...nents/dataset-reinstaller/src/main/kotlin/vdi/component/reinstaller/DatasetReinstaller.kt
@@ -2,7 +2,9 @@ package vdi.component.reinstaller
 
 import org.slf4j.LoggerFactory
 import org.veupathdb.lib.s3.s34k.S3Api
+import org.veupathdb.vdi.lib.common.compression.Zip
 import org.veupathdb.vdi.lib.common.field.ProjectID
+import org.veupathdb.vdi.lib.common.fs.TempFiles
 import org.veupathdb.vdi.lib.db.app.AppDB
 import org.veupathdb.vdi.lib.db.app.AppDatabaseRegistry
 import org.veupathdb.vdi.lib.db.app.model.DatasetInstallMessage
@@ -18,8 +20,14 @@ import org.veupathdb.vdi.lib.handler.client.response.uni.UninstallUnexpectedErro
 import org.veupathdb.vdi.lib.handler.mapping.PluginHandlers
 import org.veupathdb.vdi.lib.s3.datasets.DatasetDirectory
 import org.veupathdb.vdi.lib.s3.datasets.DatasetManager
+import org.veupathdb.vdi.lib.s3.datasets.paths.S3Paths
 import vdi.component.metrics.Metrics
+import java.io.InputStream
+import java.nio.file.Path
 import java.util.concurrent.locks.ReentrantLock
+import kotlin.io.path.deleteIfExists
+import kotlin.io.path.inputStream
+import kotlin.io.path.outputStream
 
 object DatasetReinstaller {
 
@@ -143,10 +151,7 @@ object DatasetReinstaller {
       return
     }
 
-    val response = directory.getInstallReadyFile()
-      .loadContents()!!
-      .buffered()
-      .use { client.postInstallData(dataset.datasetID, projectID, it) }
+    val response = withInstallBundle(directory) { client.postInstallData(dataset.datasetID, projectID, it) }
 
     when (response.type) {
       InstallDataResponseType.Success
@@ -271,4 +276,44 @@ object DatasetReinstaller {
 
     throw Exception(response.message)
   }
+
+  private fun <T> withInstallBundle(s3Dir: DatasetDirectory, fn: (upload: InputStream) -> T) =
+    TempFiles.withTempDirectory { tmpDir ->
+      val files = ArrayList<Path>(8)
+
+      TempFiles.withTempFile { zipFile ->
+        zipFile.outputStream()
+          .buffered()
+          .use { out -> s3Dir.getInstallReadyFile().loadContents()!!.buffered().use { inp -> inp.transferTo(out) } }
+
+        Zip.zipEntries(zipFile)
+          .forEach { (entry, stream) ->
+            tmpDir.resolve(entry.name)
+              .also(files::add)
+              .outputStream()
+              .buffered()
+              .use { stream.buffered().transferTo(it) }
+          }
+      }
+
+      tmpDir.resolve(S3Paths.MetadataFileName)
+        .also(files::add)
+        .outputStream()
+        .buffered()
+        .use { out -> s3Dir.getMetaFile().loadContents()!!.use { input -> input.transferTo(out) } }
+
+      tmpDir.resolve(S3Paths.ManifestFileName)
+        .also(files::add)
+        .outputStream()
+        .buffered()
+        .use { out -> s3Dir.getManifestFile().loadContents()!!.use { input -> input.transferTo(out) } }
+
+      val zip = tmpDir.resolve("install-bundle.zip")
+        .also { Zip.compress(it, files, Zip.Level(0u)) }
+
+      files.forEach { it.deleteIfExists() }
+      files.clear()
+
+      zip.inputStream().buffered().use(fn)
+    }
 }
diff --git a/makefile b/makefile
@@ -67,7 +67,15 @@ log-plugin-genelist:
 log-plugin-rnaseq:
 	@docker logs -f vdi-service-plugin-rnaseq-1
 
-.PHONY: log-plugin-rnaseq
+.PHONY: log-plugin-isasimple
+log-plugin-isasimple:
+	@docker logs -f vdi-service-plugin-isasimple-1
+
+.PHONY: log-plugin-biom
+log-plugin-biom:
+	@docker logs -f vdi-service-plugin-biom-1
+
+.PHONY: log-plugin-bigwig
 log-plugin-bigwig:
 	@docker logs -f vdi-service-plugin-bigwig-1
 

diff --git a/...t-handler/src/main/kotlin/vdi/module/handler/imports/triggers/ImportTriggerHandlerImpl.kt b/...t-handler/src/main/kotlin/vdi/module/handler/imports/triggers/ImportTriggerHandlerImpl.kt
@@ -183,7 +183,7 @@ internal class ImportTriggerHandlerImpl(private val config: ImportTriggerHandler
     } catch (e: Throwable) {
       log.debug("import request to handler server failed with exception:", e)
       cacheDB.withTransaction { tran ->
-        tran.updateImportControl(datasetID, DatasetImportStatus.Invalid)
+        tran.updateImportControl(datasetID, DatasetImportStatus.Failed)
         tran.tryInsertImportMessages(datasetID, "Process error: ${e.message}")
       }
       throw e