diff --git a/components/dataset-reinstaller/src/main/kotlin/vdi/component/reinstaller/DatasetReinstaller.kt b/components/dataset-reinstaller/src/main/kotlin/vdi/component/reinstaller/DatasetReinstaller.kt index 2d92f9a1..592b3c5d 100644 --- a/components/dataset-reinstaller/src/main/kotlin/vdi/component/reinstaller/DatasetReinstaller.kt +++ b/components/dataset-reinstaller/src/main/kotlin/vdi/component/reinstaller/DatasetReinstaller.kt @@ -2,7 +2,9 @@ package vdi.component.reinstaller import org.slf4j.LoggerFactory import org.veupathdb.lib.s3.s34k.S3Api +import org.veupathdb.vdi.lib.common.compression.Zip import org.veupathdb.vdi.lib.common.field.ProjectID +import org.veupathdb.vdi.lib.common.fs.TempFiles import org.veupathdb.vdi.lib.db.app.AppDB import org.veupathdb.vdi.lib.db.app.AppDatabaseRegistry import org.veupathdb.vdi.lib.db.app.model.DatasetInstallMessage @@ -18,8 +20,14 @@ import org.veupathdb.vdi.lib.handler.client.response.uni.UninstallUnexpectedErro import org.veupathdb.vdi.lib.handler.mapping.PluginHandlers import org.veupathdb.vdi.lib.s3.datasets.DatasetDirectory import org.veupathdb.vdi.lib.s3.datasets.DatasetManager +import org.veupathdb.vdi.lib.s3.datasets.paths.S3Paths import vdi.component.metrics.Metrics +import java.io.InputStream +import java.nio.file.Path import java.util.concurrent.locks.ReentrantLock +import kotlin.io.path.deleteIfExists +import kotlin.io.path.inputStream +import kotlin.io.path.outputStream object DatasetReinstaller { @@ -143,10 +151,7 @@ object DatasetReinstaller { return } - val response = directory.getInstallReadyFile() - .loadContents()!! - .buffered() - .use { client.postInstallData(dataset.datasetID, projectID, it) } + val response = withInstallBundle(directory) { client.postInstallData(dataset.datasetID, projectID, it) } when (response.type) { InstallDataResponseType.Success @@ -271,4 +276,44 @@ object DatasetReinstaller { throw Exception(response.message) } + + private fun withInstallBundle(s3Dir: DatasetDirectory, fn: (upload: InputStream) -> T) = + TempFiles.withTempDirectory { tmpDir -> + val files = ArrayList(8) + + TempFiles.withTempFile { zipFile -> + zipFile.outputStream() + .buffered() + .use { out -> s3Dir.getInstallReadyFile().loadContents()!!.buffered().use { inp -> inp.transferTo(out) } } + + Zip.zipEntries(zipFile) + .forEach { (entry, stream) -> + tmpDir.resolve(entry.name) + .also(files::add) + .outputStream() + .buffered() + .use { stream.buffered().transferTo(it) } + } + } + + tmpDir.resolve(S3Paths.MetadataFileName) + .also(files::add) + .outputStream() + .buffered() + .use { out -> s3Dir.getMetaFile().loadContents()!!.use { input -> input.transferTo(out) } } + + tmpDir.resolve(S3Paths.ManifestFileName) + .also(files::add) + .outputStream() + .buffered() + .use { out -> s3Dir.getManifestFile().loadContents()!!.use { input -> input.transferTo(out) } } + + val zip = tmpDir.resolve("install-bundle.zip") + .also { Zip.compress(it, files, Zip.Level(0u)) } + + files.forEach { it.deleteIfExists() } + files.clear() + + zip.inputStream().buffered().use(fn) + } } \ No newline at end of file diff --git a/makefile b/makefile index 9ebe3cc6..5c0dc34a 100644 --- a/makefile +++ b/makefile @@ -67,7 +67,15 @@ log-plugin-genelist: log-plugin-rnaseq: @docker logs -f vdi-service-plugin-rnaseq-1 -.PHONY: log-plugin-rnaseq +.PHONY: log-plugin-isasimple +log-plugin-isasimple: + @docker logs -f vdi-service-plugin-isasimple-1 + +.PHONY: log-plugin-biom +log-plugin-biom: + @docker logs -f vdi-service-plugin-biom-1 + +.PHONY: log-plugin-bigwig log-plugin-bigwig: @docker logs -f vdi-service-plugin-bigwig-1 diff --git a/modules/import-event-handler/src/main/kotlin/vdi/module/handler/imports/triggers/ImportTriggerHandlerImpl.kt b/modules/import-event-handler/src/main/kotlin/vdi/module/handler/imports/triggers/ImportTriggerHandlerImpl.kt index 613f4a8d..b5979223 100644 --- a/modules/import-event-handler/src/main/kotlin/vdi/module/handler/imports/triggers/ImportTriggerHandlerImpl.kt +++ b/modules/import-event-handler/src/main/kotlin/vdi/module/handler/imports/triggers/ImportTriggerHandlerImpl.kt @@ -183,7 +183,7 @@ internal class ImportTriggerHandlerImpl(private val config: ImportTriggerHandler } catch (e: Throwable) { log.debug("import request to handler server failed with exception:", e) cacheDB.withTransaction { tran -> - tran.updateImportControl(datasetID, DatasetImportStatus.Invalid) + tran.updateImportControl(datasetID, DatasetImportStatus.Failed) tran.tryInsertImportMessages(datasetID, "Process error: ${e.message}") } throw e