diff --git a/components/s3/src/main/kotlin/org/veupathdb/vdi/lib/s3/datasets/DatasetManager.kt b/components/s3/src/main/kotlin/org/veupathdb/vdi/lib/s3/datasets/DatasetManager.kt index 9846db0b..adae56dc 100644 --- a/components/s3/src/main/kotlin/org/veupathdb/vdi/lib/s3/datasets/DatasetManager.kt +++ b/components/s3/src/main/kotlin/org/veupathdb/vdi/lib/s3/datasets/DatasetManager.kt @@ -162,7 +162,13 @@ class DatasetManager(private val s3Bucket: S3Bucket) { // Stream is exhausted, construct a dataset out of remaining objects. val idsFromObject = datasetIdFromS3Object(stagedObjects.first()) val pathFactory = S3DatasetPathFactory(idsFromObject.first, idsFromObject.second) - this.currentDataset = EagerlyLoadedDatasetDirectory(stagedObjects, idsFromObject.first, idsFromObject.second, pathFactory) + try { + this.currentDataset = EagerlyLoadedDatasetDirectory(stagedObjects, idsFromObject.first, idsFromObject.second, pathFactory) + } catch (e: MalformedDatasetException) { + Metrics.malformedDatasetFound.inc() + log.warn("Found a malformed dataset with ID $idsFromObject.") + return null + } stagedObjects = emptyList() return currentDataset } diff --git a/modules/reconciler/src/main/kotlin/org/veupathdb/vdi/lib/reconciler/ReconcilerInstance.kt b/modules/reconciler/src/main/kotlin/org/veupathdb/vdi/lib/reconciler/ReconcilerInstance.kt index d2fcb5d4..8127911c 100644 --- a/modules/reconciler/src/main/kotlin/org/veupathdb/vdi/lib/reconciler/ReconcilerInstance.kt +++ b/modules/reconciler/src/main/kotlin/org/veupathdb/vdi/lib/reconciler/ReconcilerInstance.kt @@ -72,10 +72,12 @@ class ReconcilerInstance( // Delete datasets until and advance target iterator until streams are aligned. while (nextTargetDataset != null && comparableS3Id.compareTo(comparableTargetId, true) > 0) { - logger().info("Attempting to delete dataset with owner ${sourceDatasetDir.ownerID} and ID ${sourceDatasetDir.datasetID} " + - "because ${nextTargetDataset!!.syncControlRecord.datasetID} is lexigraphically greater than our ID.") + + logger().info("Attempting to delete dataset with owner $comparableTargetId " + + "because $comparableS3Id is lexigraphically greater than $comparableTargetId.") tryDeleteDataset(targetDB, nextTargetDataset!!.type, nextTargetDataset!!.syncControlRecord.datasetID) nextTargetDataset = if (targetIterator.hasNext()) targetIterator.next() else null + comparableTargetId = nextTargetDataset!!.getComparableID() } }