Check if mongodump and mongorestore exist (#23)

* Check if mongodump and mongorestore exist

This checks if we have the binaries existing, so we can fail
early on, instead of silently synchronizing collections without
a restore first.

Closes #9

* No need to run ktlint and detekt on CI

We run them as part of another workflow, so no need to repeat them.

* Add tools validator to CLI defaults

.github/workflows/ci.yml

@@ -39,4 +39,4 @@ jobs:
         uses: gradle/gradle-build-action@v2
 
       - name: Execute Gradle tests
-        run: ./gradlew ${{ matrix.tests }}
+        run: ./gradlew ${{ matrix.tests }} -x ktlintCheck -x detekt

mongo-migration-stream-cli/src/main/kotlin/pl/allegro/tech/mongomigrationstream/properties/ApplicationPropertiesReader.kt

@@ -10,6 +10,7 @@ import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.DbAv
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.DbHashSynchronizationDetectorType
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.DestinationMissingCollectionType
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.LoggingSynchronizationHandler
+import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.MongoToolsValidatorType
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.QueueSizeSynchronizationDetectorType
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.SourceCollectionAvailabilityType
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.SynchronizationDetectorType
@@ -130,7 +131,9 @@ internal object ApplicationPropertiesReader {
             supportedTypes[type] ?: throw InvalidQueueFactoryType(type)
 
         class InvalidQueueFactoryType(type: String) :
-            IllegalArgumentException("Invalid queue factory type: [$type]. Possible types are: [${supportedTypes.keys}]")
+            IllegalArgumentException(
+                "Invalid queue factory type: [$type]. Possible types are: [${supportedTypes.keys}]"
+            )
     }
 
     internal object SynchronizationHandlerTypeMapper {
@@ -169,7 +172,8 @@ internal object ApplicationPropertiesReader {
         private val supportedValidators = mapOf(
             "DestinationCollectionMissing" to DestinationMissingCollectionType,
             "DbAvailability" to DbAvailabilityValidatorType,
-            "SourceCollectionAvailable" to SourceCollectionAvailabilityType
+            "SourceCollectionAvailable" to SourceCollectionAvailabilityType,
+            "MontoToolsAvailable" to MongoToolsValidatorType
         )
 
         fun from(type: String): ValidatorType =

mongo-migration-stream-core/src/integrationTest/kotlin/pl/allegro/tech/mongomigrationstream/configuration/MongoMigrationStreamTestConfig.kt

@@ -51,7 +51,7 @@ internal object MongoMigrationStreamTestConfig {
         databaseValidators = setOf(
             DbAvailabilityValidatorType,
             SourceCollectionAvailabilityType,
-            DestinationMissingCollectionType
+            DestinationMissingCollectionType,
         )
     )
 

mongo-migration-stream-core/src/integrationTest/kotlin/pl/allegro/tech/mongomigrationstream/test/preconditions/StartMigrationValidationTest.kt

@@ -1,13 +1,21 @@
 package pl.allegro.tech.mongomigrationstream.test.preconditions
 
+import com.mongodb.ReadPreference
 import io.kotest.core.spec.style.ShouldSpec
 import io.kotest.matchers.string.shouldContain
 import org.junit.jupiter.api.assertThrows
 import pl.allegro.tech.mongomigrationstream.configuration.CollectionsProperties
+import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties
 import pl.allegro.tech.mongomigrationstream.configuration.MongoMigrationStreamTestConfig.createMongoMigrationStream
+import pl.allegro.tech.mongomigrationstream.configuration.MongoMigrationStreamTestProperties
 import pl.allegro.tech.mongomigrationstream.configuration.MongoProperties
+import pl.allegro.tech.mongomigrationstream.configuration.PerformerProperties
+import pl.allegro.tech.mongomigrationstream.core.synchronization.ConstantValueBatchSizeProvider
 import pl.allegro.tech.mongomigrationstream.infrastructure.controller.MongoMigrationStreamStartException
+import pl.allegro.tech.mongomigrationstream.infrastructure.handler.LoggingDetectionResultHandler
+import pl.allegro.tech.mongomigrationstream.test.preconditions.StartMigrationValidationTest.TestData.generalPropertiesWithDBToolsValidator
 import pl.allegro.tech.mongomigrationstream.test.preconditions.StartMigrationValidationTest.TestData.invalidMongoProperties
+import pl.allegro.tech.mongomigrationstream.test.preconditions.StartMigrationValidationTest.TestData.invalidPerformerProperties
 
 internal class StartMigrationValidationTest : ShouldSpec({
     should("not perform migration when source db is not available") {
@@ -53,6 +61,23 @@ internal class StartMigrationValidationTest : ShouldSpec({
         // then: should fail
         result.cause!!.message.shouldContain("Non-existing collections: [[nonExistingCollection]]")
     }
+
+    should("not perform migration when database tools aren't available") {
+        // given
+        val performerProperties = invalidPerformerProperties()
+        val generalProperties = generalPropertiesWithDBToolsValidator()
+
+        // when
+        val result = assertThrows<MongoMigrationStreamStartException> {
+            createMongoMigrationStream(
+                generalProperties = generalProperties,
+                performerProperties = performerProperties
+            ).start()
+        }
+
+        // then: should fail
+        result.cause!!.message.shouldContain("MongoDB tools installation isn't working or doesn't exist")
+    }
 }) {
     private object TestData {
         fun invalidMongoProperties(): MongoProperties {
@@ -64,5 +89,28 @@ internal class StartMigrationValidationTest : ShouldSpec({
                 authenticationProperties = null
             )
         }
+
+        fun generalPropertiesWithDBToolsValidator(): GeneralProperties =
+            GeneralProperties(
+                shouldPerformTransfer = true,
+                shouldPerformSynchronization = true,
+                synchronizationHandlers = setOf(LoggingDetectionResultHandler),
+                synchronizationDetectors = setOf(
+                    GeneralProperties.DbHashSynchronizationDetectorType,
+                    GeneralProperties.QueueSizeSynchronizationDetectorType,
+                    GeneralProperties.CollectionCountSynchronizationDetectorType,
+                ),
+                databaseValidators = setOf(
+                    GeneralProperties.MongoToolsValidatorType
+                )
+            )
+
+        fun invalidPerformerProperties() = PerformerProperties(
+            rootPath = MongoMigrationStreamTestProperties.ROOT_PATH,
+            mongoToolsPath = MongoMigrationStreamTestProperties.MONGO_TOOLS_PATH,
+            queueFactory = PerformerProperties.BiqQueueFactoryType,
+            dumpReadPreference = ReadPreference.primary(),
+            batchSizeProvider = ConstantValueBatchSizeProvider(1000)
+        )
     }
 }

mongo-migration-stream-core/src/main/kotlin/pl/allegro/tech/mongomigrationstream/configuration/GeneralProperties.kt

@@ -21,4 +21,5 @@ data class GeneralProperties(
     object DbAvailabilityValidatorType : ValidatorType()
     object DestinationMissingCollectionType : ValidatorType()
     object SourceCollectionAvailabilityType : ValidatorType()
+    object MongoToolsValidatorType : ValidatorType()
 }

mongo-migration-stream-core/src/main/kotlin/pl/allegro/tech/mongomigrationstream/core/validation/ExternalDependenciesValidator.kt

@@ -3,6 +3,7 @@ package pl.allegro.tech.mongomigrationstream.core.validation
 import pl.allegro.tech.mongomigrationstream.configuration.ApplicationProperties
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.DbAvailabilityValidatorType
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.DestinationMissingCollectionType
+import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.MongoToolsValidatorType
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.SourceCollectionAvailabilityType
 import pl.allegro.tech.mongomigrationstream.configuration.GeneralProperties.ValidatorType
 import pl.allegro.tech.mongomigrationstream.infrastructure.mongo.MongoDbClients
@@ -42,6 +43,10 @@ internal object ExternalDependenciesValidator {
                 properties.collectionsProperties.destinationCollections
             )
         )
+
+        MongoToolsValidatorType -> listOf(
+            MongoToolsValidator(properties)
+        )
     }
 
     private fun performValidation(validators: List<Validator>) {
@@ -53,5 +58,5 @@ internal object ExternalDependenciesValidator {
     }
 }
 
-internal class ExternalDependencyValidationException(val causes: List<ValidationFailure>) :
+internal class ExternalDependencyValidationException(causes: List<ValidationFailure>) :
     RuntimeException(causes.joinToString { it.message })

mongo-migration-stream-core/src/main/kotlin/pl/allegro/tech/mongomigrationstream/core/validation/MongoToolsValidator.kt

@@ -0,0 +1,36 @@
+package pl.allegro.tech.mongomigrationstream.core.validation
+
+import io.github.oshai.kotlinlogging.KotlinLogging
+import io.github.oshai.kotlinlogging.withLoggingContext
+import pl.allegro.tech.mongomigrationstream.configuration.ApplicationProperties
+import java.nio.file.Paths
+import kotlin.io.path.exists
+import kotlin.io.path.isExecutable
+
+private val logger = KotlinLogging.logger { }
+
+internal class MongoToolsValidator(private val applicationProperties: ApplicationProperties) : Validator {
+
+    private fun check(binary: String): Boolean {
+        withLoggingContext("binary" to binary) {
+            val binaryPath = Paths.get(applicationProperties.performerProperties.mongoToolsPath).resolve(binary)
+            return if (binaryPath.exists() && binaryPath.isExecutable()) {
+                logger.info { "$binary at $binaryPath exists and is executable" }
+                true
+            } else {
+                logger.error { "$binary at $binaryPath doesn't exist or isn't executable" }
+                false
+            }
+        }
+    }
+
+    override fun validate(): ValidationResult {
+        val mongoDump = check("mongodump")
+        val mongoRestore = check("mongorestore")
+        return if (mongoDump && mongoRestore) {
+            ValidationSuccess
+        } else {
+            ValidationFailure("MongoDB tools installation isn't working or doesn't exist")
+        }
+    }
+}