Configure fallback from controller-specific integration tests to regular integration tests (#2605)

IlyaMuravjov · web-flow · commit dd787d033be9 · 2023-09-19T12:35:00.000+03:00
* Extract params of `tryCreateFuzzingContext` to `data class`

* Configure `SpringIntegrationTestJavaFuzzingContext` to fallback to non-`mockMvc` tests

* Avoid `mockMvc` and non-`mockMvc` tests with same execution path in method under test
diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/util/SpringModelUtils.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/util/SpringModelUtils.kt
@@ -355,6 +355,39 @@ object SpringModelUtils {
         returnType = resultMatcherClassId
     )
 
+    private val supportedControllerParameterAnnotations = setOf(
+        pathVariableClassId,
+        requestParamClassId,
+        requestHeaderClassId,
+//        cookieValueClassId, // TODO uncomment when #2542 is fixed
+        requestAttributesClassId,
+        sessionAttributesClassId,
+        modelAttributesClassId,
+        requestBodyClassId,
+    )
+
+    /**
+     * If a controller method has a parameter of one of these types, then we don't fully support conversion
+     * of direct call of that controller method call to a request that can be done via `mockMvc` even if
+     * said parameter is annotated with one of [supportedControllerParameterAnnotations].
+     */
+    private val unsupportedControllerParameterTypes = setOf(
+        dateClassId, // see #2505
+        mapClassId, // e.g. `@RequestParam Map<String, Object>` is not yet properly handled
+    )
+
+    /**
+     * Returns `true` if for every parameter of [methodId] we have a mechanism of registering said
+     * parameter in `requestBuilder` when calling controller method via `mockMvc.perform(requestBuilder)`.
+     */
+    fun allControllerParametersAreSupported(methodId: MethodId): Boolean =
+        methodId.parameters.none { it in unsupportedControllerParameterTypes } &&
+                methodId.method.parameters.all { param ->
+                    param.annotations.any { annotation ->
+                        annotation.annotationClass.id in supportedControllerParameterAnnotations
+                    }
+                }
+
     fun createMockMvcModel(controller: UtModel?, idGenerator: () -> Int) =
         createBeanModel("mockMvc", idGenerator(), mockMvcClassId, modificationChainProvider = {
             // we need to keep controller modifications if there are any, so we add them to mockMvc
diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt
@@ -34,6 +34,7 @@ import org.utbot.framework.UtSettings.processUnknownStatesDuringConcreteExecutio
 import org.utbot.framework.UtSettings.useDebugVisualization
 import org.utbot.framework.context.ApplicationContext
 import org.utbot.framework.context.ConcreteExecutionContext
+import org.utbot.framework.context.ConcreteExecutionContext.FuzzingContextParams
 import org.utbot.framework.plugin.api.*
 import org.utbot.framework.plugin.api.Step
 import org.utbot.framework.plugin.api.util.*
@@ -119,7 +120,7 @@ class UtBotSymbolicEngine(
     userTaintConfigurationProvider: TaintConfigurationProvider? = null,
     private val solverTimeoutInMillis: Int = checkSolverTimeoutMillis,
 ) : UtContextInitializer() {
-    
+
     private val graph = methodUnderTest.sootMethod.jimpleBody().apply {
         logger.trace { "JIMPLE for $methodUnderTest:\n$this" }
     }.graph()
@@ -440,7 +441,16 @@ class UtBotSymbolicEngine(
         var testEmittedByFuzzer = 0
 
         val fuzzingContext = try {
-            concreteExecutionContext.tryCreateFuzzingContext(concreteExecutor, classUnderTest, defaultIdGenerator)
+            concreteExecutionContext.tryCreateFuzzingContext(
+                FuzzingContextParams(
+                    concreteExecutor = concreteExecutor,
+                    classUnderTest = classUnderTest,
+                    idGenerator = defaultIdGenerator,
+                    fuzzingStartTimeMillis = System.currentTimeMillis(),
+                    fuzzingEndTimeMillis = until,
+                    mockStrategy = mockStrategy,
+                )
+            )
         } catch (e: Exception) {
             emit(UtError(e.message ?: "Failed to create ValueProvider", e))
             return@flow
@@ -495,7 +505,7 @@ class UtBotSymbolicEngine(
             // in case an exception occurred from the concrete execution
             concreteExecutionResult ?: return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS)
 
-            fuzzingContext.handleFuzzedConcreteExecutionResult(concreteExecutionResult)
+            fuzzingContext.handleFuzzedConcreteExecutionResult(methodUnderTest, concreteExecutionResult)
 
             // in case of processed failure in the concrete execution
             concreteExecutionResult.processedFailure()?.let { failure ->
@@ -855,7 +865,7 @@ private fun UtConcreteExecutionResult.violatesUtMockAssumption(): Boolean {
 }
 
 private fun UtConcreteExecutionResult.processedFailure(): UtConcreteExecutionProcessedFailure?
- = result as? UtConcreteExecutionProcessedFailure
+    = result as? UtConcreteExecutionProcessedFailure
 
 private fun checkStaticMethodsMock(execution: UtSymbolicExecution) =
     execution.instrumentation.any { it is UtStaticMethodInstrumentation}
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/ConcreteExecutionContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/ConcreteExecutionContext.kt
@@ -1,5 +1,6 @@
 package org.utbot.framework.context
 
+import org.utbot.engine.MockStrategy
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
 import org.utbot.framework.plugin.api.ExecutableId
@@ -27,9 +28,14 @@ interface ConcreteExecutionContext {
         rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
     ): List<UtExecution>
 
-    fun tryCreateFuzzingContext(
-        concreteExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
-        classUnderTest: ClassId,
-        idGenerator: IdentityPreservingIdGenerator<Int>,
-    ): JavaFuzzingContext
+    fun tryCreateFuzzingContext(params: FuzzingContextParams): JavaFuzzingContext
+
+    data class FuzzingContextParams(
+        val concreteExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
+        val classUnderTest: ClassId,
+        val idGenerator: IdentityPreservingIdGenerator<Int>,
+        val fuzzingStartTimeMillis: Long,
+        val fuzzingEndTimeMillis: Long,
+        val mockStrategy: MockStrategy,
+    )
 }
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/JavaFuzzingContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/JavaFuzzingContext.kt
@@ -21,5 +21,8 @@ interface JavaFuzzingContext {
         executableToCall: ExecutableId,
     ): EnvironmentModels
 
-    fun handleFuzzedConcreteExecutionResult(concreteExecutionResult: UtConcreteExecutionResult)
+    fun handleFuzzedConcreteExecutionResult(
+        methodUnderTest: ExecutableId,
+        concreteExecutionResult: UtConcreteExecutionResult,
+    )
 }
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/MockingJavaFuzzingContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/MockingJavaFuzzingContext.kt
@@ -1,6 +1,7 @@
 package org.utbot.framework.context.custom
 
 import org.utbot.framework.context.JavaFuzzingContext
+import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.fuzzing.JavaValueProvider
 import org.utbot.fuzzing.providers.MapValueProvider
 import org.utbot.fuzzing.spring.unit.MockValueProvider
@@ -37,6 +38,11 @@ class MockingJavaFuzzingContext(
                     .with(NullValueProvider)
             )
 
-    override fun handleFuzzedConcreteExecutionResult(concreteExecutionResult: UtConcreteExecutionResult) =
+    override fun handleFuzzedConcreteExecutionResult(
+        methodUnderTest: ExecutableId,
+        concreteExecutionResult: UtConcreteExecutionResult
+    ) {
+        delegateContext.handleFuzzedConcreteExecutionResult(methodUnderTest, concreteExecutionResult)
         mockValueProvider.addMockingCandidates(concreteExecutionResult.detectedMockingCandidates)
+    }
 }
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleConcreteExecutionContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleConcreteExecutionContext.kt
@@ -1,12 +1,11 @@
 package org.utbot.framework.context.simple
 
 import org.utbot.framework.context.ConcreteExecutionContext
+import org.utbot.framework.context.ConcreteExecutionContext.FuzzingContextParams
 import org.utbot.framework.context.JavaFuzzingContext
-import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
 import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.UtExecution
-import org.utbot.fuzzer.IdentityPreservingIdGenerator
 import org.utbot.instrumentation.ConcreteExecutor
 import org.utbot.instrumentation.instrumentation.execution.SimpleUtExecutionInstrumentation
 import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
@@ -32,9 +31,6 @@ class SimpleConcreteExecutionContext(fullClassPath: String) : ConcreteExecutionC
         rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
     ): List<UtExecution> = executions
 
-    override fun tryCreateFuzzingContext(
-        concreteExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
-        classUnderTest: ClassId,
-        idGenerator: IdentityPreservingIdGenerator<Int>
-    ): JavaFuzzingContext = SimpleJavaFuzzingContext(classUnderTest, idGenerator)
+    override fun tryCreateFuzzingContext(params: FuzzingContextParams): JavaFuzzingContext =
+        SimpleJavaFuzzingContext(params.classUnderTest, params.idGenerator)
 }
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleJavaFuzzingContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleJavaFuzzingContext.kt
@@ -31,6 +31,8 @@ class SimpleJavaFuzzingContext(
         executableToCall = executableToCall
     )
 
-    override fun handleFuzzedConcreteExecutionResult(concreteExecutionResult: UtConcreteExecutionResult) =
-        Unit
+    override fun handleFuzzedConcreteExecutionResult(
+        methodUnderTest: ExecutableId,
+        concreteExecutionResult: UtConcreteExecutionResult
+    ) = Unit
 }
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/utils/ConcreteExecutionContextUtils.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/utils/ConcreteExecutionContextUtils.kt
@@ -1,23 +1,15 @@
 package org.utbot.framework.context.utils
 
 import org.utbot.framework.context.ConcreteExecutionContext
+import org.utbot.framework.context.ConcreteExecutionContext.FuzzingContextParams
 import org.utbot.framework.context.JavaFuzzingContext
-import org.utbot.framework.plugin.api.ClassId
-import org.utbot.fuzzer.IdentityPreservingIdGenerator
 import org.utbot.fuzzing.JavaValueProvider
-import org.utbot.instrumentation.ConcreteExecutor
-import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
-import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
 
 fun ConcreteExecutionContext.transformJavaFuzzingContext(
-    transformer: (JavaFuzzingContext) -> JavaFuzzingContext
+    transformer: FuzzingContextParams.(JavaFuzzingContext) -> JavaFuzzingContext
 ) = object : ConcreteExecutionContext by this {
-    override fun tryCreateFuzzingContext(
-        concreteExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
-        classUnderTest: ClassId,
-        idGenerator: IdentityPreservingIdGenerator<Int>
-    ): JavaFuzzingContext = transformer(
-        this@transformJavaFuzzingContext.tryCreateFuzzingContext(concreteExecutor, classUnderTest, idGenerator)
+    override fun tryCreateFuzzingContext(params: FuzzingContextParams): JavaFuzzingContext = params.transformer(
+        this@transformJavaFuzzingContext.tryCreateFuzzingContext(params)
     )
 }
 
diff --git a/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestConcreteExecutionContext.kt b/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestConcreteExecutionContext.kt
@@ -2,11 +2,14 @@ package org.utbot.framework.context.spring
 
 import mu.KotlinLogging
 import org.utbot.framework.context.ConcreteExecutionContext
+import org.utbot.framework.context.ConcreteExecutionContext.FuzzingContextParams
 import org.utbot.framework.context.JavaFuzzingContext
-import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
+import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.SpringSettings
-import org.utbot.fuzzer.IdentityPreservingIdGenerator
+import org.utbot.framework.plugin.api.UtExecution
+import org.utbot.framework.plugin.api.isSuccess
+import org.utbot.framework.plugin.api.util.SpringModelUtils
 import org.utbot.instrumentation.ConcreteExecutor
 import org.utbot.instrumentation.getRelevantSpringRepositories
 import org.utbot.instrumentation.instrumentation.execution.RemovingConstructFailsUtExecutionInstrumentation
@@ -49,25 +52,46 @@ class SpringIntegrationTestConcreteExecutionContext(
             }
         }
 
-    override fun tryCreateFuzzingContext(
-        concreteExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
-        classUnderTest: ClassId,
-        idGenerator: IdentityPreservingIdGenerator<Int>
-    ): JavaFuzzingContext {
-        if (springApplicationContext.getBeansAssignableTo(classUnderTest).isEmpty())
+    override fun tryCreateFuzzingContext(params: FuzzingContextParams): JavaFuzzingContext {
+        if (springApplicationContext.getBeansAssignableTo(params.classUnderTest).isEmpty())
             error(
-                "No beans of type ${classUnderTest.name} are found. " +
+                "No beans of type ${params.classUnderTest} are found. " +
                         "Try choosing different Spring configuration or adding beans to " +
                         springSettings.configuration.fullDisplayName
             )
 
-        val relevantRepositories = concreteExecutor.getRelevantSpringRepositories(classUnderTest)
-        logger.info { "Detected relevant repositories for class $classUnderTest: $relevantRepositories" }
+        val relevantRepositories = params.concreteExecutor.getRelevantSpringRepositories(params.classUnderTest)
+        logger.info { "Detected relevant repositories for class ${params.classUnderTest}: $relevantRepositories" }
 
         return SpringIntegrationTestJavaFuzzingContext(
-            delegateContext = delegateContext.tryCreateFuzzingContext(concreteExecutor, classUnderTest, idGenerator),
+            delegateContext = delegateContext.tryCreateFuzzingContext(params),
             relevantRepositories = relevantRepositories,
             springApplicationContext = springApplicationContext,
+            fuzzingStartTimeMillis = params.fuzzingStartTimeMillis,
+            fuzzingEndTimeMillis = params.fuzzingEndTimeMillis,
         )
     }
+
+    override fun transformExecutionsBeforeMinimization(
+        executions: List<UtExecution>,
+        methodUnderTest: ExecutableId
+    ): List<UtExecution> {
+        val (mockMvcExecutions, regularExecutions) =
+            delegateContext.transformExecutionsBeforeMinimization(executions, methodUnderTest)
+                .partition { it.executableToCall == SpringModelUtils.mockMvcPerformMethodId }
+
+        val classUnderTestName = methodUnderTest.classId.name
+        val methodUnderTestSignature = methodUnderTest.signature
+
+        fun UtExecution.getMethodUnderTestCoverage(): List<Long>? =
+            coverage?.coveredInstructions?.filter {
+                it.className == classUnderTestName && it.methodSignature == methodUnderTestSignature
+            }?.map { it.id }
+
+        fun UtExecution.getKey() = getMethodUnderTestCoverage()?.let { it to result.isSuccess }
+
+        val mockMvcExecutionKeys = mockMvcExecutions.mapNotNullTo(mutableSetOf()) { it.getKey() }
+
+        return mockMvcExecutions + regularExecutions.filter { it.getKey() !in mockMvcExecutionKeys }
+    }
 }
diff --git a/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestJavaFuzzingContext.kt b/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestJavaFuzzingContext.kt
@@ -9,8 +9,11 @@ import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.FieldId
 import org.utbot.framework.plugin.api.MethodId
 import org.utbot.framework.plugin.api.SpringRepositoryId
+import org.utbot.framework.plugin.api.UtExecutionSuccess
 import org.utbot.framework.plugin.api.UtModel
+import org.utbot.framework.plugin.api.UtSpringMockMvcResultActionsModel
 import org.utbot.framework.plugin.api.util.SpringModelUtils
+import org.utbot.framework.plugin.api.util.SpringModelUtils.allControllerParametersAreSupported
 import org.utbot.framework.plugin.api.util.allDeclaredFieldIds
 import org.utbot.framework.plugin.api.util.jField
 import org.utbot.framework.plugin.api.util.utContext
@@ -27,11 +30,14 @@ import org.utbot.fuzzing.spring.valid.EmailValueProvider
 import org.utbot.fuzzing.spring.valid.NotBlankStringValueProvider
 import org.utbot.fuzzing.spring.valid.NotEmptyStringValueProvider
 import org.utbot.fuzzing.spring.valid.ValidEntityValueProvider
+import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
 
 class SpringIntegrationTestJavaFuzzingContext(
-    val delegateContext: JavaFuzzingContext,
+    private val delegateContext: JavaFuzzingContext,
     relevantRepositories: Set<SpringRepositoryId>,
     springApplicationContext: SpringApplicationContext,
+    private val fuzzingStartTimeMillis: Long,
+    private val fuzzingEndTimeMillis: Long,
 ) : JavaFuzzingContext by delegateContext {
     companion object {
         private val logger = KotlinLogging.logger {}
@@ -93,11 +99,13 @@ class SpringIntegrationTestJavaFuzzingContext(
         })
     }
 
+    private val methodsSuccessfullyCalledViaMockMvc = mutableSetOf<ExecutableId>()
+
     override fun createStateBefore(
         thisInstance: UtModel?,
         parameters: List<UtModel>,
         statics: Map<FieldId, UtModel>,
-        executableToCall: ExecutableId,
+        executableToCall: ExecutableId
     ): EnvironmentModels {
         val delegateStateBefore = delegateContext.createStateBefore(thisInstance, parameters, statics, executableToCall)
         return when (executableToCall) {
@@ -107,7 +115,13 @@ class SpringIntegrationTestJavaFuzzingContext(
                     methodId = executableToCall,
                     arguments = parameters,
                     idGenerator = { idGenerator.createId() }
-                ) ?: return delegateStateBefore
+                )?.takeIf {
+                    val halfOfFuzzingTimePassed = System.currentTimeMillis() > (fuzzingStartTimeMillis + fuzzingEndTimeMillis) / 2
+                    val allParamsSupported = allControllerParametersAreSupported(executableToCall)
+                    val successfullyCalledViaMockMvc = executableToCall in methodsSuccessfullyCalledViaMockMvc
+                    !halfOfFuzzingTimePassed || allParamsSupported && successfullyCalledViaMockMvc
+                } ?: return delegateStateBefore
+
                 delegateStateBefore.copy(
                     thisInstance = SpringModelUtils.createMockMvcModel(controller = thisInstance) { idGenerator.createId() },
                     parameters = listOf(requestBuilderModel),
@@ -116,4 +130,15 @@ class SpringIntegrationTestJavaFuzzingContext(
             }
         }
     }
+
+    override fun handleFuzzedConcreteExecutionResult(
+        methodUnderTest: ExecutableId,
+        concreteExecutionResult: UtConcreteExecutionResult
+    ) {
+        delegateContext.handleFuzzedConcreteExecutionResult(methodUnderTest, concreteExecutionResult)
+        ((concreteExecutionResult.result as? UtExecutionSuccess)?.model as? UtSpringMockMvcResultActionsModel)?.let {
+            if (it.status < 400)
+                methodsSuccessfullyCalledViaMockMvc.add(methodUnderTest)
+        }
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -21,5 +21,8 @@ interface JavaFuzzingContext {`
`21`	`21`	`executableToCall: ExecutableId,`
`22`	`22`	`): EnvironmentModels`
`23`	`23`
`24`		`- fun handleFuzzedConcreteExecutionResult(concreteExecutionResult: UtConcreteExecutionResult)`
	`24`	`+ fun handleFuzzedConcreteExecutionResult(`
	`25`	`+ methodUnderTest: ExecutableId,`
	`26`	`+ concreteExecutionResult: UtConcreteExecutionResult,`
	`27`	`+ )`
`25`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -31,6 +31,8 @@ class SimpleJavaFuzzingContext(`
`31`	`31`	`executableToCall = executableToCall`
`32`	`32`	`)`
`33`	`33`
`34`		`- override fun handleFuzzedConcreteExecutionResult(concreteExecutionResult: UtConcreteExecutionResult) =`
`35`		`- Unit`
	`34`	`+ override fun handleFuzzedConcreteExecutionResult(`
	`35`	`+ methodUnderTest: ExecutableId,`
	`36`	`+ concreteExecutionResult: UtConcreteExecutionResult`
	`37`	`+ ) = Unit`
`36`	`38`	`}`