Make flow execution output work with JUNIT output format

mobile-dev-inc · Apr 19, 2024 · 5361d82 · 5361d82
1 parent baba15d
commit 5361d82
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 171 deletions.
diff --git a/maestro-cli/src/main/java/maestro/cli/command/RecordCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/RecordCommand.kt
@@ -23,6 +23,7 @@ import maestro.cli.App
 import maestro.cli.CliError
 import maestro.cli.DisableAnsiMixin
 import maestro.cli.api.ApiClient
+import maestro.cli.model.TestExecutionSummary
 import maestro.cli.report.TestDebugReporter
 import maestro.cli.runner.TestRunner
 import maestro.cli.runner.resultview.AnsiResultView
@@ -41,7 +42,7 @@ import java.util.concurrent.Callable
         "Render a beautiful video of your Flow - Great for demos and bug reports"
     ]
 )
-class RecordCommand : Callable<Int> {
+class RecordCommand : Callable<TestExecutionSummary.FlowResult> {
 
     @CommandLine.Mixin
     var disableANSIMixin: DisableAnsiMixin? = null
@@ -64,7 +65,7 @@ class RecordCommand : Callable<Int> {
     )
     private var debugOutput: String? = null
 
-    override fun call(): Int {
+    override fun call(): TestExecutionSummary.FlowResult {
         if (!flowFile.exists()) {
             throw CommandLine.ParameterException(
                 commandSpec.commandLine(),

diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt
@@ -22,12 +22,14 @@ package maestro.cli.command
 import maestro.cli.App
 import maestro.cli.CliError
 import maestro.cli.DisableAnsiMixin
+import maestro.cli.model.FlowStatus
 import maestro.cli.report.ReportFormat
 import maestro.cli.report.ReporterFactory
 import maestro.cli.report.TestDebugReporter
 import maestro.cli.runner.TestRunner
 import maestro.cli.runner.TestSuiteInteractor
 import maestro.cli.runner.resultview.AnsiResultView
+import maestro.cli.runner.resultview.NoopResultView
 import maestro.cli.runner.resultview.PlainTextResultView
 import maestro.cli.session.MaestroSessionManager
 import maestro.cli.util.PrintUtils
@@ -72,6 +74,12 @@ class TestCommand : Callable<Int> {
     )
     private var format: ReportFormat = ReportFormat.NOOP
 
+    @Option(
+        names = ["--hide-execution"],
+        description = ["Hides the execution flow of the tests."],
+    )
+    private var hideExecution: Boolean = false
+
     @Option(
         names = ["--test-suite-name"],
         description = ["Test suite name"],
@@ -136,8 +144,11 @@ class TestCommand : Callable<Int> {
         return MaestroSessionManager.newSession(parent?.host, parent?.port, deviceId) { session ->
             val maestro = session.maestro
             val device = session.device
+            val resultView = if (!hideExecution) {
+                if (DisableAnsiMixin.ansiEnabled) AnsiResultView() else PlainTextResultView()
+            } else NoopResultView
 
-            if (flowFile.isDirectory || format != ReportFormat.NOOP) {
+            if (flowFile.isDirectory) {
                 if (continuous) {
                     throw CommandLine.ParameterException(
                         commandSpec.commandLine(),
@@ -158,6 +169,7 @@ class TestCommand : Callable<Int> {
                                 .sink()
                                 .buffer()
                         },
+                    view = resultView,
                     debugOutputPath = debugOutputPath
                 )
 
@@ -173,13 +185,12 @@ class TestCommand : Callable<Int> {
                     TestDebugReporter.deleteOldFiles()
                     TestRunner.runContinuous(maestro, device, flowFile, env)
                 } else {
-                    val resultView = if (DisableAnsiMixin.ansiEnabled) AnsiResultView() else PlainTextResultView()
                     val resultSingle = TestRunner.runSingle(maestro, device, flowFile, env, resultView, debugOutputPath)
-                    if (resultSingle == 1) {
+                    if (resultSingle.status != FlowStatus.SUCCESS) {
                         printExitDebugMessage()
                     }
                     TestDebugReporter.deleteOldFiles()
-                    return@newSession resultSingle
+                    return@newSession if (resultSingle.status != FlowStatus.SUCCESS) 1 else 0
                 }
             }
         }

diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt
@@ -1,36 +1,29 @@
 package maestro.cli.runner
 
-import com.github.michaelbull.result.Err
-import com.github.michaelbull.result.Ok
-import com.github.michaelbull.result.Result
-import com.github.michaelbull.result.get
-import com.github.michaelbull.result.getOr
-import com.github.michaelbull.result.onFailure
+import com.github.michaelbull.result.*
 import maestro.Maestro
 import maestro.cli.device.Device
+import maestro.cli.model.FlowStatus
+import maestro.cli.model.TestExecutionSummary
 import maestro.cli.report.FlowDebugMetadata
 import maestro.cli.report.TestDebugReporter
 import maestro.cli.runner.resultview.AnsiResultView
 import maestro.cli.runner.resultview.ResultView
 import maestro.cli.runner.resultview.UiState
 import maestro.cli.util.PrintUtils
 import maestro.cli.view.ErrorViewUtils
-import maestro.debuglog.DebugLogStore
-import maestro.debuglog.LogConfig
 import maestro.orchestra.MaestroCommand
 import maestro.orchestra.MaestroInitFlow
 import maestro.orchestra.OrchestraAppState
 import maestro.orchestra.util.Env.withEnv
 import maestro.orchestra.yaml.YamlCommandReader
 import org.slf4j.LoggerFactory
 import java.io.File
-import java.nio.file.Files
 import java.nio.file.Path
-import java.nio.file.Paths
-import java.time.LocalDateTime
-import java.time.format.DateTimeFormatter
 import kotlin.concurrent.thread
-import kotlin.io.path.absolutePathString
+import kotlin.system.measureTimeMillis
+import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.measureTime
 
 object TestRunner {
 
@@ -43,27 +36,42 @@ object TestRunner {
         env: Map<String, String>,
         resultView: ResultView,
         debugOutputPath: Path
-    ): Int {
+    ): TestExecutionSummary.FlowResult {
 
         // debug
         val debug = FlowDebugMetadata()
 
-        val result = runCatching(resultView, maestro) {
-            val commands = YamlCommandReader.readCommands(flowFile.toPath())
-                .withEnv(env)
-            MaestroCommandRunner.runCommands(
-                maestro,
-                device,
-                resultView,
-                commands,
-                debug
-            )
-        }
+        val result: Result<MaestroCommandRunner.Result, Exception>
+        val time = measureTimeMillis {
+            result = runCatching(resultView, maestro) {
+                val commands = YamlCommandReader.readCommands(flowFile.toPath())
+                    .withEnv(env)
+                MaestroCommandRunner.runCommands(
+                    maestro,
+                    device,
+                    resultView,
+                    commands,
+                    debug
+                )
+            }
+        }.milliseconds
 
         TestDebugReporter.saveFlow(flowFile.name, debug, debugOutputPath)
         if (debug.exception != null) PrintUtils.err("${debug.exception?.message}")
 
-        return if (result.get()?.flowSuccess == true) 0 else 1
+        val flowStatus = if (result.get()?.flowSuccess == true) FlowStatus.SUCCESS else FlowStatus.ERROR
+
+        return TestExecutionSummary.FlowResult(
+            name = flowFile.nameWithoutExtension,
+            fileName = flowFile.nameWithoutExtension,
+            status = flowStatus,
+            failure = if (flowStatus == FlowStatus.ERROR) {
+                TestExecutionSummary.Failure(
+                    message = result.getError()?.message ?: debug.exception?.message ?: "Unknown error",
+                )
+            } else null,
+            duration = time,
+        )
     }
 
     fun runContinuous(

diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt
@@ -1,27 +1,19 @@
 package maestro.cli.runner
 
 import maestro.Maestro
-import maestro.MaestroException
 import maestro.cli.CliError
 import maestro.cli.device.Device
 import maestro.cli.model.FlowStatus
 import maestro.cli.model.TestExecutionSummary
-import maestro.cli.report.*
+import maestro.cli.report.TestSuiteReporter
+import maestro.cli.runner.resultview.ResultView
 import maestro.cli.util.PrintUtils
-import maestro.cli.util.TimeUtils
-import maestro.cli.view.ErrorViewUtils
 import maestro.cli.view.TestSuiteStatusView
 import maestro.cli.view.TestSuiteStatusView.TestSuiteViewModel
-import maestro.orchestra.Orchestra
-import maestro.orchestra.util.Env.withEnv
 import maestro.orchestra.workspace.WorkspaceExecutionPlanner
-import maestro.orchestra.yaml.YamlCommandReader
 import okio.Sink
 import org.slf4j.LoggerFactory
-import java.io.File
 import java.nio.file.Path
-import kotlin.math.roundToLong
-import kotlin.system.measureTimeMillis
 import kotlin.time.Duration.Companion.seconds
 
 class TestSuiteInteractor(
@@ -35,6 +27,7 @@ class TestSuiteInteractor(
     fun runTestSuite(
         executionPlan: WorkspaceExecutionPlanner.ExecutionPlan,
         reportOut: Sink?,
+        view: ResultView,
         env: Map<String, String>,
         debugOutputPath: Path
     ): TestExecutionSummary {
@@ -52,7 +45,7 @@ class TestSuiteInteractor(
         // first run sequence of flows if present
         val flowSequence = executionPlan.sequence
         for (flow in flowSequence?.flows ?: emptyList()) {
-            val result = runFlow(flow.toFile(), env, maestro, debugOutputPath)
+            val result = TestRunner.runSingle(maestro, device, flow.toFile(),env, view, debugOutputPath)
             flowResults.add(result)
 
             if (result.status == FlowStatus.ERROR) {
@@ -67,7 +60,7 @@ class TestSuiteInteractor(
 
         // proceed to run all other Flows
         executionPlan.flowsToRun.forEach { flow ->
-            val result = runFlow(flow.toFile(), env, maestro, debugOutputPath)
+            val result = TestRunner.runSingle(maestro, device, flow.toFile(), env, view, debugOutputPath)
 
             if (result.status == FlowStatus.ERROR) {
                 passed = false
@@ -115,131 +108,4 @@ class TestSuiteInteractor(
         return summary
     }
 
-    private fun runFlow(
-        flowFile: File,
-        env: Map<String, String>,
-        maestro: Maestro,
-        debugOutputPath: Path
-    ): TestExecutionSummary.FlowResult {
-        var flowName: String = flowFile.nameWithoutExtension
-        var flowStatus: FlowStatus
-        var errorMessage: String? = null
-
-        // debug
-        val debug = FlowDebugMetadata()
-        val debugCommands = debug.commands
-        val debugScreenshots = debug.screenshots
-
-        fun takeDebugScreenshot(status: CommandStatus): File? {
-            val containsFailed = debugScreenshots.any { it.status == CommandStatus.FAILED }
-
-            // Avoids duplicate failed images from parent commands
-            if (containsFailed && status == CommandStatus.FAILED) {
-                return null
-            }
-
-            val result = kotlin.runCatching {
-                val out = File.createTempFile("screenshot-${System.currentTimeMillis()}", ".png")
-                    .also { it.deleteOnExit() } // save to another dir before exiting
-                maestro.takeScreenshot(out, false)
-                debugScreenshots.add(
-                    ScreenshotDebugMetadata(
-                        screenshot = out,
-                        timestamp = System.currentTimeMillis(),
-                        status = status
-                    )
-                )
-                out
-            }
-
-            return result.getOrNull()
-        }
-
-        val flowTimeMillis = measureTimeMillis {
-            try {
-                val commands = YamlCommandReader.readCommands(flowFile.toPath())
-                    .withEnv(env)
-
-                val config = YamlCommandReader.getConfig(commands)
-
-                val orchestra = Orchestra(
-                    maestro = maestro,
-                    onCommandStart = { _, command ->
-                        logger.info("${command.description()} RUNNING")
-                        debugCommands[command] = CommandDebugMetadata(
-                            timestamp = System.currentTimeMillis(),
-                            status = CommandStatus.RUNNING
-                        )
-                    },
-                    onCommandComplete = { _, command ->
-                        logger.info("${command.description()} COMPLETED")
-                        debugCommands[command]?.let {
-                            it.status = CommandStatus.COMPLETED
-                            it.calculateDuration()
-                        }
-                    },
-                    onCommandFailed = { _, command, e ->
-                        logger.info("${command.description()} FAILED")
-                        if (e is MaestroException) debug.exception = e
-                        debugCommands[command]?.let {
-                            it.status = CommandStatus.FAILED
-                            it.calculateDuration()
-                            it.error = e
-                        }
-
-                        takeDebugScreenshot(CommandStatus.FAILED)
-                        Orchestra.ErrorResolution.FAIL
-                    },
-                    onCommandSkipped = { _, command ->
-                        logger.info("${command.description()} SKIPPED")
-                        debugCommands[command]?.let {
-                            it.status = CommandStatus.SKIPPED
-                        }
-                    },
-                    onCommandReset = { command ->
-                        logger.info("${command.description()} PENDING")
-                        debugCommands[command]?.let {
-                            it.status = CommandStatus.PENDING
-                        }
-                    },
-                )
-
-                config?.name?.let {
-                    flowName = it
-                }
-
-                val flowSuccess = orchestra.runFlow(commands)
-                flowStatus = if (flowSuccess) FlowStatus.SUCCESS else FlowStatus.ERROR
-            } catch (e: Exception) {
-                logger.error("Failed to complete flow", e)
-                flowStatus = FlowStatus.ERROR
-                errorMessage = ErrorViewUtils.exceptionToMessage(e)
-            }
-        }
-        val flowDuration = TimeUtils.durationInSeconds(flowTimeMillis)
-
-        TestDebugReporter.saveFlow(flowName, debug, debugOutputPath)
-
-        TestSuiteStatusView.showFlowCompletion(
-            TestSuiteViewModel.FlowResult(
-                name = flowName,
-                status = flowStatus,
-                duration = flowDuration,
-                error = debug.exception?.message,
-            )
-        )
-
-        return TestExecutionSummary.FlowResult(
-            name = flowName,
-            fileName = flowFile.nameWithoutExtension,
-            status = flowStatus,
-            failure = if (flowStatus == FlowStatus.ERROR) {
-                TestExecutionSummary.Failure(
-                    message = errorMessage ?: debug.exception?.message ?: "Unknown error",
-                )
-            } else null,
-            duration = flowDuration,
-        )
-    }
-
 }
diff --git a/maestro-cli/src/main/java/maestro/cli/runner/resultview/NoopResultView.kt b/maestro-cli/src/main/java/maestro/cli/runner/resultview/NoopResultView.kt
@@ -0,0 +1,5 @@
+package maestro.cli.runner.resultview
+
+object NoopResultView : ResultView {
+    override fun setState(state: UiState) { }
+}