Add Commands and Screenshots to HTML report

maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt

@@ -324,8 +324,6 @@ class CloudInteractor(
         val reportOutputSink = reportFormat.fileExtension
             ?.let { extension ->
                 (reportOutput ?: File("report$extension"))
-                    .sink()
-                    .buffer()
             }
 
         if (reportOutputSink != null) {
@@ -352,7 +350,7 @@ class CloudInteractor(
         reportFormat: ReportFormat,
         passed: Boolean,
         suiteResult: TestExecutionSummary.SuiteResult,
-        reportOutputSink: BufferedSink,
+        reportOutput: File,
         testSuiteName: String?
     ) {
         ReporterFactory.buildReporter(reportFormat, testSuiteName)
@@ -361,7 +359,7 @@ class CloudInteractor(
                     passed = passed,
                     suites = listOf(suiteResult)
                 ),
-                reportOutputSink,
+                reportOutput,
             )
     }
 

maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt

@@ -352,11 +352,10 @@ class TestCommand : Callable<Int> {
 
         format.fileExtension?.let { extension ->
             (output ?: File("report$extension"))
-                .sink()
-        }?.also { sink ->
+        }?.also { file ->
             reporter.report(
                 this,
-                sink,
+                file,
             )
         }
     }

maestro-cli/src/main/java/maestro/cli/model/TestExecutionSummary.kt

@@ -1,5 +1,10 @@
 package maestro.cli.model
 
+import maestro.MaestroException
+import maestro.cli.report.CommandDebugMetadata
+import maestro.cli.report.ScreenshotDebugMetadata
+import maestro.orchestra.MaestroCommand
+import java.util.*
 import kotlin.time.Duration
 
 data class TestExecutionSummary(
@@ -26,6 +31,8 @@ data class TestExecutionSummary(
 
     data class Failure(
         val message: String,
+        val commands: IdentityHashMap<MaestroCommand, CommandDebugMetadata>? = null,
+        val screenshots: MutableList<ScreenshotDebugMetadata>? = null,
+        var exception: MaestroException? = null
     )
-
-}
+}

maestro-cli/src/main/java/maestro/cli/report/CompositeTestSuiteReporter.kt

@@ -0,0 +1,42 @@
+package maestro.cli.report
+
+import maestro.cli.model.TestExecutionSummary
+import java.io.File
+
+class CompositeTestSuiteReporter(val reporters: Set<TestSuiteReporter>, override val fileExtension: String?) : TestSuiteReporter {
+    override fun report(
+        summary: TestExecutionSummary,
+        out: File
+    ) {
+        val baseReportDirectory = getBaseReportDirectory(out)
+
+        reporters.forEach { reporter ->
+            val reportFolder = File(baseReportDirectory, simpleReportFolderName(reporter))
+            reportFolder.mkdirs()
+
+            val reportFile = File(reportFolder, "report${reporter.fileExtension}")
+
+            if (reportFile.exists()) {
+                reportFile.delete()
+            }
+
+            reporter.report(summary, reportFile)
+        }
+    }
+
+    private fun getBaseReportDirectory(out: File): File {
+        return if (out.absoluteFile.isDirectory) {
+            out.absoluteFile
+        } else {
+            out.absoluteFile.parentFile
+        }
+    }
+
+    private fun simpleReportFolderName(reporter: TestSuiteReporter): String {
+        return reporter.javaClass.simpleName.replace(interfaceName, "")
+    }
+
+    companion object {
+        private val interfaceName = TestSuiteReporter::class.simpleName!!
+    }
+}

maestro-cli/src/main/java/maestro/cli/report/HtmlTestSuiteReporter.kt

@@ -1,15 +1,19 @@
 package maestro.cli.report
 
 import maestro.cli.model.TestExecutionSummary
-import okio.Sink
 import okio.buffer
 import kotlinx.html.*
 import kotlinx.html.stream.appendHTML
+import maestro.cli.runner.CommandStatus
+import maestro.orchestra.MaestroCommand
+import okio.sink
+import java.io.File
+import java.net.URLEncoder
 
-class HtmlTestSuiteReporter : TestSuiteReporter {
-  override fun report(summary: TestExecutionSummary, out: Sink) {
-    val bufferedOut = out.buffer()
-    val htmlContent = buildHtmlReport(summary)
+class HtmlTestSuiteReporter(override val fileExtension: String?) : TestSuiteReporter {
+  override fun report(summary: TestExecutionSummary, out: File) {
+    val bufferedOut = out.sink().buffer()
+    val htmlContent = buildHtmlReport(summary, out.parentFile)
     bufferedOut.writeUtf8(htmlContent)
     bufferedOut.close()
   }
@@ -26,9 +30,8 @@ class HtmlTestSuiteReporter : TestSuiteReporter {
     return failedTest
   }
 
-  private fun buildHtmlReport(summary: TestExecutionSummary): String {
+  private fun buildHtmlReport(summary: TestExecutionSummary, reportFolder: File): String {
     val failedTest = getFailedTest(summary)
-
     return buildString {
       appendHTML().html {
         head {
@@ -109,6 +112,7 @@ class HtmlTestSuiteReporter : TestSuiteReporter {
                           p(classes = "card-text text-danger"){
                             +flow.failure.message
                           }
+                          failureDetailsFormatted(flow.failure, flow, reportFolder)
                         }
                       }
                     }
@@ -122,4 +126,72 @@ class HtmlTestSuiteReporter : TestSuiteReporter {
       }
     }
   }
+
+    private fun DIV.failureDetailsFormatted(
+        failure: TestExecutionSummary.Failure,
+        flow: TestExecutionSummary.FlowResult,
+        reportFolder: File
+    ) {
+        br {}
+        p(classes = "card-text") {
+            table(classes = "table table-bordered") {
+                thead {
+                    tr {
+                        td { +"Command Type" }
+                        td { +"Description" }
+                        td { +"Status" }
+                        td { +"Error message" }
+                        td { +"Duration" }
+                        td { +"Stack Trace" }
+                    }
+                }
+                tbody {
+                    val sortedCommands = failure.commands?.toList()?.sortedBy { it.second.timestamp }
+
+                    sortedCommands?.forEach {
+                        val command: MaestroCommand = it.first
+                        val metadata: CommandDebugMetadata = it.second
+                        val status: CommandStatus? = metadata.status
+                        val rowClass: String = if (status == null) {
+                            ""
+                        } else {
+                            when (status) {
+                                CommandStatus.COMPLETED -> "table-success"
+                                CommandStatus.FAILED -> "table-danger"
+                                CommandStatus.PENDING -> "table-light"
+                                CommandStatus.RUNNING -> "table-info"
+                                CommandStatus.SKIPPED -> "table-secondary"
+                            }
+                        }
+                        tr(classes = "$rowClass") {
+                            td { +command.asCommand()?.javaClass?.simpleName.toString().replace(commandRegex, "") }
+                            td(classes = "strong") { +command.description() }
+                            td { +"${status?.name}" }
+                            td { +metadata.error?.message.orEmpty() }
+                            td { +"${metadata.duration ?: ""}" }
+                            td { pre { +metadata.error?.stackTraceToString().orEmpty() } }
+                        }
+                    }
+                }
+            }
+        }
+        p {
+            flow.failure?.screenshots?.forEach {
+                val sanitisedFlowName = URLEncoder.encode(flow.name.replace(" ", "_"), "utf-8")
+                val screenshotCopyFileName = "${sanitisedFlowName}_${flow.status.name}_${it.timestamp}"
+                val screenshotCopyFile = File(reportFolder, "${screenshotCopyFileName}${it.screenshot.extension}")
+                it.screenshot.copyTo(screenshotCopyFile, overwrite = true)
+
+                img {
+                    src = screenshotCopyFile.name
+                    alt = "Screenshot: ${screenshotCopyFileName}, ${flow.status.name}"
+                    style = "display: block; width: 400px; height: auto;"
+                }
+            }
+        }
+    }
+
+    companion object {
+        private val commandRegex = Regex("Command$")
+    }
 }

maestro-cli/src/main/java/maestro/cli/report/JUnitTestSuiteReporter.kt

@@ -12,22 +12,24 @@ import com.fasterxml.jackson.dataformat.xml.ser.ToXmlGenerator
 import com.fasterxml.jackson.module.kotlin.KotlinModule
 import maestro.cli.model.FlowStatus
 import maestro.cli.model.TestExecutionSummary
-import okio.Sink
 import okio.buffer
+import okio.sink
+import java.io.File
 
 class JUnitTestSuiteReporter(
     private val mapper: ObjectMapper,
-    private val testSuiteName: String?
+    private val testSuiteName: String?,
+    override val fileExtension: String?
 ) : TestSuiteReporter {
 
     override fun report(
         summary: TestExecutionSummary,
-        out: Sink
+        out: File
     ) {
         mapper
             .writerWithDefaultPrettyPrinter()
             .writeValue(
-                out.buffer().outputStream(),
+                out.sink().buffer().outputStream(),
                 TestSuites(
                     suites = summary
                         .suites
@@ -93,13 +95,14 @@ class JUnitTestSuiteReporter(
 
     companion object {
 
-        fun xml(testSuiteName: String? = null) = JUnitTestSuiteReporter(
+        fun xml(testSuiteName: String?, fileExtension: String?) = JUnitTestSuiteReporter(
             mapper = XmlMapper().apply {
                 registerModule(KotlinModule.Builder().build())
                 setSerializationInclusion(JsonInclude.Include.NON_NULL)
                 configure(ToXmlGenerator.Feature.WRITE_XML_DECLARATION, true)
             },
-            testSuiteName = testSuiteName
+            testSuiteName = testSuiteName,
+            fileExtension = fileExtension,
         )
 
     }

maestro-cli/src/main/java/maestro/cli/report/ReportFormat.kt

@@ -3,9 +3,32 @@ package maestro.cli.report
 enum class ReportFormat(
     val fileExtension: String?
 ) {
+    JUNIT(".xml") {
+        override fun createReporter(testSuiteName: String?): TestSuiteReporter {
+            return JUnitTestSuiteReporter.xml(testSuiteName, fileExtension)
+        }
+    },
+    HTML(".html") {
+        override fun createReporter(testSuiteName: String?): TestSuiteReporter {
+            return HtmlTestSuiteReporter(fileExtension)
+        }
+    },
+    COMPOSITE(".composite") {
+        override fun createReporter(testSuiteName: String?): TestSuiteReporter {
+            return CompositeTestSuiteReporter(
+                setOf(
+                    HTML.createReporter(testSuiteName),
+                    JUNIT.createReporter(testSuiteName)
+                ),
+                fileExtension
+            )
+        }
+    },
+    NOOP(null) {
+        override fun createReporter(testSuiteName: String?): TestSuiteReporter {
+            return TestSuiteReporter.NOOP
+        }
+    };
 
-    JUNIT(".xml"),
-    HTML(".html"),
-    NOOP(null),
-
-}
+    abstract fun createReporter(testSuiteName: String?): TestSuiteReporter
+}

maestro-cli/src/main/java/maestro/cli/report/ReporterFactory.kt

@@ -1,16 +1,7 @@
 package maestro.cli.report
 
-import maestro.cli.model.TestExecutionSummary
-import okio.BufferedSink
-
 object ReporterFactory {
-
     fun buildReporter(format: ReportFormat, testSuiteName: String?): TestSuiteReporter {
-        return when (format) {
-            ReportFormat.JUNIT -> JUnitTestSuiteReporter.xml(testSuiteName)
-            ReportFormat.NOOP -> TestSuiteReporter.NOOP
-            ReportFormat.HTML -> HtmlTestSuiteReporter()
-        }
+        return format.createReporter(testSuiteName)
     }
-
-}
+}

maestro-cli/src/main/java/maestro/cli/report/TestSuiteReporter.kt

@@ -1,21 +1,23 @@
 package maestro.cli.report
 
 import maestro.cli.model.TestExecutionSummary
-import okio.Sink
+import java.io.File
 
 interface TestSuiteReporter {
+    val fileExtension: String?
 
     fun report(
         summary: TestExecutionSummary,
-        out: Sink,
+        out: File,
     )
 
     companion object {
         val NOOP: TestSuiteReporter = object : TestSuiteReporter {
-            override fun report(summary: TestExecutionSummary, out: Sink) {
+            override val fileExtension: String? = null
+
+            override fun report(summary: TestExecutionSummary, out: File) {
                 // no-op
             }
         }
     }
-
-}
+}

maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt

@@ -35,7 +35,7 @@ class TestSuiteInteractor(
 
     fun runTestSuite(
         executionPlan: WorkspaceExecutionPlanner.ExecutionPlan,
-        reportOut: Sink?,
+        reportOut: File?,
         env: Map<String, String>,
         debugOutputPath: Path
     ): TestExecutionSummary {
@@ -239,6 +239,9 @@ class TestSuiteInteractor(
             failure = if (flowStatus == FlowStatus.ERROR) {
                 TestExecutionSummary.Failure(
                     message = errorMessage ?: debug.exception?.message ?: "Unknown error",
+                    commands = debug.commands,
+                    screenshots = debug.screenshots,
+                    exception = debug.exception
                 )
             } else null,
             duration = flowDuration,