From 0d45e6a680a4328e924dc7ce1515a665d80262ba Mon Sep 17 00:00:00 2001 From: Vyacheslav Frolov Date: Tue, 30 Jul 2024 22:23:18 +0100 Subject: [PATCH 1/2] Add Composite reporter that generates JUnit and HTML --- .../java/maestro/cli/cloud/CloudInteractor.kt | 6 +-- .../java/maestro/cli/command/TestCommand.kt | 5 +-- .../cli/report/CompositeTestSuiteReporter.kt | 42 +++++++++++++++++++ .../cli/report/HtmlTestSuiteReporter.kt | 9 ++-- .../cli/report/JUnitTestSuiteReporter.kt | 15 ++++--- .../java/maestro/cli/report/ReportFormat.kt | 33 ++++++++++++--- .../maestro/cli/report/ReporterFactory.kt | 13 +----- .../maestro/cli/report/TestSuiteReporter.kt | 12 +++--- .../maestro/cli/runner/TestSuiteInteractor.kt | 2 +- .../report/CompositeTestSuiteReporterTest.kt | 33 +++++++++++++++ .../cli/report/JUnitTestSuiteReporterTest.kt | 26 ++++++------ 11 files changed, 144 insertions(+), 52 deletions(-) create mode 100644 maestro-cli/src/main/java/maestro/cli/report/CompositeTestSuiteReporter.kt create mode 100644 maestro-cli/src/test/kotlin/maestro/cli/report/CompositeTestSuiteReporterTest.kt diff --git a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt index 5e56957030..d9677bdd49 100644 --- a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt +++ b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt @@ -324,8 +324,6 @@ class CloudInteractor( val reportOutputSink = reportFormat.fileExtension ?.let { extension -> (reportOutput ?: File("report$extension")) - .sink() - .buffer() } if (reportOutputSink != null) { @@ -352,7 +350,7 @@ class CloudInteractor( reportFormat: ReportFormat, passed: Boolean, suiteResult: TestExecutionSummary.SuiteResult, - reportOutputSink: BufferedSink, + reportOutput: File, testSuiteName: String? ) { ReporterFactory.buildReporter(reportFormat, testSuiteName) @@ -361,7 +359,7 @@ class CloudInteractor( passed = passed, suites = listOf(suiteResult) ), - reportOutputSink, + reportOutput, ) } diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index cc659d1453..e0a383e3fa 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -352,11 +352,10 @@ class TestCommand : Callable { format.fileExtension?.let { extension -> (output ?: File("report$extension")) - .sink() - }?.also { sink -> + }?.also { file -> reporter.report( this, - sink, + file, ) } } diff --git a/maestro-cli/src/main/java/maestro/cli/report/CompositeTestSuiteReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/CompositeTestSuiteReporter.kt new file mode 100644 index 0000000000..fe69dedc1a --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/report/CompositeTestSuiteReporter.kt @@ -0,0 +1,42 @@ +package maestro.cli.report + +import maestro.cli.model.TestExecutionSummary +import java.io.File + +class CompositeTestSuiteReporter(val reporters: Set, override val fileExtension: String?) : TestSuiteReporter { + override fun report( + summary: TestExecutionSummary, + out: File + ) { + val baseReportDirectory = getBaseReportDirectory(out) + + reporters.forEach { reporter -> + val reportFolder = File(baseReportDirectory, simpleReportFolderName(reporter)) + reportFolder.mkdirs() + + val reportFile = File(reportFolder, "report${reporter.fileExtension}") + + if (reportFile.exists()) { + reportFile.delete() + } + + reporter.report(summary, reportFile) + } + } + + private fun getBaseReportDirectory(out: File): File { + return if (out.absoluteFile.isDirectory) { + out.absoluteFile + } else { + out.absoluteFile.parentFile + } + } + + private fun simpleReportFolderName(reporter: TestSuiteReporter): String { + return reporter.javaClass.simpleName.replace(interfaceName, "") + } + + companion object { + private val interfaceName = TestSuiteReporter::class.simpleName!! + } +} diff --git a/maestro-cli/src/main/java/maestro/cli/report/HtmlTestSuiteReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/HtmlTestSuiteReporter.kt index c02551c0cf..004960238c 100644 --- a/maestro-cli/src/main/java/maestro/cli/report/HtmlTestSuiteReporter.kt +++ b/maestro-cli/src/main/java/maestro/cli/report/HtmlTestSuiteReporter.kt @@ -1,14 +1,15 @@ package maestro.cli.report import maestro.cli.model.TestExecutionSummary -import okio.Sink import okio.buffer import kotlinx.html.* import kotlinx.html.stream.appendHTML +import okio.sink +import java.io.File -class HtmlTestSuiteReporter : TestSuiteReporter { - override fun report(summary: TestExecutionSummary, out: Sink) { - val bufferedOut = out.buffer() +class HtmlTestSuiteReporter(override val fileExtension: String?) : TestSuiteReporter { + override fun report(summary: TestExecutionSummary, out: File) { + val bufferedOut = out.sink().buffer() val htmlContent = buildHtmlReport(summary) bufferedOut.writeUtf8(htmlContent) bufferedOut.close() diff --git a/maestro-cli/src/main/java/maestro/cli/report/JUnitTestSuiteReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/JUnitTestSuiteReporter.kt index ecddbed8e3..82b9f3ba37 100644 --- a/maestro-cli/src/main/java/maestro/cli/report/JUnitTestSuiteReporter.kt +++ b/maestro-cli/src/main/java/maestro/cli/report/JUnitTestSuiteReporter.kt @@ -12,22 +12,24 @@ import com.fasterxml.jackson.dataformat.xml.ser.ToXmlGenerator import com.fasterxml.jackson.module.kotlin.KotlinModule import maestro.cli.model.FlowStatus import maestro.cli.model.TestExecutionSummary -import okio.Sink import okio.buffer +import okio.sink +import java.io.File class JUnitTestSuiteReporter( private val mapper: ObjectMapper, - private val testSuiteName: String? + private val testSuiteName: String?, + override val fileExtension: String? ) : TestSuiteReporter { override fun report( summary: TestExecutionSummary, - out: Sink + out: File ) { mapper .writerWithDefaultPrettyPrinter() .writeValue( - out.buffer().outputStream(), + out.sink().buffer().outputStream(), TestSuites( suites = summary .suites @@ -93,13 +95,14 @@ class JUnitTestSuiteReporter( companion object { - fun xml(testSuiteName: String? = null) = JUnitTestSuiteReporter( + fun xml(testSuiteName: String?, fileExtension: String?) = JUnitTestSuiteReporter( mapper = XmlMapper().apply { registerModule(KotlinModule.Builder().build()) setSerializationInclusion(JsonInclude.Include.NON_NULL) configure(ToXmlGenerator.Feature.WRITE_XML_DECLARATION, true) }, - testSuiteName = testSuiteName + testSuiteName = testSuiteName, + fileExtension = fileExtension, ) } diff --git a/maestro-cli/src/main/java/maestro/cli/report/ReportFormat.kt b/maestro-cli/src/main/java/maestro/cli/report/ReportFormat.kt index fd1bc1f8cb..ba478ae382 100644 --- a/maestro-cli/src/main/java/maestro/cli/report/ReportFormat.kt +++ b/maestro-cli/src/main/java/maestro/cli/report/ReportFormat.kt @@ -3,9 +3,32 @@ package maestro.cli.report enum class ReportFormat( val fileExtension: String? ) { + JUNIT(".xml") { + override fun createReporter(testSuiteName: String?): TestSuiteReporter { + return JUnitTestSuiteReporter.xml(testSuiteName, fileExtension) + } + }, + HTML(".html") { + override fun createReporter(testSuiteName: String?): TestSuiteReporter { + return HtmlTestSuiteReporter(fileExtension) + } + }, + COMPOSITE(".composite") { + override fun createReporter(testSuiteName: String?): TestSuiteReporter { + return CompositeTestSuiteReporter( + setOf( + HTML.createReporter(testSuiteName), + JUNIT.createReporter(testSuiteName) + ), + fileExtension + ) + } + }, + NOOP(null) { + override fun createReporter(testSuiteName: String?): TestSuiteReporter { + return TestSuiteReporter.NOOP + } + }; - JUNIT(".xml"), - HTML(".html"), - NOOP(null), - -} \ No newline at end of file + abstract fun createReporter(testSuiteName: String?): TestSuiteReporter +} diff --git a/maestro-cli/src/main/java/maestro/cli/report/ReporterFactory.kt b/maestro-cli/src/main/java/maestro/cli/report/ReporterFactory.kt index f035572a13..b6dbd4557b 100644 --- a/maestro-cli/src/main/java/maestro/cli/report/ReporterFactory.kt +++ b/maestro-cli/src/main/java/maestro/cli/report/ReporterFactory.kt @@ -1,16 +1,7 @@ package maestro.cli.report -import maestro.cli.model.TestExecutionSummary -import okio.BufferedSink - object ReporterFactory { - fun buildReporter(format: ReportFormat, testSuiteName: String?): TestSuiteReporter { - return when (format) { - ReportFormat.JUNIT -> JUnitTestSuiteReporter.xml(testSuiteName) - ReportFormat.NOOP -> TestSuiteReporter.NOOP - ReportFormat.HTML -> HtmlTestSuiteReporter() - } + return format.createReporter(testSuiteName) } - -} \ No newline at end of file +} diff --git a/maestro-cli/src/main/java/maestro/cli/report/TestSuiteReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/TestSuiteReporter.kt index 40a7fe89f8..082df74df9 100644 --- a/maestro-cli/src/main/java/maestro/cli/report/TestSuiteReporter.kt +++ b/maestro-cli/src/main/java/maestro/cli/report/TestSuiteReporter.kt @@ -1,21 +1,23 @@ package maestro.cli.report import maestro.cli.model.TestExecutionSummary -import okio.Sink +import java.io.File interface TestSuiteReporter { + val fileExtension: String? fun report( summary: TestExecutionSummary, - out: Sink, + out: File, ) companion object { val NOOP: TestSuiteReporter = object : TestSuiteReporter { - override fun report(summary: TestExecutionSummary, out: Sink) { + override val fileExtension: String? = null + + override fun report(summary: TestExecutionSummary, out: File) { // no-op } } } - -} \ No newline at end of file +} diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt index e90564c40f..0f266f95ed 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt @@ -35,7 +35,7 @@ class TestSuiteInteractor( fun runTestSuite( executionPlan: WorkspaceExecutionPlanner.ExecutionPlan, - reportOut: Sink?, + reportOut: File?, env: Map, debugOutputPath: Path ): TestExecutionSummary { diff --git a/maestro-cli/src/test/kotlin/maestro/cli/report/CompositeTestSuiteReporterTest.kt b/maestro-cli/src/test/kotlin/maestro/cli/report/CompositeTestSuiteReporterTest.kt new file mode 100644 index 0000000000..558a282db2 --- /dev/null +++ b/maestro-cli/src/test/kotlin/maestro/cli/report/CompositeTestSuiteReporterTest.kt @@ -0,0 +1,33 @@ +package maestro.cli.report + +import com.google.common.truth.Truth.assertThat +import org.junit.jupiter.api.Test + +class CompositeTestSuiteReporterTest { + + @Test + fun `Composite test report created`() { + // Given + val reportFormat = ReportFormat.COMPOSITE + + // When + val reporter = reportFormat.createReporter("testSuiteName") + + // Then + assertThat(reporter is CompositeTestSuiteReporter).isTrue() + } + + @Test + fun `Composite test report contains JUNIT and HTML reports`() { + // Given + val reportFormat = ReportFormat.COMPOSITE + + // When + val reporter = reportFormat.createReporter("testSuiteName") + val reporters = (reporter as CompositeTestSuiteReporter).reporters + + // Then + assertThat(reporters.any { it is JUnitTestSuiteReporter }).isTrue() + assertThat(reporters.any { it is HtmlTestSuiteReporter }).isTrue() + } +} diff --git a/maestro-cli/src/test/kotlin/maestro/cli/report/JUnitTestSuiteReporterTest.kt b/maestro-cli/src/test/kotlin/maestro/cli/report/JUnitTestSuiteReporterTest.kt index 4550734b1b..67fcec1a64 100644 --- a/maestro-cli/src/test/kotlin/maestro/cli/report/JUnitTestSuiteReporterTest.kt +++ b/maestro-cli/src/test/kotlin/maestro/cli/report/JUnitTestSuiteReporterTest.kt @@ -3,8 +3,8 @@ package maestro.cli.report import com.google.common.truth.Truth.assertThat import maestro.cli.model.FlowStatus import maestro.cli.model.TestExecutionSummary -import okio.Buffer import org.junit.jupiter.api.Test +import java.io.File import kotlin.time.Duration.Companion.seconds class JUnitTestSuiteReporterTest { @@ -12,7 +12,7 @@ class JUnitTestSuiteReporterTest { @Test fun `XML - Test passed`() { // Given - val testee = JUnitTestSuiteReporter.xml() + val testee = JUnitTestSuiteReporter.xml(null, null) val summary = TestExecutionSummary( passed = true, @@ -38,14 +38,14 @@ class JUnitTestSuiteReporterTest { ) ) ) - val sink = Buffer() + val reportOutput = File("tmpReportOutput") // When testee.report( summary = summary, - out = sink + out = reportOutput ) - val resultStr = sink.readUtf8() + val resultStr = reportOutput.readText(Charsets.UTF_8) // Then assertThat(resultStr).isEqualTo( @@ -65,7 +65,7 @@ class JUnitTestSuiteReporterTest { @Test fun `XML - Test failed`() { // Given - val testee = JUnitTestSuiteReporter.xml() + val testee = JUnitTestSuiteReporter.xml(null, null) val summary = TestExecutionSummary( passed = false, @@ -91,14 +91,14 @@ class JUnitTestSuiteReporterTest { ) ) ) - val sink = Buffer() + val reportOutput = File("tmpReportOutput") // When testee.report( summary = summary, - out = sink + out = reportOutput ) - val resultStr = sink.readUtf8() + val resultStr = reportOutput.readText(Charsets.UTF_8) // Then assertThat(resultStr).isEqualTo( @@ -120,7 +120,7 @@ class JUnitTestSuiteReporterTest { @Test fun `XML - Custom test suite name is used when present`() { // Given - val testee = JUnitTestSuiteReporter.xml("Custom test suite name") + val testee = JUnitTestSuiteReporter.xml("Custom test suite name", "fileExtension") val summary = TestExecutionSummary( passed = true, @@ -145,14 +145,14 @@ class JUnitTestSuiteReporterTest { ) ) ) - val sink = Buffer() + val reportOutput = File("tmpReportOutput") // When testee.report( summary = summary, - out = sink + out = reportOutput ) - val resultStr = sink.readUtf8() + val resultStr = reportOutput.readText(Charsets.UTF_8) // Then assertThat(resultStr).isEqualTo( From 0a45ea47077c714e584bb7dde588e2d36568cae9 Mon Sep 17 00:00:00 2001 From: Vyacheslav Frolov Date: Wed, 31 Jul 2024 21:13:45 +0100 Subject: [PATCH 2/2] Add Commands and Screenshots to HTML report --- .../maestro/cli/model/TestExecutionSummary.kt | 11 ++- .../cli/report/HtmlTestSuiteReporter.kt | 77 ++++++++++++++++++- .../maestro/cli/runner/TestSuiteInteractor.kt | 3 + 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/maestro-cli/src/main/java/maestro/cli/model/TestExecutionSummary.kt b/maestro-cli/src/main/java/maestro/cli/model/TestExecutionSummary.kt index 475a184ff7..c488a3d955 100644 --- a/maestro-cli/src/main/java/maestro/cli/model/TestExecutionSummary.kt +++ b/maestro-cli/src/main/java/maestro/cli/model/TestExecutionSummary.kt @@ -1,5 +1,10 @@ package maestro.cli.model +import maestro.MaestroException +import maestro.cli.report.CommandDebugMetadata +import maestro.cli.report.ScreenshotDebugMetadata +import maestro.orchestra.MaestroCommand +import java.util.* import kotlin.time.Duration data class TestExecutionSummary( @@ -26,6 +31,8 @@ data class TestExecutionSummary( data class Failure( val message: String, + val commands: IdentityHashMap? = null, + val screenshots: MutableList? = null, + var exception: MaestroException? = null ) - -} \ No newline at end of file +} diff --git a/maestro-cli/src/main/java/maestro/cli/report/HtmlTestSuiteReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/HtmlTestSuiteReporter.kt index 004960238c..36fc5c0e01 100644 --- a/maestro-cli/src/main/java/maestro/cli/report/HtmlTestSuiteReporter.kt +++ b/maestro-cli/src/main/java/maestro/cli/report/HtmlTestSuiteReporter.kt @@ -4,13 +4,16 @@ import maestro.cli.model.TestExecutionSummary import okio.buffer import kotlinx.html.* import kotlinx.html.stream.appendHTML +import maestro.cli.runner.CommandStatus +import maestro.orchestra.MaestroCommand import okio.sink import java.io.File +import java.net.URLEncoder class HtmlTestSuiteReporter(override val fileExtension: String?) : TestSuiteReporter { override fun report(summary: TestExecutionSummary, out: File) { val bufferedOut = out.sink().buffer() - val htmlContent = buildHtmlReport(summary) + val htmlContent = buildHtmlReport(summary, out.parentFile) bufferedOut.writeUtf8(htmlContent) bufferedOut.close() } @@ -27,9 +30,8 @@ class HtmlTestSuiteReporter(override val fileExtension: String?) : TestSuiteRepo return failedTest } - private fun buildHtmlReport(summary: TestExecutionSummary): String { + private fun buildHtmlReport(summary: TestExecutionSummary, reportFolder: File): String { val failedTest = getFailedTest(summary) - return buildString { appendHTML().html { head { @@ -110,6 +112,7 @@ class HtmlTestSuiteReporter(override val fileExtension: String?) : TestSuiteRepo p(classes = "card-text text-danger"){ +flow.failure.message } + failureDetailsFormatted(flow.failure, flow, reportFolder) } } } @@ -123,4 +126,72 @@ class HtmlTestSuiteReporter(override val fileExtension: String?) : TestSuiteRepo } } } + + private fun DIV.failureDetailsFormatted( + failure: TestExecutionSummary.Failure, + flow: TestExecutionSummary.FlowResult, + reportFolder: File + ) { + br {} + p(classes = "card-text") { + table(classes = "table table-bordered") { + thead { + tr { + td { +"Command Type" } + td { +"Description" } + td { +"Status" } + td { +"Error message" } + td { +"Duration" } + td { +"Stack Trace" } + } + } + tbody { + val sortedCommands = failure.commands?.toList()?.sortedBy { it.second.timestamp } + + sortedCommands?.forEach { + val command: MaestroCommand = it.first + val metadata: CommandDebugMetadata = it.second + val status: CommandStatus? = metadata.status + val rowClass: String = if (status == null) { + "" + } else { + when (status) { + CommandStatus.COMPLETED -> "table-success" + CommandStatus.FAILED -> "table-danger" + CommandStatus.PENDING -> "table-light" + CommandStatus.RUNNING -> "table-info" + CommandStatus.SKIPPED -> "table-secondary" + } + } + tr(classes = "$rowClass") { + td { +command.asCommand()?.javaClass?.simpleName.toString().replace(commandRegex, "") } + td(classes = "strong") { +command.description() } + td { +"${status?.name}" } + td { +metadata.error?.message.orEmpty() } + td { +"${metadata.duration ?: ""}" } + td { pre { +metadata.error?.stackTraceToString().orEmpty() } } + } + } + } + } + } + p { + flow.failure?.screenshots?.forEach { + val sanitisedFlowName = URLEncoder.encode(flow.name.replace(" ", "_"), "utf-8") + val screenshotCopyFileName = "${sanitisedFlowName}_${flow.status.name}_${it.timestamp}" + val screenshotCopyFile = File(reportFolder, "${screenshotCopyFileName}${it.screenshot.extension}") + it.screenshot.copyTo(screenshotCopyFile, overwrite = true) + + img { + src = screenshotCopyFile.name + alt = "Screenshot: ${screenshotCopyFileName}, ${flow.status.name}" + style = "display: block; width: 400px; height: auto;" + } + } + } + } + + companion object { + private val commandRegex = Regex("Command$") + } } diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt index 0f266f95ed..87467f2572 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt @@ -239,6 +239,9 @@ class TestSuiteInteractor( failure = if (flowStatus == FlowStatus.ERROR) { TestExecutionSummary.Failure( message = errorMessage ?: debug.exception?.message ?: "Unknown error", + commands = debug.commands, + screenshots = debug.screenshots, + exception = debug.exception ) } else null, duration = flowDuration,