From 2f1ceceb62f734ddd2882249345e047895cac80e Mon Sep 17 00:00:00 2001 From: vitekkor Date: Sat, 29 May 2021 00:15:38 +0300 Subject: [PATCH 1/2] ZestTest support --- pom.xml | 2 +- .../org/jetbrains/research/runner/Main.kt | 16 ++++-- .../jetbrains/research/runner/util/Util.kt | 15 ++++- .../runner/util/ZestTestRunListener.kt | 57 +++++++++++++++++++ 4 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 src/main/kotlin/org/jetbrains/research/runner/util/ZestTestRunListener.kt diff --git a/pom.xml b/pom.xml index 76482a4..6640d86 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.jetbrains.research kfirst-runner - 20.0.0 + 21.0.0 kfirst-runner diff --git a/src/main/kotlin/org/jetbrains/research/runner/Main.kt b/src/main/kotlin/org/jetbrains/research/runner/Main.kt index c98ce93..207cb74 100644 --- a/src/main/kotlin/org/jetbrains/research/runner/Main.kt +++ b/src/main/kotlin/org/jetbrains/research/runner/Main.kt @@ -4,14 +4,12 @@ import com.xenomachina.argparser.ArgParser import com.xenomachina.argparser.default import org.jetbrains.research.runner.data.* import org.jetbrains.research.runner.jackson.makeMapper -import org.jetbrains.research.runner.util.ConsoleReportListener -import org.jetbrains.research.runner.util.CustomContextClassLoaderExecutor -import org.jetbrains.research.runner.util.NoExitSecurityManager -import org.jetbrains.research.runner.util.TestReportListener +import org.jetbrains.research.runner.util.* import org.junit.platform.engine.discovery.DiscoverySelectors.selectPackage import org.junit.platform.launcher.EngineFilter.includeEngines import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder import org.junit.platform.launcher.core.LauncherFactory +import org.junit.runner.JUnitCore import org.slf4j.Logger import org.slf4j.LoggerFactory import ru.spbstu.kotlin.generate.combinators.Arbitrary @@ -156,9 +154,17 @@ class KFirstRunner { } .execute(testPlan) + val zestListener = ZestTestRunListener() + val zestClasses = loadZestClasses(packages) + + //Execute ZestTests + JUnitCore().apply { addListener(zestListener) }.run(*zestClasses) + val zestTestsData = zestListener.testDatum.map { it.value }.let(::TestData) + with(testReport) { logger.info("Result: $testData") + logger.info("Zest result: $zestTestsData") val mapper = makeMapper() @@ -183,6 +189,8 @@ class KFirstRunner { } + totalTestData += zestTestsData + File(args.resultFile).writer().use { mapper.writerWithDefaultPrettyPrinter().writeValue(it, totalTestData) } diff --git a/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt b/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt index 9ffff7c..8db72f4 100644 --- a/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt +++ b/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt @@ -1,9 +1,22 @@ package org.jetbrains.research.runner.util fun String.mapLines(f: (String) -> String): String = - lineSequence().map(f).joinToString("\n") + lineSequence().map(f).joinToString("\n") fun codifyString(any: Any?, indent: String): String { val result = "$any" return "```\n$result\n```".mapLines { " " + indent + it } } + +fun loadZestClasses(packages: List): Array> { + val result = ArrayList>() + for (pkg in packages) { + try { + val zestCass = Thread.currentThread().contextClassLoader.loadClass("$pkg.ZestTests") + result.add(zestCass) + } catch (ex: ClassNotFoundException) { + + } + } + return result.toTypedArray() +} diff --git a/src/main/kotlin/org/jetbrains/research/runner/util/ZestTestRunListener.kt b/src/main/kotlin/org/jetbrains/research/runner/util/ZestTestRunListener.kt new file mode 100644 index 0000000..8c793c8 --- /dev/null +++ b/src/main/kotlin/org/jetbrains/research/runner/util/ZestTestRunListener.kt @@ -0,0 +1,57 @@ +package org.jetbrains.research.runner.util + +import common.TestFailureException +import org.jetbrains.research.runner.data.* +import org.junit.jupiter.api.Tag +import org.junit.runner.Description +import org.junit.runner.notification.Failure +import org.junit.runner.notification.RunListener +import java.util.concurrent.TimeoutException + +class ZestTestRunListener : RunListener() { + val testDatum = mutableMapOf() + + override fun testFinished(description: Description) { + val pkg = description.testClass.name + val mtd = description.displayName.split("(")[0].removeSuffix("Test") + val tags = description.annotations.filterIsInstance().map { it.value }.toSet() + val resultStatus = TestResultStatus.SUCCESSFUL + val test = TestDatum(pkg, mtd, tags, listOf(TestResult(resultStatus))) + testDatum.merge(mtd, test) { old, new -> + if (old.results.isNotEmpty() && old.results.any { + it.status in setOf( + TestResultStatus.FAILED, + TestResultStatus.ABORTED, + TestResultStatus.NOT_IMPLEMENTED + ) + }) old else new + } + } + + override fun testFailure(failure: Failure) { + val pkg = failure.description.testClass.name + val mtd = failure.description.displayName.split("(")[0].removeSuffix("Test") + val tags = failure.description.annotations.filterIsInstance().map { it.value }.toSet() + var failureDatum: FailureDatum? = null + val resultStatus = when (val ex = failure.exception) { + is NotImplementedError -> TestResultStatus.NOT_IMPLEMENTED + is TestFailureException -> { + failureDatum = + TestFailureDatum(TestInput(ex.input), ex.output, ex.expectedOutput, "${ex.inner}".take(8096)) + TestResultStatus.FAILED + } + is TimeoutException -> { + failureDatum = UnknownFailureDatum(ex.toString()) + TestResultStatus.FAILED + } + else -> { + failureDatum = UnknownFailureDatum(ex.toString()) + TestResultStatus.ABORTED + } + } + val test = TestDatum(pkg, mtd, tags, listOf(TestResult(resultStatus, failureDatum))) + testDatum.merge(mtd, test) { old, new -> + if (old.isFailure) old else new + } + } +} \ No newline at end of file From d9d80f1134e7eb06b208c54fbe06e2ec7caeaad4 Mon Sep 17 00:00:00 2001 From: vitekkor Date: Fri, 18 Jun 2021 17:22:32 +0300 Subject: [PATCH 2/2] ZestTest support --- src/main/kotlin/org/jetbrains/research/runner/util/Util.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt b/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt index 8db72f4..f60fea7 100644 --- a/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt +++ b/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt @@ -10,10 +10,12 @@ fun codifyString(any: Any?, indent: String): String { fun loadZestClasses(packages: List): Array> { val result = ArrayList>() + val contextLoader = Thread.currentThread().contextClassLoader + val classPath = com.google.common.reflect.ClassPath.from(contextLoader) for (pkg in packages) { try { - val zestCass = Thread.currentThread().contextClassLoader.loadClass("$pkg.ZestTests") - result.add(zestCass) + val zestClass = classPath.getTopLevelClasses(pkg).filter { it.name.contains("ZestTests") }.map { contextLoader.loadClass(it.name) } + result.addAll(zestClass) } catch (ex: ClassNotFoundException) { }