From 732dfc5877fef9b986fe615aacebac5b098b3488 Mon Sep 17 00:00:00 2001 From: Benedikt Fein Date: Wed, 20 Jul 2022 14:14:59 +0200 Subject: [PATCH] Parse nested test suites (#19) * handle nested test suites * add unit tests * ensure consistency of number of tests, errors, and failures --- pom.xml | 51 ++++++++-- ...dTestResultsNotificationPostBuildTask.java | 3 +- .../notifications/model/Testsuite.java | 99 +++++++++++++++++++ .../notifications/model/TestsuiteTest.java | 78 +++++++++++++++ .../testsuite_examples/nested_successful.xml | 27 +++++ .../nested_with_failures.xml | 43 ++++++++ 6 files changed, 293 insertions(+), 8 deletions(-) create mode 100644 src/test/java/de/tum/in/www1/jenkins/notifications/model/TestsuiteTest.java create mode 100644 src/test/resources/testsuite_examples/nested_successful.xml create mode 100644 src/test/resources/testsuite_examples/nested_with_failures.xml diff --git a/pom.xml b/pom.xml index aea6a50..1f00e1c 100644 --- a/pom.xml +++ b/pom.xml @@ -44,14 +44,21 @@ import pom + + org.junit + junit-bom + 5.8.2 + pom + import + - - io.jenkins.plugins - jaxb - 2.3.0.1 - + + io.jenkins.plugins + jaxb + 2.3.0.1 + org.jenkins-ci.plugins git @@ -59,7 +66,7 @@ com.google.code.gson gson - 2.8.7 + 2.9.0 org.jenkins-ci.plugins @@ -69,7 +76,17 @@ de.tum.in.ase static-code-analysis-parser - 1.3.4 + 1.4.0 + + + org.junit.platform + junit-platform-launcher + test + + + org.junit.jupiter + junit-jupiter-engine + test @@ -92,6 +109,26 @@ --> + + org.jacoco + jacoco-maven-plugin + 0.8.8 + + + prepare-agent + + prepare-agent + + + + report + test + + report + + + + com.diffplug.spotless spotless-maven-plugin diff --git a/src/main/java/de/tum/in/www1/jenkins/notifications/SendTestResultsNotificationPostBuildTask.java b/src/main/java/de/tum/in/www1/jenkins/notifications/SendTestResultsNotificationPostBuildTask.java index 8e1d15a..d94347b 100644 --- a/src/main/java/de/tum/in/www1/jenkins/notifications/SendTestResultsNotificationPostBuildTask.java +++ b/src/main/java/de/tum/in/www1/jenkins/notifications/SendTestResultsNotificationPostBuildTask.java @@ -110,7 +110,8 @@ private List extractTestResults(@Nonnull TaskListener taskListener, F try { final JAXBContext context = createJAXBContext(); final Unmarshaller unmarshaller = context.createUnmarshaller(); - return (Testsuite) unmarshaller.unmarshal(report.read()); + Testsuite testsuite = (Testsuite) unmarshaller.unmarshal(report.read()); + return testsuite.flatten(); } catch (JAXBException | IOException | InterruptedException e) { taskListener.error(e.getMessage(), e); diff --git a/src/main/java/de/tum/in/www1/jenkins/notifications/model/Testsuite.java b/src/main/java/de/tum/in/www1/jenkins/notifications/model/Testsuite.java index c3d4cae..fa11f4f 100644 --- a/src/main/java/de/tum/in/www1/jenkins/notifications/model/Testsuite.java +++ b/src/main/java/de/tum/in/www1/jenkins/notifications/model/Testsuite.java @@ -1,6 +1,8 @@ package de.tum.in.www1.jenkins.notifications.model; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; @@ -30,6 +32,8 @@ public class Testsuite { @XmlAttribute private int tests; + private List testSuites; + private List testCases; @XmlAttribute @@ -75,4 +79,99 @@ public List getTestCases() { public void setTestCases(List testCases) { this.testCases = testCases; } + + @Exported + public List getTestSuites() { + return testSuites; + } + + @XmlElement(name = "testsuite") + public void setTestSuites(List testSuites) { + this.testSuites = testSuites; + } + + /** + * Combines the test suite and all child test suites recursively into a single test suite. + * + * @return This test suite with all children suites merged into it. + */ + public Testsuite flatten() { + initTestCaseCounts(); + + if (testSuites != null) { + testSuites.stream().map(Testsuite::flatten).forEach(this::addOther); + + // make sure the testSuites are null, as they should not be exported in the JSON back to Artemis + testSuites = null; + } + + return this; + } + + /** + * Initializes the number of test cases, errors, and failures from the list of test cases. + *

+ * Not all test runners add the number of errors and failures to the suite information. + * Therefore, consistency has to be ensured manually here. + */ + private void initTestCaseCounts() { + errors = 0; + failures = 0; + tests = 0; + + if (testCases != null) { + updateTestCaseCounts(testCases); + } + } + + /** + * Merges the other test suite into this one. + * + * @param other Some other test suite. + */ + private void addOther(final Testsuite other) { + if (testCases == null) { + testCases = new ArrayList<>(); + } + + if (other.testCases != null) { + List otherTestCases = other.testCases.stream().map(testCase -> { + testCase.setName(buildTestCaseName(other, testCase)); + return testCase; + }).collect(Collectors.toList()); + + skipped += other.skipped; + updateTestCaseCounts(otherTestCases); + testCases.addAll(otherTestCases); + } + } + + /** + * Updates the test case count, number of errors, and number of failures in this test suite. + *

+ * Does not add the test cases themselves to the list of {@link #getTestCases()}. + * + * @param additionalTestCases The test case that will be separately added to the test cases of this suite. + */ + private void updateTestCaseCounts(final List additionalTestCases) { + tests += additionalTestCases.size(); + + for (final TestCase testCase : additionalTestCases) { + if (testCase.getFailures() != null) { + failures += 1; + } + if (testCase.getErrors() != null) { + errors += 1; + } + } + } + + private static String buildTestCaseName(final Testsuite suite, final TestCase testCase) { + if (suite.name == null) { + return testCase.getName(); + } + else { + return suite.name + '.' + testCase.getName(); + } + } } diff --git a/src/test/java/de/tum/in/www1/jenkins/notifications/model/TestsuiteTest.java b/src/test/java/de/tum/in/www1/jenkins/notifications/model/TestsuiteTest.java new file mode 100644 index 0000000..851ee43 --- /dev/null +++ b/src/test/java/de/tum/in/www1/jenkins/notifications/model/TestsuiteTest.java @@ -0,0 +1,78 @@ +package de.tum.in.www1.jenkins.notifications.model; + +import com.sun.xml.bind.v2.ContextFactory; +import org.junit.jupiter.api.Test; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; +import java.io.*; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +class TestsuiteTest { + + @Test + void testFlattenNestedSuiteSuccessful() throws Exception { + Testsuite input = loadTestSuite(Paths.get("nested_successful.xml")); + assertEquals(2, input.getTestSuites().size()); + + Testsuite flattened = input.flatten(); + assertNull(flattened.getTestSuites()); + + assertEquals(12, flattened.getTests()); + assertEquals(12, flattened.getTestCases().size()); + assertEquals(0, flattened.getErrors()); + assertEquals(0, flattened.getFailures()); + } + + @Test + void testFlattenBuildTestCaseNames() throws Exception { + Testsuite testSuite = loadTestSuite(Paths.get("nested_successful.xml")).flatten(); + + List expectedTestCaseNames = new ArrayList<>(); + expectedTestCaseNames.add("Properties.Checked by SmallCheck.Testing filtering in A"); + expectedTestCaseNames.add("Testing selectAndReflectA (0,0) []"); + + List actualTestCaseNames = testSuite.getTestCases().stream().map(TestCase::getName).collect(Collectors.toList()); + + for (String testCaseName : expectedTestCaseNames) { + Optional testCase = actualTestCaseNames.stream().filter(testCaseName::equals).findFirst(); + assertTrue(testCase.isPresent(), String.format("Did not find test case '%s' in %s", testCaseName, actualTestCaseNames)); + } + } + + @Test + void testFlattenNestedSuiteWithFailures() throws Exception { + Testsuite input = loadTestSuite(Paths.get("nested_with_failures.xml")); + assertEquals(2, input.getTestSuites().size()); + + Testsuite flattened = input.flatten(); + assertNull(flattened.getTestSuites()); + + assertEquals(12, flattened.getTests()); + assertEquals(12, flattened.getTestCases().size()); + assertEquals(2, flattened.getFailures()); + assertEquals(1, flattened.getErrors()); + } + + private Testsuite loadTestSuite(final Path reportXml) throws JAXBException { + Path resourcePath = new File("testsuite_examples").toPath().resolve(reportXml); + URL resource = getClass().getClassLoader().getResource(resourcePath.toString()); + + final JAXBContext context = createJAXBContext(); + final Unmarshaller unmarshaller = context.createUnmarshaller(); + return (Testsuite) unmarshaller.unmarshal(resource); + } + + private JAXBContext createJAXBContext() throws JAXBException { + return ContextFactory.createContext(ObjectFactory.class.getPackage().getName(), ObjectFactory.class.getClassLoader(), null); + } +} diff --git a/src/test/resources/testsuite_examples/nested_successful.xml b/src/test/resources/testsuite_examples/nested_successful.xml new file mode 100644 index 0000000..bdfe480 --- /dev/null +++ b/src/test/resources/testsuite_examples/nested_successful.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/resources/testsuite_examples/nested_with_failures.xml b/src/test/resources/testsuite_examples/nested_with_failures.xml new file mode 100644 index 0000000..ac4a721 --- /dev/null +++ b/src/test/resources/testsuite_examples/nested_with_failures.xml @@ -0,0 +1,43 @@ + + + + + + there exist (0,1) [(1,0)] such that + condition is false + + + + + + + + Some error message + + + + + *** Failed! (after 5 tests and 5 shrinks): + + >>>>>>>>>>>>>> expected + [ ( 1 , 0 ) ] + >>>>>>>>>>>>>> but got + [ ( -2 , 0 ) ] + (0,1) + [(1,0)] + Use --quickcheck-replay=220998 to reproduce. + + + + + + + + + + + +