Parse nested test suites (#19)

* handle nested test suites

* add unit tests

* ensure consistency of number of tests, errors, and failures

pom.xml

@@ -44,22 +44,29 @@
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>
+            <dependency>
+                <groupId>org.junit</groupId>
+                <artifactId>junit-bom</artifactId>
+                <version>5.8.2</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
         </dependencies>
     </dependencyManagement>
     <dependencies>
-    	<dependency>
-			<groupId>io.jenkins.plugins</groupId>
-			<artifactId>jaxb</artifactId>
-			<version>2.3.0.1</version>
-		</dependency>
+        <dependency>
+            <groupId>io.jenkins.plugins</groupId>
+            <artifactId>jaxb</artifactId>
+            <version>2.3.0.1</version>
+        </dependency>
         <dependency>
             <groupId>org.jenkins-ci.plugins</groupId>
             <artifactId>git</artifactId>
         </dependency>
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
-            <version>2.8.7</version>
+            <version>2.9.0</version>
         </dependency>
         <dependency>
             <groupId>org.jenkins-ci.plugins</groupId>
@@ -69,7 +76,17 @@
         <dependency>
             <groupId>de.tum.in.ase</groupId>
             <artifactId>static-code-analysis-parser</artifactId>
-            <version>1.3.4</version>
+            <version>1.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.platform</groupId>
+            <artifactId>junit-platform-launcher</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
         </dependency>
     </dependencies>
 
@@ -92,6 +109,26 @@
     -->
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+                <version>0.8.8</version>
+                <executions>
+                    <execution>
+                        <id>prepare-agent</id>
+                        <goals>
+                            <goal>prepare-agent</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>report</id>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>report</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>com.diffplug.spotless</groupId>
                 <artifactId>spotless-maven-plugin</artifactId>

src/main/java/de/tum/in/www1/jenkins/notifications/SendTestResultsNotificationPostBuildTask.java

@@ -110,7 +110,8 @@ private List<Testsuite> extractTestResults(@Nonnull TaskListener taskListener, F
             try {
                 final JAXBContext context = createJAXBContext();
                 final Unmarshaller unmarshaller = context.createUnmarshaller();
-                return (Testsuite) unmarshaller.unmarshal(report.read());
+                Testsuite testsuite = (Testsuite) unmarshaller.unmarshal(report.read());
+                return testsuite.flatten();
             }
             catch (JAXBException | IOException | InterruptedException e) {
                 taskListener.error(e.getMessage(), e);

src/main/java/de/tum/in/www1/jenkins/notifications/model/Testsuite.java

@@ -1,6 +1,8 @@
 package de.tum.in.www1.jenkins.notifications.model;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlElement;
@@ -30,6 +32,8 @@ public class Testsuite {
     @XmlAttribute
     private int tests;
 
+    private List<Testsuite> testSuites;
+
     private List<TestCase> testCases;
 
     @XmlAttribute
@@ -75,4 +79,99 @@ public List<TestCase> getTestCases() {
     public void setTestCases(List<TestCase> testCases) {
         this.testCases = testCases;
     }
+
+    @Exported
+    public List<Testsuite> getTestSuites() {
+        return testSuites;
+    }
+
+    @XmlElement(name = "testsuite")
+    public void setTestSuites(List<Testsuite> testSuites) {
+        this.testSuites = testSuites;
+    }
+
+    /**
+     * Combines the test suite and all child test suites recursively into a single test suite.
+     *
+     * @return This test suite with all children suites merged into it.
+     */
+    public Testsuite flatten() {
+        initTestCaseCounts();
+
+        if (testSuites != null) {
+            testSuites.stream().map(Testsuite::flatten).forEach(this::addOther);
+
+            // make sure the testSuites are null, as they should not be exported in the JSON back to Artemis
+            testSuites = null;
+        }
+
+        return this;
+    }
+
+    /**
+     * Initializes the number of test cases, errors, and failures from the list of test cases.
+     * <p>
+     * Not all test runners add the number of errors and failures to the suite information.
+     * Therefore, consistency has to be ensured manually here.
+     */
+    private void initTestCaseCounts() {
+        errors = 0;
+        failures = 0;
+        tests = 0;
+
+        if (testCases != null) {
+            updateTestCaseCounts(testCases);
+        }
+    }
+
+    /**
+     * Merges the other test suite into this one.
+     *
+     * @param other Some other test suite.
+     */
+    private void addOther(final Testsuite other) {
+        if (testCases == null) {
+            testCases = new ArrayList<>();
+        }
+
+        if (other.testCases != null) {
+            List<TestCase> otherTestCases = other.testCases.stream().map(testCase -> {
+                testCase.setName(buildTestCaseName(other, testCase));
+                return testCase;
+            }).collect(Collectors.toList());
+
+            skipped += other.skipped;
+            updateTestCaseCounts(otherTestCases);
+            testCases.addAll(otherTestCases);
+        }
+    }
+
+    /**
+     * Updates the test case count, number of errors, and number of failures in this test suite.
+     * <p>
+     * Does <em>not</em> add the test cases themselves to the list of {@link #getTestCases()}.
+     *
+     * @param additionalTestCases The test case that will be separately added to the test cases of this suite.
+     */
+    private void updateTestCaseCounts(final List<TestCase> additionalTestCases) {
+        tests += additionalTestCases.size();
+
+        for (final TestCase testCase : additionalTestCases) {
+            if (testCase.getFailures() != null) {
+                failures += 1;
+            }
+            if (testCase.getErrors() != null) {
+                errors += 1;
+            }
+        }
+    }
+
+    private static String buildTestCaseName(final Testsuite suite, final TestCase testCase) {
+        if (suite.name == null) {
+            return testCase.getName();
+        }
+        else {
+            return suite.name + '.' + testCase.getName();
+        }
+    }
 }

src/test/java/de/tum/in/www1/jenkins/notifications/model/TestsuiteTest.java

@@ -0,0 +1,78 @@
+package de.tum.in.www1.jenkins.notifications.model;
+
+import com.sun.xml.bind.v2.ContextFactory;
+import org.junit.jupiter.api.Test;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+import java.io.*;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class TestsuiteTest {
+
+    @Test
+    void testFlattenNestedSuiteSuccessful() throws Exception {
+        Testsuite input = loadTestSuite(Paths.get("nested_successful.xml"));
+        assertEquals(2, input.getTestSuites().size());
+
+        Testsuite flattened = input.flatten();
+        assertNull(flattened.getTestSuites());
+
+        assertEquals(12, flattened.getTests());
+        assertEquals(12, flattened.getTestCases().size());
+        assertEquals(0, flattened.getErrors());
+        assertEquals(0, flattened.getFailures());
+    }
+
+    @Test
+    void testFlattenBuildTestCaseNames() throws Exception {
+        Testsuite testSuite = loadTestSuite(Paths.get("nested_successful.xml")).flatten();
+
+        List<String> expectedTestCaseNames = new ArrayList<>();
+        expectedTestCaseNames.add("Properties.Checked by SmallCheck.Testing filtering in A");
+        expectedTestCaseNames.add("Testing selectAndReflectA (0,0) []");
+
+        List<String> actualTestCaseNames = testSuite.getTestCases().stream().map(TestCase::getName).collect(Collectors.toList());
+
+        for (String testCaseName : expectedTestCaseNames) {
+            Optional<String> testCase = actualTestCaseNames.stream().filter(testCaseName::equals).findFirst();
+            assertTrue(testCase.isPresent(), String.format("Did not find test case '%s' in %s", testCaseName, actualTestCaseNames));
+        }
+    }
+
+    @Test
+    void testFlattenNestedSuiteWithFailures() throws Exception {
+        Testsuite input = loadTestSuite(Paths.get("nested_with_failures.xml"));
+        assertEquals(2, input.getTestSuites().size());
+
+        Testsuite flattened = input.flatten();
+        assertNull(flattened.getTestSuites());
+
+        assertEquals(12, flattened.getTests());
+        assertEquals(12, flattened.getTestCases().size());
+        assertEquals(2, flattened.getFailures());
+        assertEquals(1, flattened.getErrors());
+    }
+
+    private Testsuite loadTestSuite(final Path reportXml) throws JAXBException {
+        Path resourcePath = new File("testsuite_examples").toPath().resolve(reportXml);
+        URL resource = getClass().getClassLoader().getResource(resourcePath.toString());
+
+        final JAXBContext context = createJAXBContext();
+        final Unmarshaller unmarshaller = context.createUnmarshaller();
+        return (Testsuite) unmarshaller.unmarshal(resource);
+    }
+
+    private JAXBContext createJAXBContext() throws JAXBException {
+        return ContextFactory.createContext(ObjectFactory.class.getPackage().getName(), ObjectFactory.class.getClassLoader(), null);
+    }
+}

src/test/resources/testsuite_examples/nested_successful.xml

@@ -0,0 +1,27 @@
+<?xml version='1.0' ?>
+<testsuite name="Tests" tests="12">
+    <testsuite name="Properties" tests="9">
+        <testsuite name="Checked by SmallCheck" tests="6">
+            <testcase name="Testing filtering in A" time="0.004" classname="Tests.Properties.Checked by SmallCheck" />
+            <testcase name="Testing mapping in A" time="0.000" classname="Tests.Properties.Checked by SmallCheck" />
+            <testcase name="Testing filtering in B" time="0.000" classname="Tests.Properties.Checked by SmallCheck" />
+            <testcase name="Testing mapping in B" time="0.000" classname="Tests.Properties.Checked by SmallCheck" />
+            <testcase name="Testing filtering in C" time="0.003" classname="Tests.Properties.Checked by SmallCheck" />
+            <testcase name="Testing mapping in C" time="0.000" classname="Tests.Properties.Checked by SmallCheck" />
+        </testsuite>
+        <testsuite name="Checked by QuickCheck" tests="3">
+            <testcase name="Testing A against sample solution" time="0.001" classname="Tests.Properties.Checked by QuickCheck" />
+            <testcase name="Testing B against sample solution" time="0.001" classname="Tests.Properties.Checked by QuickCheck" />
+            <testcase name="Testing C against sample solution" time="0.001" classname="Tests.Properties.Checked by QuickCheck" />
+        </testsuite>
+    </testsuite>
+    <!-- artificially introduce unnamed test suite -->
+    <testsuite tests="3">
+        <testcase name="Testing selectAndReflectA (0,0) []" time="0.000" classname="Tests.Unit Tests" />
+        <testcase name="Testing selectAndReflectB (0,1) [(0,0)]" time="0.000" classname="Tests.Unit Tests" />
+        <testcase name="Testing selectAndReflectC (0,1) [(-1,-1)]" time="0.000" classname="Tests.Unit Tests" />
+        <testsuite name="Empty Testsuite" tests="0">
+            <!-- intentionally empty -->
+        </testsuite>
+    </testsuite>
+</testsuite>

src/test/resources/testsuite_examples/nested_with_failures.xml

@@ -0,0 +1,43 @@
+<?xml version='1.0' ?>
+<testsuite name="Tests" tests="12">
+    <testsuite name="Properties" tests="9">
+        <testsuite name="Checked by SmallCheck" tests="6">
+            <testcase name="Testing filtering in A" time="0.000" classname="Tests.Properties.Checked by SmallCheck">
+                <failure>there exist (0,1) [(1,0)] such that
+                    condition is false
+                </failure>
+            </testcase>
+            <testcase name="Testing mapping in A" time="0.000" classname="Tests.Properties.Checked by SmallCheck"/>
+            <testcase name="Testing filtering in B" time="0.000" classname="Tests.Properties.Checked by SmallCheck"/>
+            <testcase name="Testing mapping in B" time="0.000" classname="Tests.Properties.Checked by SmallCheck"/>
+            <testcase name="Testing filtering in C" time="0.002" classname="Tests.Properties.Checked by SmallCheck"/>
+            <testcase name="Testing mapping in C" time="0.000" classname="Tests.Properties.Checked by SmallCheck">
+                <error>Some error message</error>
+            </testcase>
+        </testsuite>
+        <testsuite name="Checked by QuickCheck" tests="3">
+            <testcase name="Testing A against sample solution" time="0.000"
+                      classname="Tests.Properties.Checked by QuickCheck">
+                <failure>*** Failed! (after 5 tests and 5 shrinks):
+
+                    &gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt; expected
+                    [ ( 1 , 0 ) ]
+                    &gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt; but got
+                    [ ( -2 , 0 ) ]
+                    (0,1)
+                    [(1,0)]
+                    Use --quickcheck-replay=220998 to reproduce.
+                </failure>
+            </testcase>
+            <testcase name="Testing B against sample solution" time="0.000"
+                      classname="Tests.Properties.Checked by QuickCheck"/>
+            <testcase name="Testing C against sample solution" time="0.000"
+                      classname="Tests.Properties.Checked by QuickCheck"/>
+        </testsuite>
+    </testsuite>
+    <testsuite name="Unit Tests" tests="3">
+        <testcase name="Testing selectAndReflectA (0,0) []" time="0.000" classname="Tests.Unit Tests"/>
+        <testcase name="Testing selectAndReflectB (0,1) [(0,0)]" time="0.000" classname="Tests.Unit Tests"/>
+        <testcase name="Testing selectAndReflectC (0,1) [(-1,-1)]" time="0.000" classname="Tests.Unit Tests"/>
+    </testsuite>
+</testsuite>