-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add TestRunResultProcessor for JUnit-style XML output (#17)
* Add TestRunResultProcessor for JUnit-style XML output Smack will be configurable with optional TestRunResultProcessor instances, which will operate on the outcome of the executed tests. This commit adds a processor that generates an XML file that mimics the JUnit-style reporting. When Smack is configured to use the new processor introduced by this commit (`org.igniterealtime.smack.inttest.util.JUnitXmlTestRunResultProcessor`) then a file named `test-results.xml` will be created in the directory identified by the system property `logDir`. * Update .github/workflows/build.yml Co-authored-by: Dan Caseley <[email protected]> * Update src/main/resources/specifications.properties Co-authored-by: Dan Caseley <[email protected]> --------- Co-authored-by: Dan Caseley <[email protected]>
- Loading branch information
1 parent
6bf04c9
commit 1956aa6
Showing
3 changed files
with
830 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
305 changes: 305 additions & 0 deletions
305
src/main/java/org/igniterealtime/smack/inttest/util/JUnitXmlTestRunResultProcessor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,305 @@ | ||
package org.igniterealtime.smack.inttest.util; | ||
|
||
import org.igniterealtime.smack.inttest.FailedTest; | ||
import org.igniterealtime.smack.inttest.SmackIntegrationTestFramework; | ||
import org.igniterealtime.smack.inttest.TestNotPossible; | ||
import org.igniterealtime.smack.inttest.TestResult; | ||
import org.igniterealtime.smack.inttest.annotations.SmackIntegrationTest; | ||
import org.igniterealtime.smack.inttest.annotations.SpecificationReference; | ||
import org.w3c.dom.*; | ||
|
||
import javax.xml.parsers.DocumentBuilder; | ||
import javax.xml.parsers.DocumentBuilderFactory; | ||
import javax.xml.parsers.ParserConfigurationException; | ||
import javax.xml.transform.OutputKeys; | ||
import javax.xml.transform.Transformer; | ||
import javax.xml.transform.TransformerException; | ||
import javax.xml.transform.TransformerFactory; | ||
import javax.xml.transform.dom.DOMSource; | ||
import javax.xml.transform.stream.StreamResult; | ||
import java.io.FileOutputStream; | ||
import java.io.IOException; | ||
import java.io.OutputStream; | ||
import java.lang.reflect.Method; | ||
import java.net.URI; | ||
import java.nio.file.Files; | ||
import java.nio.file.Path; | ||
import java.nio.file.Paths; | ||
import java.time.Duration; | ||
import java.time.Instant; | ||
import java.time.ZoneOffset; | ||
import java.time.format.DateTimeFormatter; | ||
import java.util.*; | ||
import java.util.stream.Collectors; | ||
|
||
/** | ||
* Generates a JUnit-compatible XML file based on the test run results. | ||
* | ||
* @author Guus der Kinderen, [email protected] | ||
* @see <a href="https://github.com/testmoapp/junitxml">https://github.com/testmoapp/junitxml</a> | ||
*/ | ||
public class JUnitXmlTestRunResultProcessor implements SmackIntegrationTestFramework.TestRunResultProcessor { | ||
|
||
private final Properties specifications; | ||
private final Path logFile; | ||
|
||
public JUnitXmlTestRunResultProcessor() throws IOException | ||
{ | ||
final String logDir = System.getProperty("logDir"); | ||
if (logDir != null) { | ||
final Path logDirPath = Paths.get(logDir); | ||
try { | ||
Files.createDirectories(logDirPath); | ||
} catch (IOException e) { | ||
throw new IllegalStateException("Logging location does not exist or is not writable: " + logDirPath.toAbsolutePath(), e); | ||
} | ||
this.logFile = logDirPath.resolve("test-results.xml"); | ||
System.out.println("Saving JUnit-compatible XML file with results to " + logFile.toAbsolutePath()); | ||
} else { | ||
throw new IllegalStateException("Unable to read 'logDir' system property."); | ||
} | ||
|
||
specifications = new Properties(); | ||
specifications.load(JUnitXmlTestRunResultProcessor.class.getResourceAsStream("/specifications.properties")); | ||
} | ||
@Override | ||
public void process(SmackIntegrationTestFramework.TestRunResult testRunResult) | ||
{ | ||
// TODO Consider splitting up 'failures' in 'failures' and 'errors', by determining if the corresponding Throwable inherits from AssertionError or not. | ||
try { | ||
DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); | ||
DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); | ||
|
||
// root elements | ||
Document doc = docBuilder.newDocument(); | ||
|
||
// <testsuites> Usually the root element of a JUnit XML file. Some tools leave out | ||
// the <testsuites> element if there is only a single top-level <testsuite> element (which | ||
// is then used as the root element). | ||
// | ||
// name Name of the entire test run | ||
// tests Total number of tests in this file | ||
// failures Total number of failed tests in this file | ||
// errors Total number of errored tests in this file | ||
// skipped Total number of skipped tests in this file | ||
// assertions Total number of assertions for all tests in this file | ||
// time Aggregated time of all tests in this file in seconds | ||
// timestamp Date and time of when the test run was executed (in ISO 8601 format) | ||
final Element rootElement = doc.createElement("testsuites"); | ||
rootElement.setAttribute("name", "XMPP specification test run with ID " + testRunResult.getTestRunId()); | ||
rootElement.setAttribute("tests", String.valueOf(testRunResult.getNumberOfAvailableTests())); | ||
rootElement.setAttribute("failures", String.valueOf(testRunResult.getFailedTests().size())); | ||
rootElement.setAttribute("skipped", String.valueOf(testRunResult.getNotPossibleTests().size())); | ||
rootElement.setAttribute("time", String.valueOf(getAggregatedTime(testRunResult).toMillis() / 1000.0)); | ||
rootElement.setAttribute("timestamp", Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) ); | ||
doc.appendChild(rootElement); | ||
|
||
final Map<String, List<TestResult>> testResultsBySpecification = aggregateTestResultsBySpecification(testRunResult); | ||
for (final Map.Entry<String, List<TestResult>> entry : testResultsBySpecification.entrySet()) { | ||
// <testsuite> A test suite usually represents a class, folder or group of tests. | ||
// There can be many test suites in an XML file, and there can be test suites under other | ||
// test suites. | ||
// | ||
// name Name of the test suite (e.g. class name or folder name) | ||
// tests Total number of tests in this suite | ||
// failures Total number of failed tests in this suite | ||
// errors Total number of errored tests in this suite | ||
// skipped Total number of skipped tests in this suite | ||
// assertions Total number of assertions for all tests in this suite | ||
// time Aggregated time of all tests in this file in seconds | ||
// timestamp Date and time of when the test suite was executed (in ISO 8601 format) | ||
// file Source code file of this test suite | ||
final String specification = entry.getKey(); | ||
final String name; | ||
final String title; | ||
if (specification == null || specification.isBlank()) { | ||
name = "Without Specification Reference"; | ||
title = null; | ||
} else { | ||
title = specifications.getProperty(specification); | ||
if (title == null) { | ||
name = specification; | ||
} else { | ||
name = specification + ": " + title; | ||
} | ||
} | ||
final Collection<TestResult> testResults = entry.getValue(); | ||
final long failedTestCount = testResults.stream().filter(testResult -> testResult instanceof FailedTest).count(); | ||
final long notPossibleTestCount = testResults.stream().filter(testResult -> testResult instanceof TestNotPossible).count(); | ||
|
||
final Element testsuiteElement = doc.createElement("testsuite"); | ||
testsuiteElement.setAttribute("name", name); | ||
testsuiteElement.setAttribute("tests", String.valueOf(testResults.size())); | ||
testsuiteElement.setAttribute("failures", String.valueOf(failedTestCount)); | ||
testsuiteElement.setAttribute("skipped", String.valueOf(notPossibleTestCount)); | ||
testsuiteElement.setAttribute("time", String.valueOf(getAggregatedTime(testResults).toMillis() / 1000.0)); | ||
rootElement.appendChild(testsuiteElement); | ||
|
||
for (final TestResult testResult : testResults) { | ||
// <testcase> There are one or more test cases in a test suite. A test passed | ||
// if there isn't an additional result element (skipped, failure, error). | ||
// | ||
// name The name of this test case, often the method name | ||
// classname The name of the parent class/folder, often the same as the suite's name | ||
// assertions Number of assertions checked during test case execution | ||
// time Execution time of the test in seconds | ||
// file Source code file of this test case | ||
// line Source code line number of the start of this test case | ||
final Element testcaseElement = doc.createElement("testcase"); | ||
testcaseElement.setAttribute("name", testResult.concreteTest.toString()); | ||
testcaseElement.setAttribute("classname", testResult.concreteTest.getMethod().getDeclaringClass().getName()); | ||
testcaseElement.setAttribute("time", String.valueOf(testResult.duration / 1000.0)); | ||
if (testResult instanceof TestNotPossible) { | ||
final TestNotPossible testNotPossible = (TestNotPossible) testResult; | ||
final Element skippedElement = doc.createElement("skipped"); | ||
if (testNotPossible.testNotPossibleException != null && testNotPossible.testNotPossibleException.getMessage() != null && !testNotPossible.testNotPossibleException.getMessage().isBlank()) { | ||
skippedElement.setAttribute("message", testNotPossible.testNotPossibleException.getMessage()); | ||
} | ||
testcaseElement.appendChild(skippedElement); | ||
} | ||
if (testResult instanceof FailedTest) { | ||
final FailedTest failedTest = (FailedTest) testResult; | ||
final Element failureElement = doc.createElement("failure"); | ||
final Throwable failureReason = failedTest.failureReason; | ||
if (failureReason != null) { | ||
failureElement.setAttribute("type", failureReason.getClass().getSimpleName()); | ||
if (failureReason.getMessage() != null && !failureReason.getMessage().isBlank()) { | ||
failureElement.setAttribute("message", failureReason.getMessage()); | ||
} | ||
} | ||
testcaseElement.appendChild(failureElement); | ||
} | ||
final Element propertiesElement = doc.createElement("properties"); | ||
final Element logfilePropertyElement = doc.createElement("property"); | ||
logfilePropertyElement.setAttribute("name", "attachment"); | ||
logfilePropertyElement.setAttribute("value", testResult.concreteTest + ".log"); // This needs to be equal to what a configured debugger is using! | ||
propertiesElement.appendChild(logfilePropertyElement); | ||
|
||
if (specification != null && !specification.isBlank()) { | ||
final Element specificationSectionIdentifierElement = doc.createElement("property"); | ||
specificationSectionIdentifierElement.setAttribute("name", "specification identifier"); | ||
specificationSectionIdentifierElement.setAttribute("value", specification); | ||
propertiesElement.appendChild(specificationSectionIdentifierElement); | ||
} | ||
|
||
if (title != null) { | ||
final Element specificationSectionTitleElement = doc.createElement("property"); | ||
specificationSectionTitleElement.setAttribute("name", "specification title"); | ||
specificationSectionTitleElement.setAttribute("value", title); | ||
propertiesElement.appendChild(specificationSectionTitleElement); | ||
} | ||
|
||
final String specificationSection = getSpecificationSection(testResult.concreteTest.getMethod()); | ||
if (specificationSection != null) { | ||
final Element specificationSectionElement = doc.createElement("property"); | ||
specificationSectionElement.setAttribute("name", "specification section"); | ||
specificationSectionElement.setAttribute("value", specificationSection); | ||
propertiesElement.appendChild(specificationSectionElement); | ||
} | ||
|
||
final String specificationQuote = getSpecificationQuote(testResult.concreteTest.getMethod()); | ||
if (specificationQuote != null) { | ||
final Element specificationQuoteElement = doc.createElement("property"); | ||
specificationQuoteElement.setAttribute("name", "specification quote"); | ||
specificationQuoteElement.setAttribute("value", specificationQuote); | ||
propertiesElement.appendChild(specificationQuoteElement); | ||
} | ||
|
||
if (specification != null && !specification.isBlank()) { | ||
final Element specificationUrlElement = doc.createElement("property"); | ||
specificationUrlElement.setAttribute("name", "specification URL"); | ||
String link = "https://xmpp.org/extensions/" + specification.toLowerCase() + ".html"; | ||
if (specificationSection != null) { | ||
link += "#" + specificationSection; // FIXME this is wrong for XEPs, as they use the title of the section, not its number, as the anchor. Maybe convince someone to add both? | ||
} | ||
specificationUrlElement.setAttribute("value", URI.create(link).toString()); | ||
propertiesElement.appendChild(specificationUrlElement); | ||
} | ||
testcaseElement.appendChild(propertiesElement); | ||
|
||
// Seems to always be null. | ||
if (testResult.logMessages != null && !testResult.logMessages.isEmpty()) { | ||
final Element sysOutElement = doc.createElement("system-out"); | ||
sysOutElement.setTextContent(String.join(System.lineSeparator(), testResult.logMessages)); | ||
testcaseElement.appendChild(sysOutElement); | ||
} | ||
|
||
testsuiteElement.appendChild(testcaseElement); | ||
} | ||
} | ||
|
||
// write dom document to a file | ||
try (final FileOutputStream output = new FileOutputStream(logFile.toFile())) { | ||
writeXml(doc, output); | ||
} catch (IOException | TransformerException e) { | ||
throw new RuntimeException(e); | ||
} | ||
} catch (ParserConfigurationException e) { | ||
throw new RuntimeException(e); | ||
} | ||
} | ||
|
||
public static void writeXml(Document doc, OutputStream output) throws TransformerException | ||
{ | ||
final TransformerFactory transformerFactory = TransformerFactory.newInstance(); | ||
final Transformer transformer = transformerFactory.newTransformer(); | ||
final DOMSource source = new DOMSource(doc); | ||
final StreamResult result = new StreamResult(output); | ||
|
||
transformer.setOutputProperty(OutputKeys.INDENT, "yes"); | ||
transformer.transform(source, result); | ||
} | ||
|
||
public static Map<String, List<TestResult>> aggregateTestResultsBySpecification(final SmackIntegrationTestFramework.TestRunResult testRunResult) { | ||
final Collection<TestResult> allTestResults = new ArrayList<>(); | ||
allTestResults.addAll(testRunResult.getFailedTests()); | ||
allTestResults.addAll(testRunResult.getSuccessfulTests()); | ||
allTestResults.addAll(testRunResult.getNotPossibleTests()); | ||
|
||
return allTestResults.stream().collect(Collectors.groupingBy(e -> getSpecificationReference(e.concreteTest.getMethod()))); | ||
} | ||
|
||
private static String getSpecificationReference(Method method) { | ||
final SpecificationReference spec = method.getDeclaringClass().getAnnotation(SpecificationReference.class); | ||
if (spec == null || spec.document().isBlank()) { | ||
return ""; | ||
} | ||
return normalizeSpecification(spec.document().trim()); | ||
} | ||
|
||
private static String getSpecificationSection(Method method) { | ||
final SmackIntegrationTest test = method.getAnnotation(SmackIntegrationTest.class); | ||
if (!test.section().isBlank()) { | ||
return test.section().trim(); | ||
} | ||
return null; | ||
} | ||
|
||
private static String getSpecificationQuote(Method method) { | ||
final SmackIntegrationTest test = method.getAnnotation(SmackIntegrationTest.class); | ||
if (!test.quote().isBlank()) { | ||
return test.quote().trim(); | ||
} | ||
return null; | ||
} | ||
|
||
static String normalizeSpecification(String specification) { | ||
if (specification == null || specification.isBlank()) { | ||
return ""; | ||
} | ||
return specification.replaceAll("\\s", "").toUpperCase(); | ||
} | ||
|
||
public static Duration getAggregatedTime(final SmackIntegrationTestFramework.TestRunResult testRunResult) { | ||
Duration total = Duration.ZERO; | ||
total = total.plus( getAggregatedTime(testRunResult.getSuccessfulTests())); | ||
total = total.plus( getAggregatedTime(testRunResult.getFailedTests())); | ||
total = total.plus( getAggregatedTime(testRunResult.getNotPossibleTests())); | ||
return total; | ||
} | ||
|
||
public static Duration getAggregatedTime(final Collection<? extends TestResult> tests) { | ||
final long millis = tests.stream().mapToLong(test -> test.duration).sum(); | ||
return Duration.ofMillis(millis); | ||
} | ||
} |
Oops, something went wrong.