diff --git a/doc/README.md b/doc/README.md index f467094c..f5508bce 100644 --- a/doc/README.md +++ b/doc/README.md @@ -107,10 +107,10 @@ build. Conversely, the project view side panel link won't go away until after configuration change/resave in project. We're working on a solution to this. -![](images/loghtml_sidepanel.png) +![sidepanel link](images/sidepanel.png) *side panel link* -![](images/robo_loglink_projectpage.png) +![summary link](images/robot_4_table.png) *Summary link* #### Configuring Robot overall pass/fail to show in the project list @@ -128,7 +128,7 @@ solution to this. [http://YOURJENKINSHOST/configure](http://yourjenkinshost/configure) )and select your newly created view to be the default one. -![](images/robot_view_column.png) +![List view](images/robot_view_column.png) *List view column in action* #### Using token macros in e-mail report @@ -178,25 +178,36 @@ and [here](https://content-security-policy.com/) how to change you CSP settings but be aware that **Changing CSP settings will potentially expose you Jenkins instance for security vulnerabilities**. -### Overall Screenshots +### Robot Framework 4.x compatibility -##### Config +The plugin supports both Robot Framework 3.x and 4.x output files. However, in order to support both, the plugin +shows some extra information for both. [In Robot Framework 4.0 test criticality was removed and "SKIP" status was added](https://github.com/robotframework/robotframework/blob/master/doc/releasenotes/rf-4.0.rst). So for 3.x the +results overview will show a `Skipped` column, which will always be 0 and for Robot Frameowork 4.x output files +the `Critical tests` row will always be 0. -![](images/config1.png) +Skipped tests aren't taken into account when calculating pass percentage, but they are calculated to the total +amount of tests. -##### Project view +Because criticality was removed in Robot Framework 4.0, having the `Use thresholds for critical tests only` checkbox +checked will always result in a passing step (because pass percentage is always considered to be 100% when there are +0 tests). In order to have set build status correctly, you **must** uncheck the checkbox or use `onlyCritical: false` +in your pipeline when you call `robot`. -![](images/project_page.png) +We are planning of dropping support for Robot Framework 3.x and earlier some time in distant future. + +### Overall Screenshots + + +##### Project view -##### Build view +![Build page overview](images/build_page.png) -![](images/build_page.png) ##### Detailed build view -![](images/detailed.png) +![Detailed results view](images/detailed.png) ## Bugs [List of open bugs on -robot-plugin](https://issues.jenkins-ci.org/issues/?jql=component%20%3D%20robot-plugin%20AND%20resolution%20is%20EMPTY%20ORDER%20BY%20updated%20DESC) \ No newline at end of file +robot-plugin](https://issues.jenkins-ci.org/issues/?jql=component%20%3D%20robot-plugin%20AND%20resolution%20is%20EMPTY%20ORDER%20BY%20updated%20DESC) diff --git a/doc/images/build_page.png b/doc/images/build_page.png index a36c49c2..6c047d60 100644 Binary files a/doc/images/build_page.png and b/doc/images/build_page.png differ diff --git a/doc/images/detailed.png b/doc/images/detailed.png index fbfbdd56..d81abd2b 100644 Binary files a/doc/images/detailed.png and b/doc/images/detailed.png differ diff --git a/doc/images/loghtml_sidepanel.png b/doc/images/loghtml_sidepanel.png deleted file mode 100644 index 33757362..00000000 Binary files a/doc/images/loghtml_sidepanel.png and /dev/null differ diff --git a/doc/images/robo_loglink_projectpage.png b/doc/images/robo_loglink_projectpage.png deleted file mode 100644 index 1125f781..00000000 Binary files a/doc/images/robo_loglink_projectpage.png and /dev/null differ diff --git a/doc/images/robot_4_table.png b/doc/images/robot_4_table.png new file mode 100644 index 00000000..15dd2738 Binary files /dev/null and b/doc/images/robot_4_table.png differ diff --git a/doc/images/robot_view_column.png b/doc/images/robot_view_column.png index d3e987a3..5de3bc10 100644 Binary files a/doc/images/robot_view_column.png and b/doc/images/robot_view_column.png differ diff --git a/doc/images/sidepanel.png b/doc/images/sidepanel.png new file mode 100644 index 00000000..b9eabfe7 Binary files /dev/null and b/doc/images/sidepanel.png differ diff --git a/src/main/java/hudson/plugins/robot/AggregatedRobotAction.java b/src/main/java/hudson/plugins/robot/AggregatedRobotAction.java index 87863f26..5302ba91 100644 --- a/src/main/java/hudson/plugins/robot/AggregatedRobotAction.java +++ b/src/main/java/hudson/plugins/robot/AggregatedRobotAction.java @@ -109,7 +109,7 @@ public static class AggregatedRobotResult extends RobotResult { private static final long serialVersionUID = 1L; private final transient AggregatedRobotAction parent; - private int passed, failed, criticalPassed, criticalFailed; + private int passed, failed, skipped, criticalPassed, criticalFailed; public AggregatedRobotResult(AggregatedRobotAction parent) { this.parent = parent; @@ -120,6 +120,7 @@ public void addResult(RobotResult result) { criticalPassed += result.getCriticalPassed(); failed += result.getOverallFailed(); passed += result.getOverallPassed(); + skipped += result.getOverallSkipped(); } @Override @@ -147,9 +148,14 @@ public long getOverallFailed() { return failed; } + @Override + public long getOverallSkipped() { + return skipped; + } + @Override public long getOverallTotal() { - return failed + passed; + return failed + passed + skipped; } @Override @@ -162,6 +168,11 @@ public int getPassed() { return (int) getOverallPassed(); } + @Override + public int getSkipped() { + return (int) getOverallSkipped(); + } + @Override public AbstractBuild getOwner() { return parent.getOwner(); diff --git a/src/main/java/hudson/plugins/robot/RobotParser.java b/src/main/java/hudson/plugins/robot/RobotParser.java index b7526e10..cb1809e5 100644 --- a/src/main/java/hudson/plugins/robot/RobotParser.java +++ b/src/main/java/hudson/plugins/robot/RobotParser.java @@ -18,7 +18,6 @@ import hudson.AbortException; import hudson.FilePath; import hudson.Util; -import hudson.model.AbstractBuild; import hudson.model.Run; import hudson.plugins.robot.model.RobotTestObject; import hudson.plugins.robot.model.RobotCaseResult; @@ -46,9 +45,8 @@ public class RobotParser { public RobotResult parse(String outputFileLocations, String outputPath, Run build, FilePath workSpace, String logFileName, String reportFileName) throws InterruptedException, IOException { - RobotResult result = new FilePath(workSpace, outputPath).act( + return new FilePath(workSpace, outputPath).act( new RobotParserCallable(outputFileLocations, logFileName, reportFileName)); - return result; } public static final class RobotParserCallable implements @@ -91,7 +89,7 @@ public RobotResult invoke(File ws, VirtualChannel channel) //get the potential directories emerging from the use of GLOB filemask accounted in the splitted file parsing String dirFromFileGLOB = new File(file).getParent(); if(dirFromFileGLOB != null) - baseDirectory = new File(baseDirectory, dirFromFileGLOB.toString()); + baseDirectory = new File(baseDirectory, dirFromFileGLOB); FileInputStream inputStream = new FileInputStream(reportFile); try { XMLStreamReader reader = factory.createXMLStreamReader(inputStream, "UTF-8"); @@ -202,7 +200,7 @@ private RobotSuiteResult getSplitXMLSuite(RobotTestObject parent, File baseDirec } private String ignoreUntilStarts(XMLStreamReader reader, String... elements) throws XMLStreamException { - List elementStack = new ArrayList(); + List elementStack = new ArrayList<>(); while(reader.hasNext()) { reader.next(); if (reader.isStartElement()) { @@ -237,7 +235,7 @@ private boolean isNameInElements(String name, String[] elements) { } private void ignoreUntilEnds(XMLStreamReader reader, String element) throws XMLStreamException { - List elementStack = new ArrayList(); + List elementStack = new ArrayList<>(); while(reader.hasNext()) { reader.next(); if (reader.isStartElement()) { @@ -263,7 +261,10 @@ private void ignoreUntilEnds(XMLStreamReader reader, String element) throws XMLS } private String getSpacesPerNestedLevel(int level) { - StringBuffer spaces = new StringBuffer(); + StringBuilder spaces = new StringBuilder(); + if (level > 0) { + spaces.append("\n"); + } for (int i = 0; i < level; i++) { spaces.append(" "); } @@ -280,71 +281,47 @@ private RobotCaseResult processTest(XMLStreamReader reader, RobotSuiteResult res caseResult.setId(reader.getAttributeValue(null, "id")); //parse test tags caseResult.setDescription(""); - caseResult.addTags(new ArrayList()); - StringBuffer stackTrace = new StringBuffer(); + caseResult.addTags(new ArrayList<>()); + StringBuilder stackTrace = new StringBuilder(); //parse stacktrace - String xmlTag = ignoreUntilStarts(reader, "kw", "doc", "tags", "status"); - if (xmlTag == "kw") { - //get all sequential keywords - int nestedCount = 0; - do { - //get all nested keywords - do { - String kw = reader.getAttributeValue(null, "name"); - stackTrace.append(getSpacesPerNestedLevel(nestedCount) + kw); - xmlTag = ignoreUntilStarts(reader, "kw", "arguments", "status"); - //get arguments of current keyword if any - if (xmlTag == "arguments") { - while (reader.hasNext() && !(reader.isEndElement() && reader.getLocalName() == "arguments")) { - reader.next(); - if (reader.isStartElement() && reader.getLocalName() == "arg") { - reader.next(); //skip arg start - stackTrace.append(" " + reader.getText()); - reader.next(); //skip text - reader.next(); //skip arg end - } - } - xmlTag = ignoreUntilStarts(reader, "kw", "status"); - } - stackTrace.append("\n"); - nestedCount++; - } while (xmlTag == "kw"); - //unroll: after reading a status, the kw must end too - do { - ignoreUntilEnds(reader, "status"); - ignoreUntilEnds(reader, "kw"); - nestedCount--; - //it depends on the next tag: - //- kw: a sequential kw follows - //- status: a. status of previous kw -> unroll further; b. status of test - //- doc||tags: follows last kw (nestedCount must be 0) - xmlTag = ignoreUntilStarts(reader, "kw", "doc", "tags", "status"); - } while (xmlTag == "status" && nestedCount > 0); - } while (xmlTag == "kw"); + String xmlTag = ignoreUntilStarts(reader, "kw", "for", "if", "doc", "tags", "tag", "status"); + while (xmlTag.equals("kw") || xmlTag.equals("for") || xmlTag.equals("if")) { + if (xmlTag.equals("if")) { + stackTrace.append(processIf(reader, 0)); + } else if (xmlTag.equals("for")) { + stackTrace.append(processForLoop(reader, 0)); + } else { + stackTrace.append(processKeyword(reader, 0)); + } + xmlTag = ignoreUntilStarts(reader, "kw", "for", "if", "doc", "tags", "tag", "status"); } - caseResult.setStackTrace(stackTrace.toString()); - if (xmlTag == "doc") { + caseResult.setStackTrace(stackTrace.toString().trim().replaceAll("\n+", "\n")); + + if (xmlTag.equals("doc")) { reader.next(); if (reader.hasText()) { caseResult.setDescription(reader.getText()); } reader.next(); - xmlTag = ignoreUntilStarts(reader, "tags", "status"); + xmlTag = ignoreUntilStarts(reader, "tags", "tag", "status"); } - if (xmlTag == "tags") { + if (xmlTag.equals("tags") || xmlTag.equals("tag")) { caseResult.addTags(processTags(reader)); - ignoreUntilStarts(reader, "status"); + if (!reader.getLocalName().equals("status")) { + ignoreUntilStarts(reader, "status"); + } } //parse test details from nested status caseResult.setPassed("PASS".equals(reader.getAttributeValue(null, "status"))); + caseResult.setSkipped("SKIP".equals(reader.getAttributeValue(null, "status"))); caseResult.setStarttime(reader.getAttributeValue(null, "starttime")); caseResult.setEndtime(reader.getAttributeValue(null, "endtime")); setCriticalityIfAvailable(reader, caseResult); while(reader.hasNext()){ reader.next(); - if(reader.getEventType() == XMLStreamReader.CHARACTERS){ + if(reader.isCharacters()){ String error = reader.getText(); caseResult.setErrorMsg(error); } else if (reader.isEndElement()) { @@ -357,40 +334,176 @@ private RobotCaseResult processTest(XMLStreamReader reader, RobotSuiteResult res } } // reset stack trace if the test is passed - if (caseResult.isPassed()) { + if (caseResult.isPassed() || caseResult.isSkipped()) { caseResult.setStackTrace(""); } ignoreUntilEnds(reader, "test"); return caseResult; } + private String processForLoop(XMLStreamReader reader, int nestedCount) throws XMLStreamException { + StringBuilder stackTrace = new StringBuilder(); + String indentation = getSpacesPerNestedLevel(nestedCount); + stackTrace.append(indentation + "FOR " + reader.getAttributeValue(null, "flavor")); + while (reader.hasNext()) { + if (reader.isEndElement() && reader.getLocalName().equals("for")) { + break; + } + if (reader.isStartElement() && reader.getLocalName().equals("iter")) { + stackTrace.append(processIteration(reader, nestedCount+1)); + } + reader.next(); + } + stackTrace.append(indentation + "END\n"); + return stackTrace.toString(); + } + + private String processIteration(XMLStreamReader reader, int nestedCount) throws XMLStreamException { + StringBuilder stackTrace = new StringBuilder(); + while(reader.hasNext()) { + if (reader.isEndElement() && reader.getLocalName().equals("iter")) { + break; + } + if (reader.isStartElement()) { + String xmlTag = reader.getLocalName(); + if (xmlTag.equals("for")) { + stackTrace.append(processForLoop(reader, nestedCount)); + } + if (xmlTag.equals("kw")) { + stackTrace.append(processKeyword(reader, nestedCount)); + } + if (xmlTag.equals("if")) { + stackTrace.append(processIf(reader, nestedCount)); + } + } + reader.next(); + } + + return stackTrace.toString(); + } + + private String processIf(XMLStreamReader reader, int nestedCount) throws XMLStreamException { + StringBuilder stackTrace = new StringBuilder(); + String indentation = getSpacesPerNestedLevel(nestedCount); + while (reader.hasNext()) { + if (reader.isEndElement() && reader.getLocalName().equals("if")) { + break; + } + if (reader.isStartElement() && reader.getLocalName().equals("branch")) { + stackTrace.append(indentation + reader.getAttributeValue(null, "type")); + stackTrace.append(processBranch(reader, nestedCount+1)); + } + reader.next(); + } + stackTrace.append(indentation + "END\n"); + return stackTrace.toString(); + } + + private String processBranch(XMLStreamReader reader, int nestedCount) throws XMLStreamException { + StringBuilder stackTrace = new StringBuilder(); + while(reader.hasNext()) { + if (reader.isEndElement() && reader.getLocalName().equals("branch")) { + break; + } + if (reader.isStartElement()) { + String xmlTag = reader.getLocalName(); + if (xmlTag.equals("for")) { + stackTrace.append(processForLoop(reader, nestedCount)); + } + if (xmlTag.equals("kw")) { + stackTrace.append(processKeyword(reader, nestedCount)); + } + if (xmlTag.equals("if")) { + stackTrace.append(processIf(reader, nestedCount)); + } + } + reader.next(); + } + return stackTrace.toString(); + } + + private String processKeyword(XMLStreamReader reader, int nestedCount) throws XMLStreamException { + StringBuilder stackTrace = new StringBuilder(); + String kw = reader.getAttributeValue(null, "name"); + String indentation = getSpacesPerNestedLevel(nestedCount); + stackTrace.append(indentation).append(kw); + reader.next(); + while(reader.hasNext()) { + if (reader.isEndElement() && reader.getLocalName().equals("kw")) { + break; + } + if (reader.isStartElement()) { + String xmlTag = reader.getLocalName(); + switch (xmlTag) { + case "arguments": + case "arg": + stackTrace.append(processArgs(reader)); + continue; // processArgs returns with us already in . We don't want to use reader.next() + case "for": + stackTrace.append(processForLoop(reader, nestedCount+1)); + break; + case "kw": + stackTrace.append(processKeyword(reader, nestedCount+1)); + break; + case "if": + stackTrace.append(processIf(reader, nestedCount+1)); + break; + default: + break; + } + } + reader.next(); + } + stackTrace.append("\n"); + return stackTrace.toString(); + } + + private String processArgs(XMLStreamReader reader) throws XMLStreamException { + StringBuilder stringBuilder = new StringBuilder(); + + while(reader.hasNext()) { + if (reader.isEndElement() || reader.isStartElement()) { + String xmlTag = reader.getLocalName(); + if (reader.isEndElement() && xmlTag.equals("arguments") || + reader.isStartElement() && xmlTag.equals("status") || + reader.isStartElement() && xmlTag.equals("kw")) { + break; + } + if (reader.isStartElement() && xmlTag.equals("arg")) { + reader.next(); + stringBuilder.append(" ").append(reader.getText()); + } + } + reader.next(); + } + return stringBuilder.toString(); + } + private List processTags(XMLStreamReader reader) throws XMLStreamException { - List taglist = new ArrayList(); + List tagList = new ArrayList<>(); + while(reader.hasNext()){ - reader.next(); if(reader.isStartElement() && "tag".equals(reader.getLocalName())){ while(reader.hasNext()){ reader.next(); - if(reader.getEventType() == XMLStreamReader.CHARACTERS){ - taglist.add(reader.getText()); + if(reader.isCharacters()){ + tagList.add(reader.getText()); } else if(reader.isEndElement() && "tag".equals(reader.getLocalName())){ break; } } - } else if(reader.isEndElement() && "tags".equals(reader.getLocalName())){ + } else if((reader.isEndElement() && "tags".equals(reader.getLocalName())) || (reader.isStartElement() && "status".equals(reader.getLocalName()))){ break; } + reader.next(); } - return taglist; + return tagList; } private static void setCriticalityIfAvailable(XMLStreamReader reader, RobotCaseResult caseResult) { String criticality = reader.getAttributeValue(null, "critical"); if (criticality != null) { - if(criticality.equals("yes")) - caseResult.setCritical(true); - else - caseResult.setCritical(false); + caseResult.setCritical(criticality.equals("yes")); } } diff --git a/src/main/java/hudson/plugins/robot/RobotPublisher.java b/src/main/java/hudson/plugins/robot/RobotPublisher.java index 51b38015..52995d6e 100755 --- a/src/main/java/hudson/plugins/robot/RobotPublisher.java +++ b/src/main/java/hudson/plugins/robot/RobotPublisher.java @@ -233,7 +233,7 @@ public String getOverwriteXAxisLabel() { */ @Override public Collection getProjectActions(AbstractProject project) { - Collection actions = new ArrayList(); + Collection actions = new ArrayList<>(); RobotProjectAction roboAction = new RobotProjectAction(project); actions.add(roboAction); return actions; diff --git a/src/main/java/hudson/plugins/robot/blueocean/BlueRobotTestResult.java b/src/main/java/hudson/plugins/robot/blueocean/BlueRobotTestResult.java index 8cf4211c..e090b6c0 100644 --- a/src/main/java/hudson/plugins/robot/blueocean/BlueRobotTestResult.java +++ b/src/main/java/hudson/plugins/robot/blueocean/BlueRobotTestResult.java @@ -27,7 +27,9 @@ public String getName() { @Override public Status getStatus() { - return result.isPassed() ? Status.PASSED : Status.FAILED; + if (result.isPassed()) return Status.PASSED; + if (result.isSkipped()) return Status.SKIPPED; + return Status.FAILED; } @Override diff --git a/src/main/java/hudson/plugins/robot/graph/RobotGraphHelper.java b/src/main/java/hudson/plugins/robot/graph/RobotGraphHelper.java index 3f9a5cb3..f502bfac 100644 --- a/src/main/java/hudson/plugins/robot/graph/RobotGraphHelper.java +++ b/src/main/java/hudson/plugins/robot/graph/RobotGraphHelper.java @@ -54,9 +54,9 @@ public static RobotGraph createTestResultsGraphForTestObject(RobotTestObject roo boolean criticalOnly, String labelFormat, int maxBuildsToShow) { - List values = new ArrayList(); - List rows = new ArrayList(); - List columns = new ArrayList(); + List values = new ArrayList<>(); + List rows = new ArrayList<>(); + List columns = new ArrayList<>(); double lowerbound = 0; double upperbound = 0; @@ -67,11 +67,13 @@ public static RobotGraph createTestResultsGraphForTestObject(RobotTestObject roo { Number failed = !criticalOnly ? testObject.getFailed() : testObject.getCriticalFailed(); Number passed = 0; + Number skipped = 0; int compareLowerBoundTo; if ( failedOnly) { compareLowerBoundTo = failed.intValue(); } else { passed = !criticalOnly ? testObject.getPassed() : testObject.getCriticalPassed(); + skipped = testObject.getSkipped(); compareLowerBoundTo = passed.intValue(); } @@ -92,6 +94,10 @@ public static RobotGraph createTestResultsGraphForTestObject(RobotTestObject roo values.add(failed); rows.add(Messages.robot_trendgraph_failed()); columns.add(label); + + values.add(skipped); + rows.add(Messages.robot_trendgraph_skipped()); + columns.add(label); } if(significantData){ @@ -100,7 +106,7 @@ public static RobotGraph createTestResultsGraphForTestObject(RobotTestObject roo } int graphScale = hd ? 3 : 1; return RobotGraph.getRobotGraph(rootObject.getOwner(), createSortedDataset(values, rows, columns), Messages.robot_trendgraph_testcases(), - Messages.robot_trendgraph_builds(), graphScale, false, binarydata, lowerbound, upperbound, Color.green, Color.red); + Messages.robot_trendgraph_builds(), graphScale, false, binarydata, lowerbound, upperbound, Color.orange, Color.green, Color.red); } /** @@ -118,8 +124,8 @@ public static RobotGraph createDurationGraphForTestObject(RobotTestObject rootOb int scale = 1; int buildsLeftToShow = maxBuildsToShow > 0? maxBuildsToShow: -1; - List labels = new ArrayList(); - List durations = new ArrayList(); + List labels = new ArrayList<>(); + List durations = new ArrayList<>(); for (RobotTestObject testObject = rootObject; testObject != null && buildsLeftToShow != 0; @@ -144,8 +150,8 @@ private static CategoryDataset createSortedDataset(List values, List rowSet = new TreeSet(rows); - TreeSet colSet = new TreeSet(columns); + TreeSet rowSet = new TreeSet<>(rows); + TreeSet colSet = new TreeSet<>(columns); Comparable[] _rows = rowSet.toArray(new Comparable[rowSet.size()]); Comparable[] _cols = colSet.toArray(new Comparable[colSet.size()]); diff --git a/src/main/java/hudson/plugins/robot/model/RobotCaseResult.java b/src/main/java/hudson/plugins/robot/model/RobotCaseResult.java index eb70b5f1..493d5d91 100644 --- a/src/main/java/hudson/plugins/robot/model/RobotCaseResult.java +++ b/src/main/java/hudson/plugins/robot/model/RobotCaseResult.java @@ -40,6 +40,7 @@ public class RobotCaseResult extends RobotTestObject{ private static final Logger LOGGER = Logger.getLogger(RobotCaseResult.class.getName()); private boolean passed; + private boolean skipped; private boolean critical; private String errorMsg; private String name; @@ -156,6 +157,10 @@ public void setPassed(boolean passed) { this.passed = passed; } + public void setSkipped(boolean skipped) { + this.skipped = skipped; + } + public void setCritical(boolean critical) { this.critical = critical; } @@ -172,13 +177,17 @@ public boolean isPassed() { return passed; } + public boolean isSkipped() { + return skipped; + } + public boolean isCritical() { return critical; } public List getTags(){ if(tags == null) - return new ArrayList(); + return new ArrayList<>(); return tags; } @@ -187,7 +196,7 @@ public String getCommaSeparatedTags(){ if (tags.size()==0) return ""; else { - StringBuffer buf = new StringBuffer(); + StringBuilder buf = new StringBuilder(); for (String tag: tags) buf.append(tag+", "); String result = buf.toString(); @@ -197,7 +206,7 @@ public String getCommaSeparatedTags(){ public void addTags(List taglist){ if(tags == null) - tags = new ArrayList(); + tags = new ArrayList<>(); tags.addAll(taglist); } @@ -206,7 +215,7 @@ public void addTags(List taglist){ * @return number of build */ public int getFailedSince(){ - if (failedSince == 0 && !isPassed()) { + if (failedSince == 0 && (!isPassed() && !isSkipped())) { RobotCaseResult previous = getPreviousResult(); if(previous != null && !previous.isPassed()) this.failedSince = previous.getFailedSince(); @@ -276,7 +285,7 @@ public void doGraph(StaplerRequest req, StaplerResponse rsp) @Override public int getFailed() { - if(isPassed()) return 0; + if(isPassed() || isSkipped()) return 0; return 1; } @@ -286,9 +295,15 @@ public int getPassed() { return 0; } + @Override + public int getSkipped() { + if (isSkipped()) return 1; + return 0; + } + @Override public long getCriticalFailed() { - if(!isPassed() && isCritical()) return 1; + if((!isPassed() && !isSkipped()) && isCritical()) return 1; return 0; } diff --git a/src/main/java/hudson/plugins/robot/model/RobotResult.java b/src/main/java/hudson/plugins/robot/model/RobotResult.java index a5ea690a..b013faf4 100755 --- a/src/main/java/hudson/plugins/robot/model/RobotResult.java +++ b/src/main/java/hudson/plugins/robot/model/RobotResult.java @@ -50,7 +50,7 @@ public class RobotResult extends RobotTestObject { private String timeStamp; - private transient int passed, failed, criticalPassed, criticalFailed; + private transient int passed, failed, skipped, criticalPassed, criticalFailed; //backwards compatibility with old builds private transient List overallStats; @@ -136,13 +136,24 @@ public long getOverallFailed(){ return overallStats.get(1).getFail(); } + /** + * Get number of all skipped tests. + * @return number of all skipped tests + */ + @Exported + public long getOverallSkipped(){ + if(overallStats == null) return skipped; + if(overallStats.isEmpty()) return 0; + return overallStats.get(1).getSkip(); + } + /** * Get number of all tests. * @return number of all tests */ @Exported public long getOverallTotal(){ - if(overallStats == null) return failed + passed; + if(overallStats == null) return failed + passed + skipped; if(overallStats.isEmpty()) return 0; return overallStats.get(1).getTotal(); } @@ -188,7 +199,8 @@ public double getPassPercentage(boolean onlyCritical) { total = getCriticalTotal(); } else { passed = getOverallPassed(); - total = getOverallTotal(); + // Skipped tests don't count towards pass percentage + total = getOverallTotal() - getOverallSkipped(); } if(total == 0) return 100; @@ -202,6 +214,12 @@ public double getPassPercentage(){ return getPassPercentage(false); } + @Exported + public double getSkipPercentage() { + double percentage = (double) getSkipped() / getOverallTotal() * 100; + return roundToDecimals(percentage, 1); + } + private static double roundToDecimals(double value, int decimals){ BigDecimal bd = new BigDecimal(Double.toString(value)); bd = bd.setScale(decimals, BigDecimal.ROUND_DOWN); @@ -238,7 +256,7 @@ public RobotSuiteResult getSuite(String name) { */ public void addSuite(RobotSuiteResult suite){ if(suites == null) - this.suites = new HashMap(); + this.suites = new HashMap<>(); int i = 1; String originalName = suite.getName(); String checkedSuiteName = originalName; @@ -262,10 +280,10 @@ public Collection getSuites(){ @Exported public List getExecutedSuites() { - List executedSuites = new ArrayList(); + List executedSuites = new ArrayList<>(); for (RobotSuiteResult robotSuiteResult : this.getAllSuites()) { RobotTestObject rto = robotSuiteResult.getParent(); - String name = robotSuiteResult.getName(); + String name = robotSuiteResult.getName(); while (rto != null && !rto.getName().isEmpty()) { name = rto.getName()+"."+name; rto = rto.getParent(); @@ -281,7 +299,7 @@ public List getExecutedSuites() { * @return List of suiteresults */ public List getAllSuites(){ - List allSuites = new ArrayList(); + List allSuites = new ArrayList<>(); for(RobotSuiteResult suite : getSuites()){ allSuites.add(suite); List childSuites = suite.getAllChildSuites(); @@ -295,7 +313,7 @@ public List getAllSuites(){ * @return list of test case results */ public List getAllFailedCases(){ - List allFailedCases = new ArrayList(); + List allFailedCases = new ArrayList<>(); for(RobotSuiteResult suite : getSuites()){ List failedCases = suite.getAllFailedCases(); allFailedCases.addAll(failedCases); @@ -306,7 +324,7 @@ public List getAllFailedCases(){ @Exported public List getFailedCases() { - List failedCases = new ArrayList(); + List failedCases = new ArrayList<>(); for (RobotCaseResult robotCaseResult : this.getAllFailedCases()) { RobotTestObject rto = robotCaseResult.getParent(); String name = robotCaseResult.getName(); @@ -327,17 +345,19 @@ public void tally(RobotBuildAction robotBuildAction){ setParentAction(robotBuildAction); failed = 0; passed = 0; + skipped = 0; criticalPassed = 0; criticalFailed = 0; duration = 0; Collection newSuites = getSuites(); - HashMap newMap = new HashMap(newSuites.size()); + HashMap newMap = new HashMap<>(newSuites.size()); for (RobotSuiteResult suite : newSuites) { suite.tally(robotBuildAction); failed += suite.getFailed(); passed += suite.getPassed(); + skipped += suite.getSkipped(); criticalFailed += suite.getCriticalFailed(); criticalPassed += suite.getCriticalPassed(); duration += suite.getDuration(); @@ -425,6 +445,11 @@ public int getPassed() { return (int) getOverallPassed(); } + @Override + public int getSkipped() { + return (int)getOverallSkipped(); + } + public Api getApi() { return new Api(this); } public List getAllCases() { diff --git a/src/main/java/hudson/plugins/robot/model/RobotResultStatistics.java b/src/main/java/hudson/plugins/robot/model/RobotResultStatistics.java index 08b3b044..43778158 100755 --- a/src/main/java/hudson/plugins/robot/model/RobotResultStatistics.java +++ b/src/main/java/hudson/plugins/robot/model/RobotResultStatistics.java @@ -27,6 +27,7 @@ public class RobotResultStatistics implements Serializable { private long pass; private long fail; + private long skip; //backwards compatibility with old builds private transient String name; @@ -47,7 +48,15 @@ public void setFail(long fail) { this.fail = fail; } + public long getSkip() { + return skip; + } + + public void setSkip(long skip) { + this.skip = skip; + } + public long getTotal(){ - return fail + pass; + return fail + pass + skip; } } diff --git a/src/main/java/hudson/plugins/robot/model/RobotSuiteResult.java b/src/main/java/hudson/plugins/robot/model/RobotSuiteResult.java index 9664ae12..7cd17a8f 100644 --- a/src/main/java/hudson/plugins/robot/model/RobotSuiteResult.java +++ b/src/main/java/hudson/plugins/robot/model/RobotSuiteResult.java @@ -45,6 +45,7 @@ public class RobotSuiteResult extends RobotTestObject { private String endTime; private transient int failed; private transient int passed; + private transient int skipped; private transient int criticalPassed; private transient int criticalFailed; @@ -56,7 +57,7 @@ public class RobotSuiteResult extends RobotTestObject { */ public void addChild(RobotSuiteResult child) { if(children == null) - this.children = new HashMap(); + this.children = new HashMap<>(); int i = 1; String originalName = child.getName(); String checkedSuiteName = originalName; @@ -143,12 +144,16 @@ public int getPassed() { return passed; } + public int getSkipped() { + return skipped; + } + /** * Get number of all tests * @return number of all tests */ public int getTotal() { - return passed + failed; + return passed + failed + skipped; } /** @@ -182,7 +187,7 @@ public int getCriticalTotal() { */ public void addCaseResult(RobotCaseResult caseResult) { if(caseResults == null) - this.caseResults = new HashMap(); + this.caseResults = new HashMap<>(); int i = 1; String originalName = caseResult.getName(); String checkedTestName = originalName; @@ -294,7 +299,7 @@ public Object getDynamic(String token, StaplerRequest req, * @return all children of this suite */ public List getAllChildSuites() { - List allChildSuites = new ArrayList(); + List allChildSuites = new ArrayList<>(); for (RobotSuiteResult suite : getChildSuites()) { allChildSuites.add(suite); List childSuites = suite.getAllChildSuites(); @@ -309,9 +314,9 @@ public List getAllChildSuites() { * @return all failed cases in this suite and its child suites */ public List getAllFailedCases() { - List failedCases = new ArrayList(); + List failedCases = new ArrayList<>(); for(RobotCaseResult caseResult : getCaseResults()){ - if(!caseResult.isPassed()) failedCases.add(caseResult); + if(!caseResult.isPassed() && !caseResult.isSkipped()) failedCases.add(caseResult); } for(RobotSuiteResult suite : getChildSuites()){ failedCases.addAll(suite.getAllFailedCases()); @@ -325,7 +330,7 @@ public List getAllFailedCases() { * @return all cases in this suite and its child suites */ public List getAllCases() { - List cases = new ArrayList(); + List cases = new ArrayList<>(); cases.addAll(getCaseResults()); for(RobotSuiteResult suite : getChildSuites()){ cases.addAll(suite.getAllCases()); @@ -349,15 +354,18 @@ public void tally(RobotBuildAction parentAction) { setParentAction(parentAction); failed = 0; passed = 0; + skipped = 0; criticalPassed = 0; criticalFailed = 0; duration = 0; - HashMap newCases = new HashMap(); + HashMap newCases = new HashMap<>(); for(RobotCaseResult caseResult : getCaseResults()) { if(caseResult.isPassed()) { if(caseResult.isCritical()) criticalPassed++; passed++; + } else if(caseResult.isSkipped()) { + skipped++; } else { if(caseResult.isCritical()) criticalFailed++; failed++; @@ -368,11 +376,12 @@ public void tally(RobotBuildAction parentAction) { } caseResults = newCases; - HashMap newSuites = new HashMap(); + HashMap newSuites = new HashMap<>(); for (RobotSuiteResult suite : getChildSuites()) { suite.tally(parentAction); failed += suite.getFailed(); passed += suite.getPassed(); + skipped += suite.getSkipped(); criticalFailed += suite.getCriticalFailed(); criticalPassed += suite.getCriticalPassed(); duration += suite.getDuration(); @@ -413,9 +422,7 @@ public void addChildren(Collection childSuites) { */ public void addCaseResults(Collection newCaseResults) { for(RobotCaseResult caseResult : newCaseResults){ - if(caseResults.get(caseResult.getDuplicateSafeName()) == null){ - caseResults.put(caseResult.getDuplicateSafeName(), caseResult); - } + caseResults.putIfAbsent(caseResult.getDuplicateSafeName(), caseResult); } } } diff --git a/src/main/java/hudson/plugins/robot/model/RobotTestObject.java b/src/main/java/hudson/plugins/robot/model/RobotTestObject.java index 28270ca9..f0754ffc 100644 --- a/src/main/java/hudson/plugins/robot/model/RobotTestObject.java +++ b/src/main/java/hudson/plugins/robot/model/RobotTestObject.java @@ -240,6 +240,8 @@ public String getDurationDiff(RobotTestObject comparable){ public abstract int getPassed(); + public abstract int getSkipped(); + public abstract long getCriticalPassed(); public abstract long getCriticalFailed(); diff --git a/src/main/java/hudson/plugins/robot/view/RobotListViewColumn.java b/src/main/java/hudson/plugins/robot/view/RobotListViewColumn.java index e255754a..1e494676 100644 --- a/src/main/java/hudson/plugins/robot/view/RobotListViewColumn.java +++ b/src/main/java/hudson/plugins/robot/view/RobotListViewColumn.java @@ -27,30 +27,33 @@ public String getColumnCaption() { public long getPass(Item job){ RobotResult lastRobotResult = getLastRobotResult(job); - if(lastRobotResult != null){ - return lastRobotResult.getOverallPassed(); - } - return 0; + return lastRobotResult != null ? lastRobotResult.getOverallPassed() : 0; } public long getTotal(Item job){ RobotResult lastRobotResult = getLastRobotResult(job); - if(lastRobotResult != null){ - return lastRobotResult.getOverallTotal(); - } - return 0; + return lastRobotResult != null ? lastRobotResult.getOverallTotal() : 0; + } + + public long getSkipped(Item job) { + RobotResult lastRobotResult = getLastRobotResult(job); + return lastRobotResult != null ? lastRobotResult.getOverallSkipped() : 0; } public double getPassPercent(Item job) { RobotResult lastRobotResult = getLastRobotResult(job); - if (lastRobotResult==null) return 100; - return lastRobotResult.getPassPercentage(); + return lastRobotResult != null ? lastRobotResult.getPassPercentage() : 100; + } + + public double getSkipPercent(Item job) { + RobotResult lastRobotResult = getLastRobotResult(job); + return lastRobotResult != null ? lastRobotResult.getSkipPercentage() : 0; } public String getRobotPath(Item job) { if (job instanceof Job) { Run build = ((Job)job).getLastCompletedBuild(); - int lastBuildNr = build==null? 1 : build.number; + int lastBuildNr = build == null ? 1 : build.number; return job.getShortUrl() + lastBuildNr+ "/robot/"; } return null; diff --git a/src/main/resources/hudson/plugins/robot/Messages.properties b/src/main/resources/hudson/plugins/robot/Messages.properties index f36f9579..78564906 100644 --- a/src/main/resources/hudson/plugins/robot/Messages.properties +++ b/src/main/resources/hudson/plugins/robot/Messages.properties @@ -27,5 +27,6 @@ robot.config.percentvalidation=Entry must be percentage value between 0-100 robot.trendgraph.passed=Passed robot.trendgraph.failed=Failed +robot.trendgraph.skipped=Skipped robot.trendgraph.testcases=Number of test cases robot.trendgraph.builds=Build diff --git a/src/main/resources/hudson/plugins/robot/model/RobotCaseResult/index.jelly b/src/main/resources/hudson/plugins/robot/model/RobotCaseResult/index.jelly index 8bb14ff7..aa7ec510 100644 --- a/src/main/resources/hudson/plugins/robot/model/RobotCaseResult/index.jelly +++ b/src/main/resources/hudson/plugins/robot/model/RobotCaseResult/index.jelly @@ -25,6 +25,9 @@ limitations under the License. + + + diff --git a/src/main/resources/hudson/plugins/robot/model/RobotResult/index.jelly b/src/main/resources/hudson/plugins/robot/model/RobotResult/index.jelly index b8c30978..f501a9b2 100644 --- a/src/main/resources/hudson/plugins/robot/model/RobotResult/index.jelly +++ b/src/main/resources/hudson/plugins/robot/model/RobotResult/index.jelly @@ -43,8 +43,8 @@ limitations under the License. Executed:${it.timeStamp} Duration:${it.humanReadableDuration} (${it.getDurationDiff(prevresult)}) - Status:${it.criticalTotal} critical test, ${it.criticalPassed} passed, ${it.criticalFailed} failed
- ${it.overallTotal} test total (${h.getDiffString(it.total - prevresult.total)}), ${it.passed} passed, ${it.failed} failed + Status:${it.criticalTotal} critical test, ${it.criticalPassed} passed, ${it.criticalFailed} failed, 0 skipped
+ ${it.overallTotal} test total (${h.getDiffString(it.total - prevresult.total)}), ${it.passed} passed, ${it.failed} failed, ${it.skipped} skipped Results: ${it.reportFile}
 ${it.logFile}
 diff --git a/src/main/resources/hudson/plugins/robot/model/RobotSuiteResult/index.jelly b/src/main/resources/hudson/plugins/robot/model/RobotSuiteResult/index.jelly index 25981668..785422c9 100644 --- a/src/main/resources/hudson/plugins/robot/model/RobotSuiteResult/index.jelly +++ b/src/main/resources/hudson/plugins/robot/model/RobotSuiteResult/index.jelly @@ -43,8 +43,8 @@ limitations under the License. TEST SUITE:${it.name} Description:${it.description} Duration:${it.humanReadableDuration} (${it.getDurationDiff(prevresult)}) - Status:${it.criticalTotal} critical test, ${it.criticalPassed} passed, ${it.criticalFailed} failed
- ${it.total} test total (${h.getDiffString(it.total - prevresult.total)}), ${it.passed} passed, ${it.failed} failed + Status:${it.criticalTotal} critical test, ${it.criticalPassed} passed, ${it.criticalFailed} failed, 0 skipped
+ ${it.total} test total (${h.getDiffString(it.total - prevresult.total)}), ${it.passed} passed, ${it.failed} failed, ${it.skipped} skipped Report File:${it.reportFile} @@ -92,6 +92,9 @@ limitations under the License. + + + @@ -100,7 +103,7 @@ limitations under the License. ${case.name} yesno - PASSFAIL + ${status} ${case.humanReadableDuration}(${case.getDurationDiff(prevcase)}) diff --git a/src/main/resources/hudson/plugins/robot/model/RobotTestObject/robotcss.jelly b/src/main/resources/hudson/plugins/robot/model/RobotTestObject/robotcss.jelly index 2fd1bbe2..8be77ea3 100644 --- a/src/main/resources/hudson/plugins/robot/model/RobotTestObject/robotcss.jelly +++ b/src/main/resources/hudson/plugins/robot/model/RobotTestObject/robotcss.jelly @@ -31,6 +31,9 @@ div.passfail p { .FAIL { color: #c00; } +.SKIP { + color: #fc0; +} span.FAIL { font-weight: bold; } diff --git a/src/main/resources/hudson/plugins/robot/util/robotsummary.jelly b/src/main/resources/hudson/plugins/robot/util/robotsummary.jelly index 02bcaed3..c0439ca2 100644 --- a/src/main/resources/hudson/plugins/robot/util/robotsummary.jelly +++ b/src/main/resources/hudson/plugins/robot/util/robotsummary.jelly @@ -48,6 +48,7 @@ limitations under the License. Total Failed Passed + Skipped Pass % @@ -61,6 +62,7 @@ limitations under the License. ${attrs.action.result.criticalFailed} ${attrs.action.result.criticalPassed} + 0 ${attrs.action.criticalPassPercentage} @@ -74,6 +76,7 @@ limitations under the License. ${attrs.action.result.overallFailed} ${attrs.action.result.overallPassed} + ${attrs.action.result.overallSkipped} ${attrs.action.overallPassPercentage} diff --git a/src/main/resources/hudson/plugins/robot/view/RobotListViewColumn/column.jelly b/src/main/resources/hudson/plugins/robot/view/RobotListViewColumn/column.jelly index af4d0300..8463c3bf 100644 --- a/src/main/resources/hudson/plugins/robot/view/RobotListViewColumn/column.jelly +++ b/src/main/resources/hudson/plugins/robot/view/RobotListViewColumn/column.jelly @@ -24,13 +24,32 @@ THE SOFTWARE. --> + diff --git a/src/test/java/hudson/plugins/robot/RobotParserTest.java b/src/test/java/hudson/plugins/robot/RobotParserTest.java index 36f5c743..5c358caa 100644 --- a/src/test/java/hudson/plugins/robot/RobotParserTest.java +++ b/src/test/java/hudson/plugins/robot/RobotParserTest.java @@ -163,4 +163,26 @@ public void testNested2() { final String mask = "nested_output2.xml"; parse(dir, mask); } + + @Test + public void testRobot4() { + final String dir = "."; + final String mask = "robot4_output.xml"; + parse(dir, mask); + } + + @Test + public void testRobot4Nested() { + final String dir = "."; + final String mask = "robot4_nested_output.xml"; + parse(dir, mask); + } + + @Test + public void testRobot4If() { + final String dir = "."; + final String mask = "robot4_if_output.xml"; + parse(dir, mask); + + } } diff --git a/src/test/java/hudson/plugins/robot/RobotPublisherSystemTest.java b/src/test/java/hudson/plugins/robot/RobotPublisherSystemTest.java index af983b6b..3e8fd168 100644 --- a/src/test/java/hudson/plugins/robot/RobotPublisherSystemTest.java +++ b/src/test/java/hudson/plugins/robot/RobotPublisherSystemTest.java @@ -193,7 +193,7 @@ public void testOldActionViewsWithData() throws Exception { WebAssert.assertElementPresentByXPath(page, "//div[@id='main-panel']//img[@id='passfailgraph']"); WebAssert.assertElementPresentByXPath(page, "//div[@id='main-panel']//a[@href='/jenkins/job/oldrobotbuild/1/robot' and contains(text(),'Browse results')]"); - this.verifyTotalsTable(page, 8, 4, "50.0", 8, 4, "50.0"); + this.verifyTotalsTable(page, 8, 4, 0, "50.0", 8, 4, "50.0"); page = wc.goTo("job/oldrobotbuild/robot/"); WebAssert.assertTitleEquals(page, "Testcases & Othercases Test Report"); @@ -204,7 +204,7 @@ public void testOldActionViewsWithData() throws Exception { WebAssert.assertElementPresentByXPath(page, "//div[@id='main-panel']//h4[contains(.,'Robot Test Summary:')]"); WebAssert.assertElementPresentByXPath(page, "//div[@id='main-panel']//a[@href='/jenkins/job/oldrobotbuild/1/robot' and contains(text(),'Browse results')]"); - this.verifyTotalsTable(page, 8, 4, "50.0", 8, 4, "50.0"); + this.verifyTotalsTable(page, 8, 4, 0,"50.0", 8, 4, "50.0"); page = wc.goTo("job/oldrobotbuild/1/robot/"); WebAssert.assertTitleEquals(page, "Testcases & Othercases Test Report"); @@ -225,7 +225,7 @@ public void testSummariesWithData() throws Exception { "//div[@id='main-panel']//a[@href='/jenkins/job/robot/1/robot/report/report.html' and contains(text(), 'Open report.html')]"); WebAssert.assertElementPresentByXPath(page, "//div[@id='main-panel']//a[@href='/jenkins/job/robot/1/robot/report/log.html' and contains(text(), 'Open log.html')]"); - verifyTotalsTable(page, 8, 4, "50.0", 8, 4, "50.0"); + verifyTotalsTable(page, 8, 4, 0,"50.0", 8, 4, "50.0"); page = wc.goTo("job/robot/1/"); WebAssert.assertElementPresentByXPath(page, "//div[@id='tasks']//a[@href='/jenkins/job/robot/1/robot']"); @@ -236,7 +236,7 @@ public void testSummariesWithData() throws Exception { "//div[@id='main-panel']//a[@href='/jenkins/job/robot/1/robot/report/report.html' and contains(text(), 'Open report.html')]"); WebAssert.assertElementPresentByXPath(page, "//div[@id='main-panel']//a[@href='/jenkins/job/robot/1/robot/report/log.html' and contains(text(), 'Open log.html')]"); - verifyTotalsTable(page, 8, 4, "50.0", 8, 4, "50.0"); + verifyTotalsTable(page, 8, 4, 0,"50.0", 8, 4, "50.0"); } @LocalData @@ -254,7 +254,7 @@ public void testSummariesWithVariablesInFileNames() throws Exception { "//div[@id='main-panel']//a[@href='/jenkins/job/files-with-env-vars/1/robot/report/report_1.html' and contains(text(), 'Open report_1.html')]"); WebAssert.assertElementPresentByXPath(page, "//div[@id='main-panel']//a[@href='/jenkins/job/files-with-env-vars/1/robot/report/log_1.html' and contains(text(), 'Open log_1.html')]"); - verifyTotalsTable(page, 6, 1, "83.3", 6, 1, "83.3"); + verifyTotalsTable(page, 6, 1, 0,"83.3", 6, 1, "83.3"); page = wc.goTo("job/files-with-env-vars/1/"); WebAssert.assertElementPresentByXPath(page, "//div[@id='tasks']//a[@href='/jenkins/job/files-with-env-vars/1/robot']"); @@ -265,7 +265,7 @@ public void testSummariesWithVariablesInFileNames() throws Exception { "//div[@id='main-panel']//a[@href='/jenkins/job/files-with-env-vars/1/robot/report/report_1.html' and contains(text(), 'Open report_1.html')]"); WebAssert.assertElementPresentByXPath(page, "//div[@id='main-panel']//a[@href='/jenkins/job/files-with-env-vars/1/robot/report/log_1.html' and contains(text(), 'Open log_1.html')]"); - verifyTotalsTable(page, 6, 1, "83.3", 6, 1, "83.3"); + verifyTotalsTable(page, 6, 1, 0,"83.3", 6, 1, "83.3"); } @LocalData @@ -273,7 +273,7 @@ public void testSummariesWithVariablesInFileNames() throws Exception { public void testRobot29Outputs() throws Exception { WebClient wc = this.executeJobAndGetWebClient("robot29output"); HtmlPage page = wc.goTo("job/robot29output/"); - verifyTotalsTable(page, 1, 0, "100.0", 1, 0, "100.0"); + verifyTotalsTable(page, 1, 0, 0,"100.0", 1, 0, "100.0"); } @LocalData @@ -292,7 +292,7 @@ public void testCombinedOutputs() throws Exception { "//div[@id='main-panel']//a[@href='/jenkins/job/several-outputs/1/robot/report/**/report.html' and contains(text(), 'Open **/report.html')]"); WebAssert.assertElementPresentByXPath(page, "//div[@id='main-panel']//a[@href='/jenkins/job/several-outputs/1/robot/report/**/log.html' and contains(text(), 'Open **/log.html')]"); - verifyTotalsTable(page, 2, 0, "100.0", 2, 0, "100.0"); + verifyTotalsTable(page, 2, 0, 0,"100.0", 2, 0, "100.0"); } @LocalData @@ -472,18 +472,20 @@ private Project getProject(String projectName) { return testProject; } - private void verifyTotalsTable(HtmlPage page, int totalTests, int totalFailed, String totalPercents, + private void verifyTotalsTable(HtmlPage page, int totalTests, int totalFailed, int totalSkipped, String totalPercents, int totalCritical, int criticalFailed, String criticalPercents) { HtmlTable table = page.getHtmlElementById("robot-summary-table"); - String expectedTable = "TotalFailedPassedPass%Criticaltests" - + totalCritical + "" + criticalFailed - + "" + (totalCritical - totalFailed) - + "" + criticalPercents - + "Alltests" + totalTests - + "" + totalFailed - + "" + (totalTests - totalFailed) + "" + totalPercents - + ""; + String expectedTable = "TotalFailedPassedSkippedPass%Criticaltests" + + "" + totalCritical + "" + + "" + criticalFailed + "" + + "" + (totalCritical - totalFailed) + "" + + "0" + + "" + criticalPercents + "" + + "Alltests" + totalTests + "" + + "" + totalFailed + "" + + "" + (totalTests - totalFailed) + "" + + "" + totalSkipped + "" + + "" + totalPercents + ""; assertTrue(table.asXml().replaceAll("\\s", "").contains(expectedTable)); } diff --git a/src/test/java/hudson/plugins/robot/blueocean/BlueRobotTestResultForRobot4Test.java b/src/test/java/hudson/plugins/robot/blueocean/BlueRobotTestResultForRobot4Test.java new file mode 100644 index 00000000..3d678f99 --- /dev/null +++ b/src/test/java/hudson/plugins/robot/blueocean/BlueRobotTestResultForRobot4Test.java @@ -0,0 +1,104 @@ +package hudson.plugins.robot.blueocean; + +import static org.junit.Assert.assertEquals; +import hudson.model.FreeStyleBuild; +import hudson.plugins.robot.RobotBuildAction; +import hudson.plugins.robot.RobotParser; +import hudson.plugins.robot.model.RobotResult; +import io.jenkins.blueocean.rest.Reachable; +import io.jenkins.blueocean.rest.factory.BlueTestResultFactory.Result; +import io.jenkins.blueocean.rest.hal.Link; +import io.jenkins.blueocean.rest.model.BlueTestResult; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +public class BlueRobotTestResultForRobot4Test { + private RobotResult result; + private FreeStyleBuild mockBuild; + private RobotBuildAction mockAction; + private Reachable mockReachable; + + @Before + public void setUp() throws Exception { + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("blue_skip.xml", null, null); + result = remoteOperation.invoke(new File(BlueRobotTestResultTest.class.getResource("blue_skip.xml").toURI()).getParentFile(), null); + result.tally(null); + + mockBuild = mock(FreeStyleBuild.class); + mockAction = mock(RobotBuildAction.class); + mockReachable = mock(Reachable.class); + doReturn(mockAction).when(mockBuild).getAction(RobotBuildAction.class); + doReturn(result.getAllCases()).when(mockAction).getAllTests(); + doReturn(new Link("/")).when(mockReachable).getLink(); + } + + @Test + public void testSimpleStacktrace(){ + BlueTestResult result = getResult("Test 8 Will Always Fail"); + assertEquals("Fail", result.getErrorStackTrace()); + } + + @Test + public void testStacktraceWithArguments(){ + BlueTestResult result = getResult("Test 9 Will Always Fail"); + assertEquals("Fail Optional failure message", result.getErrorStackTrace()); + } + + @Test + public void testStacktraceIsEmpty(){ + BlueTestResult result; + for(String testCase : Arrays.asList("Test 1 Will Always Pass", + "Test 2 Will Always Pass", + "Test 3 Will Always Pass", + "Test 4 Will Always Pass But Is Skipped", + "Test 5 Will Always Fail But Is Skipped", + "Test 6 Will Always Fail But Is Skipped", + "Test 7 Will Always Fail But Is Skipped", + "Test 10 Will Always Fail But Is Skipped On Failure", + "Test 11 Will Always Fail But Is Skipped On Failure", + "Test 12 Will Always Pass But Is Skipped On Failure")){ + result = getResult(testCase); + assertEquals("", result.getErrorStackTrace()); + } + } + + @Test + public void testForLoopStackTrace() throws Exception { + BlueTestResult result = getResult("For Loop Failure"); + String helper = "FOR IN RANGE\n Log ${x}\n Nested Keyword ${x}\n Log ${arg}\n Run Keyword If ${x}==1 Fail\n" + + " Log ${x}\n Nested Keyword ${x}\n Log ${arg}\n Run Keyword If ${x}==1 Fail\n Fail\nEND"; + assertEquals(helper, result.getErrorStackTrace()); + } + + @Test + public void testIfElseStackTrace() throws Exception { + BlueTestResult result = getResult("If Else Failure"); + String helper = "Set Variable mikki hiiri\nIF\n Fail\nELSE\n Nested Keyword ${var}\n Log ${arg}\n Fail ${var}\nEND"; + assertEquals(helper, result.getErrorStackTrace()); + } + + @Test + public void testIfFailure() throws Exception { + BlueTestResult result = getResult("If Failure"); + String helper = "Set Variable mikki hiiri\nIF\n Nested Keyword ${var}\n Log ${arg}\n Fail\nEND"; + assertEquals(helper, result.getErrorStackTrace()); + } + + private BlueTestResult getResult(String filterCondition){ + BlueRobotTestResult.FactoryImpl factory = new BlueRobotTestResult.FactoryImpl(); + Result blueResult = factory.getBlueTestResults(mockBuild, mockReachable); + return StreamSupport.stream(blueResult.results.spliterator(), false) + .filter(element -> element.getName().equals(filterCondition)) + .collect(Collectors.toList()).get(0); + } + + +} diff --git a/src/test/java/hudson/plugins/robot/blueocean/BlueRobotTestResultTest.java b/src/test/java/hudson/plugins/robot/blueocean/BlueRobotTestResultTest.java index 49860464..8fb9af88 100644 --- a/src/test/java/hudson/plugins/robot/blueocean/BlueRobotTestResultTest.java +++ b/src/test/java/hudson/plugins/robot/blueocean/BlueRobotTestResultTest.java @@ -5,6 +5,8 @@ import static org.mockito.Mockito.mock; import java.io.File; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.junit.Before; import org.junit.Test; @@ -38,39 +40,46 @@ public void setUp() throws Exception { doReturn(result.getAllCases()).when(mockAction).getAllTests(); doReturn(new Link("/")).when(mockReachable).getLink(); } - + + @Test + public void testSimpleTrace() { + BlueTestResult result = getResult("Failed Test"); + assertEquals("Fail This fails!", result.getErrorStackTrace()); + assertEquals("This fails!", result.getErrorDetails()); + } + + @Test + public void testNestedTrace(){ + BlueTestResult result = getResult("Nested failed test"); + assertEquals("My failed Keyword\n The real failed keyword\n Fail Really fails!", result.getErrorStackTrace()); + assertEquals("Really fails!", result.getErrorDetails()); + } + @Test - public void testBasic() { + public void testNestedNotFirst() { + BlueTestResult result = getResult("Nested with not first"); + String helper = "Should Be Equal ${MESSAGE} Hello, world!\nShould Be Equal ${MESSAGE} Hello, world!\n" + + "My failed Keyword\n The real failed keyword\n Fail Really fails!"; + assertEquals(helper, result.getErrorStackTrace()); + assertEquals("Really fails!", result.getErrorDetails()); + } + + @Test + public void testEmptyMessage(){ + BlueTestResult result1 = getResult("Another Test"); + BlueTestResult result2 = getResult("My Test"); + + assertEquals("", result1.getErrorStackTrace()); + assertEquals("", result1.getErrorDetails()); + assertEquals("", result2.getErrorStackTrace()); + assertEquals("", result2.getErrorDetails()); + } + + private BlueTestResult getResult(String filterCondition){ BlueRobotTestResult.FactoryImpl factory = new BlueRobotTestResult.FactoryImpl(); Result blueResult = factory.getBlueTestResults(mockBuild, mockReachable); - for (BlueTestResult tempResult : blueResult.results) { - String name = tempResult.getName(); - String trace = tempResult.getErrorStackTrace(); - String msg = tempResult.getErrorDetails(); - switch (name) { - case "Failed Test": - assertEquals("Fail This fails!\n", trace); - assertEquals("This fails!", msg); - break; - case "Nested failed test": - assertEquals("My failed Keyword\n The real failed keyword\n Fail Really fails!\n", trace); - assertEquals("Really fails!", msg); - break; - case "Nested with not first": - String helper = "Should Be Equal ${MESSAGE} Hello, world!\nShould Be Equal ${MESSAGE} Hello, world!\n" + - "My failed Keyword\n The real failed keyword\n Fail Really fails!\n"; - assertEquals(helper, trace); - assertEquals("Really fails!", msg); - break; - case "Another Test": - assertEquals("", trace); - assertEquals("", msg); - break; - case "My Test": - assertEquals("", trace); - assertEquals("", msg); - break; - } - } + return StreamSupport.stream(blueResult.results.spliterator(), false) + .filter(element -> element.getName().equals(filterCondition)) + .collect(Collectors.toList()).get(0); } } diff --git a/src/test/java/hudson/plugins/robot/model/RobotResultTest.java b/src/test/java/hudson/plugins/robot/model/RobotResultTest.java index a98792f1..da5a1e82 100644 --- a/src/test/java/hudson/plugins/robot/model/RobotResultTest.java +++ b/src/test/java/hudson/plugins/robot/model/RobotResultTest.java @@ -136,7 +136,7 @@ public void testShouldParseFailedCriticalCases(){ @Test public void testShouldParseFailedNewCriticalCases() throws Exception{ - RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("new_critical_output.xml", null, null); + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("new_critical_output.xml", null, null); result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("new_critical_output.xml").toURI()).getParentFile(), null); result.tally(null); @@ -251,7 +251,7 @@ public void testShouldReturnCaseTags(){ @Test public void testShouldParseSplittedOutput() throws Exception { - RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("testfile.xml", null, null); + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("testfile.xml", null, null); result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("testfile.xml").toURI()).getParentFile(), null); RobotSuiteResult suite = result.getSuite("nestedSuites"); @@ -283,4 +283,48 @@ public void testShouldParseWholeSuiteDuration() throws Exception { res.tally(null); assertEquals(10141, res.getDuration()); } + + @Test + public void testShouldParseSkippedTests() throws Exception { + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("robot4_skip.xml", null, null); + result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("robot4_skip.xml").toURI()).getParentFile(), null); + result.tally(null); + assertEquals(6, result.getSkipped()); + } + + @Test + public void testShouldParsePassedFromRobot4() throws Exception { + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("robot4_skip.xml", null, null); + result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("robot4_skip.xml").toURI()).getParentFile(), null); + result.tally(null); + assertEquals(4, result.getPassed()); + assertEquals(66.6, result.getPassPercentage(), 0.1); + } + + @Test + public void testShouldParseFailedFromRobot4() throws Exception { + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("robot4_skip.xml", null, null); + result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("robot4_skip.xml").toURI()).getParentFile(), null); + result.tally(null); + assertEquals(2, result.getFailed()); + } + + @Test + public void testShouldParseTotalFromRobot4() throws Exception { + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("robot4_skip.xml", null, null); + result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("robot4_skip.xml").toURI()).getParentFile(), null); + result.tally(null); + assertEquals(12, result.getOverallTotal()); + } + + @Test + public void testShouldParseTagsFromRobot4() throws Exception { + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("robot4_skip.xml", null, null); + result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("robot4_skip.xml").toURI()).getParentFile(), null); + result.tally(null); + + RobotCaseResult caseResult = (RobotCaseResult)result.findObjectById("Skip/Test 8 Will Always Fail"); + String tags = StringUtils.join(caseResult.getTags(), ","); + assertEquals("fail,tag2,tag3", tags); + } } diff --git a/src/test/java/hudson/plugins/robot/tokens/RobotFailedCasesTokenMacroTest.java b/src/test/java/hudson/plugins/robot/tokens/RobotFailedCasesTokenMacroTest.java index 76d80f8c..e95d6d3c 100644 --- a/src/test/java/hudson/plugins/robot/tokens/RobotFailedCasesTokenMacroTest.java +++ b/src/test/java/hudson/plugins/robot/tokens/RobotFailedCasesTokenMacroTest.java @@ -32,7 +32,7 @@ public void setUp(){ RobotResult result = Mockito.mock(RobotResult.class); - List failedList = new ArrayList(); + List failedList = new ArrayList<>(); RobotCaseResult case1 = Mockito.mock(RobotCaseResult.class); Mockito.when(case1.getRelativePackageName(result)).thenReturn("Failcases.subcases.Failure1"); diff --git a/src/test/resources/hudson/plugins/robot/blueocean/blue_skip.xml b/src/test/resources/hudson/plugins/robot/blueocean/blue_skip.xml new file mode 100644 index 00000000..edd1cad9 --- /dev/null +++ b/src/test/resources/hudson/plugins/robot/blueocean/blue_skip.xml @@ -0,0 +1,285 @@ + + + + + + Will pass + Logs the given message with the given level. + Will pass + + + pass + tag2 + + + + + Will pass + Logs the given message with the given level. + Will pass + + + pass + tag2 + tag3 + + + + + Will pass + Logs the given message with the given level. + Will pass + + + + + + skippable + Test skipped with '--skip' command line option. + + + This test has documentation + skippable + Test skipped with '--skip' command line option. + + + skippable + tag2 + Test skipped with '--skip' command line option. + + + skippable + Test skipped with '--skip' command line option. + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + This test has documentation + fail + tag2 + tag3 + AssertionError + + + + Optional failure message + Fails the test with the given message and optionally alters its tags. + Optional failure message + + + fail + Optional failure message + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + skipfail + Test failed but its tags matched '--SkipOnFailure' and it was marked skipped. + + Original failure: + AssertionError + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + skipfail + tag2 + Test failed but its tags matched '--SkipOnFailure' and it was marked skipped. + + Original failure: + AssertionError + + + + Will Pass + Logs the given message with the given level. + Will Pass + + + This test has documentation + skipfail + + + When run with `--skip skippable --skiponfailure skipfail` + will produce the following results: + PASSED 4 + FAIL 2 + SKIP 6 + + + + + + ${x} + 2 + + 0 + + ${x} + Logs the given message with the given level. + 0 + + + + ${x} + + ${arg} + Logs the given message with the given level. + 0 + + + + + + ${x}==1 + Fail + Runs the given keyword with the given arguments, if ``condition`` is true. + + + + + + 1 + + ${x} + Logs the given message with the given level. + 1 + + + + ${x} + + ${arg} + Logs the given message with the given level. + 1 + + + + + + ${x}==1 + Fail + Runs the given keyword with the given arguments, if ``condition`` is true. + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + + + + + + + failure + for + AssertionError + + + + ${var} + mikki hiiri + Returns the given values which can then be assigned to a variables. + ${var} = mikki hiiri + + + + + + ${var} + + ${arg} + Logs the given message with the given level. + mikki hiiri + + + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + + + + + failure + if + AssertionError + + + + ${var} + mikki hiiri + Returns the given values which can then be assigned to a variables. + ${var} = mikki hiiri + + + + + + Fails the test with the given message and optionally alters its tags. + + + + + + + ${var} + + ${arg} + Logs the given message with the given level. + mikki hiiri + + + + + + ${var} + Fails the test with the given message and optionally alters its tags. + mikki hiiri + + + + + + + failure + if + mikki hiiri + + + + + + All Tests + + + fail + failure + for + if + pass + skipfail + skippable + tag2 + tag3 + + + Skip + Loops + + + + + diff --git a/src/test/resources/hudson/plugins/robot/model/robot4_skip.xml b/src/test/resources/hudson/plugins/robot/model/robot4_skip.xml new file mode 100644 index 00000000..8e8d7d78 --- /dev/null +++ b/src/test/resources/hudson/plugins/robot/model/robot4_skip.xml @@ -0,0 +1,136 @@ + + + + + + Will pass + Logs the given message with the given level. + Will pass + + + pass + tag2 + + + + + Will pass + Logs the given message with the given level. + Will pass + + + pass + tag2 + tag3 + + + + + Will pass + Logs the given message with the given level. + Will pass + + + + + + skippable + Test skipped with '--skip' command line option. + + + This test has documentation + skippable + Test skipped with '--skip' command line option. + + + skippable + tag2 + Test skipped with '--skip' command line option. + + + skippable + Test skipped with '--skip' command line option. + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + This test has documentation + fail + tag2 + tag3 + AssertionError + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + fail + AssertionError + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + skipfail + Test failed but its tags matched '--SkipOnFailure' and it was marked skipped. + + Original failure: + AssertionError + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + skipfail + tag2 + Test failed but its tags matched '--SkipOnFailure' and it was marked skipped. + + Original failure: + AssertionError + + + + Will Pass + Logs the given message with the given level. + Will Pass + + + This test has documentation + skipfail + + + When run with `--skip skippable --skiponfailure skipfail` + will produce the following results: + PASSED 4 + FAIL 2 + SKIP 6 + + + + + All Tests + + + fail + pass + skipfail + skippable + tag2 + tag3 + + + Skip + + + + + diff --git a/src/test/resources/hudson/plugins/robot/robot4_if_output.xml b/src/test/resources/hudson/plugins/robot/robot4_if_output.xml new file mode 100644 index 00000000..1e4c16b1 --- /dev/null +++ b/src/test/resources/hudson/plugins/robot/robot4_if_output.xml @@ -0,0 +1,198 @@ + + + + + + ${var} + mikki hiiri + Returns the given values which can then be assigned to a variables. + ${var} = mikki hiiri + + + + + + ${var} + Logs the given message with the given level. + mikki hiiri + + + + + + + if + success + + + + + ${var} + mikki hiiri + Returns the given values which can then be assigned to a variables. + ${var} = mikki hiiri + + + + + + Fails the test with the given message and optionally alters its tags. + + + + + + + ${var} + + ${arg} + Logs the given message with the given level. + mikki hiiri + + + + + + ${var} + Logs the given message with the given level. + mikki hiiri + + + + + + + if + success + + + + + ${var} + mikki hiiri + Returns the given values which can then be assigned to a variables. + ${var} = mikki hiiri + + + + + + Fails the test with the given message and optionally alters its tags. + + + + + + + ${var} + Logs the given message with the given level. + mikki hiiri + + + + + + + Fails the test with the given message and optionally alters its tags. + + + + + + + if + success + + + + + ${var} + mikki hiiri + Returns the given values which can then be assigned to a variables. + ${var} = mikki hiiri + + + + + + ${var} + + ${arg} + Logs the given message with the given level. + mikki hiiri + + + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + + + + + failure + if + AssertionError + + + + ${var} + mikki hiiri + Returns the given values which can then be assigned to a variables. + ${var} = mikki hiiri + + + + + + Fails the test with the given message and optionally alters its tags. + + + + + + + ${var} + + ${arg} + Logs the given message with the given level. + mikki hiiri + + + + + + ${var} + Fails the test with the given message and optionally alters its tags. + mikki hiiri + + + + + + + failure + if + mikki hiiri + + + + + + All Tests + + + failure + if + success + + + Loops + + + + + diff --git a/src/test/resources/hudson/plugins/robot/robot4_nested_output.xml b/src/test/resources/hudson/plugins/robot/robot4_nested_output.xml new file mode 100644 index 00000000..9f8e6b7e --- /dev/null +++ b/src/test/resources/hudson/plugins/robot/robot4_nested_output.xml @@ -0,0 +1,248 @@ + + + + + + ${x} + 2 + + 0 + + ${y} + 2 + + 0 + + ${x}-${y} + Logs the given message with the given level. + 0-0 + + + + + + 1 + + ${x}-${y} + Logs the given message with the given level. + 0-1 + + + + + + + + + ${x} + 2 + + 0 + + ${y} + 2 + + 0 + + ${x}-${y} + Logs the given message with the given level. + 0-0 + + + + + + 1 + + ${x}-${y} + Logs the given message with the given level. + 0-1 + + + + + + + + + + 1 + + ${y} + 2 + + 0 + + ${x}-${y} + Logs the given message with the given level. + 1-0 + + + + + + 1 + + ${x}-${y} + Logs the given message with the given level. + 1-1 + + + + + + + + + + + + + + ${x}==1 + Fail + Runs the given keyword with the given arguments, if ``condition`` is true. + + + + + + 1 + + ${y} + 2 + + 0 + + ${x}-${y} + Logs the given message with the given level. + 1-0 + + + + + + 1 + + ${x}-${y} + Logs the given message with the given level. + 1-1 + + + + + + + + + ${x} + 2 + + 0 + + ${y} + 2 + + 0 + + ${x}-${y} + Logs the given message with the given level. + 0-0 + + + + + + 1 + + ${x}-${y} + Logs the given message with the given level. + 0-1 + + + + + + + + + + 1 + + ${y} + 2 + + 0 + + ${x}-${y} + Logs the given message with the given level. + 1-0 + + + + + + 1 + + ${x}-${y} + Logs the given message with the given level. + 1-1 + + + + + + + + + + + + + + ${x}==1 + Fail + Runs the given keyword with the given arguments, if ``condition`` is true. + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + + + + + + + failure + for + AssertionError + + + + ${x} + 0 + + + for + success + + + + + + + All Tests + + + failure + for + success + + + Loops + + + + + diff --git a/src/test/resources/hudson/plugins/robot/robot4_output.xml b/src/test/resources/hudson/plugins/robot/robot4_output.xml new file mode 100644 index 00000000..ea85f406 --- /dev/null +++ b/src/test/resources/hudson/plugins/robot/robot4_output.xml @@ -0,0 +1,133 @@ + + + + + + Will pass + Logs the given message with the given level. + Will pass + + + pass + tag2 + + + + + Will pass + Logs the given message with the given level. + Will pass + + + pass + tag2 + tag3 + + + + + Will pass + Logs the given message with the given level. + Will pass + + + + + + skippable + Test skipped with '--skip' command line option. + + + skippable + Test skipped with '--skip' command line option. + + + skippable + tag2 + Test skipped with '--skip' command line option. + + + skippable + Test skipped with '--skip' command line option. + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + fail + tag2 + tag3 + AssertionError + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + fail + AssertionError + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + skipfail + Test failed but its tags matched '--SkipOnFailure' and it was marked skipped. + + Original failure: + AssertionError + + + + Fails the test with the given message and optionally alters its tags. + AssertionError + + + skipfail + tag2 + Test failed but its tags matched '--SkipOnFailure' and it was marked skipped. + + Original failure: + AssertionError + + + + Will Pass + Logs the given message with the given level. + Will Pass + + + skipfail + + + When run with `--skip skippable --skiponfailure skipfail` + will produce the following results: + PASSED 4 + FAIL 2 + SKIP 6 + + + + + All Tests + + + fail + pass + skipfail + skippable + tag2 + tag3 + + + Skip + + + + +