Skip to content
This repository has been archived by the owner on May 4, 2023. It is now read-only.

Commit

Permalink
Merge pull request #2 from a-bastrykina/junit-tests-new-structure
Browse files Browse the repository at this point in the history 
New junit.xml structure
  • Loading branch information
@haiodo
haiodo authored Jan 13, 2020
2 parents a5217c5 + abfe265 commit 3abbc25
Show file tree
Hide file tree
Showing 5 changed files with 283 additions and 106 deletions.
158 changes: 126 additions & 32 deletions pkg/commands/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1319,28 +1319,109 @@ func (ctx *executionContext) findGoTest(executionConfig *config.ExecutionConfig)
return result, nil
}

func buildClusterSuiteName(clusters []*clustersGroup) string {
var clusterProviderNames = make([]string, len(clusters))
for i := 0; i < len(clusters); i++ {
clusterProviderNames[i] = clusters[i].config.Name
}
return strings.Join(clusterProviderNames, "-")
}

func (ctx *executionContext) generateJUnitReportFile() (*reporting.JUnitFile, error) {
// generate and write report
ctx.report = &reporting.JUnitFile{}

summarySuite := &reporting.Suite{
Name: "All tests",
}

// We need to group all tests by executions.
executionsTests := ctx.getAllTestTasksGroupedByExecutions()

totalFailures := 0
for _, cluster := range ctx.clusters {
failures := 0
totalTests := 0
totalTime := time.Duration(0)
suite := &reporting.Suite{
Name: cluster.config.Name,
totalTests := 0
totalTime := time.Duration(0)
// Generate suites by executions.
for execName, executionTasks := range executionsTests {
execSuite := &reporting.Suite{
Name: execName,
}

for _, test := range cluster.tasks {
totalTests, totalTime, failures = ctx.generateTestCaseReport(test, totalTests, totalTime, failures, suite)
// Group execution's test tasks by cluster type.
clustersTests := make(map[string][]*testTask)
for _, test := range executionTasks {
clusterGroupName := buildClusterSuiteName(test.clusters)
clustersTests[clusterGroupName] = append(clustersTests[clusterGroupName], test)
}

for _, test := range cluster.completed {
totalTests, totalTime, failures = ctx.generateTestCaseReport(test, totalTests, totalTime, failures, suite)
executionFailures := 0
executionTests := 0
executionTime := time.Duration(0)

// Generate nested suites by cluster types.
for clusterTaskName, tests := range clustersTests {
clusterTests, clusterTime, clusterFailures, clusterSuite := ctx.generateReportSuiteByTestTasks(clusterTaskName, tests)

executionFailures += clusterFailures
executionTests += clusterTests
executionTime += clusterTime

clusterSuite.Time = fmt.Sprintf("%v", clusterTime.Seconds())
clusterSuite.TimeComment = fmt.Sprintf(reporting.TimeCommentFormat, clusterTime.Round(time.Second))
clusterSuite.Failures = clusterFailures
clusterSuite.Tests = clusterTests
execSuite.Suites = append(execSuite.Suites, clusterSuite)
}

// Check cluster executions.
totalFailures += executionFailures
totalTests += executionTests
totalTime += executionTime

execSuite.Tests = executionTests
execSuite.Failures = executionFailures
execSuite.Time = fmt.Sprintf("%v", executionTime.Seconds())
execSuite.TimeComment = fmt.Sprintf(reporting.TimeCommentFormat, executionTime.Round(time.Second))
summarySuite.Suites = append(summarySuite.Suites, execSuite)
}

// Add a suite with cluster failures.
clusterFailuresTime, clusterFailuresCount, clusterFailuresSuite := ctx.generateClusterFailuresReportSuite()
if clusterFailuresCount > 0 {
totalFailures += clusterFailuresCount
totalTime += clusterFailuresTime
clusterFailuresSuite.Tests = clusterFailuresCount
clusterFailuresSuite.Failures = clusterFailuresCount
summarySuite.Suites = append(summarySuite.Suites, clusterFailuresSuite)
}

summarySuite.Time = fmt.Sprintf("%v", totalTime.Seconds())
summarySuite.TimeComment = fmt.Sprintf(reporting.TimeCommentFormat, totalTime.Round(time.Second))
summarySuite.Failures = totalFailures
summarySuite.Tests = totalTests
ctx.report.Suites = append(ctx.report.Suites, summarySuite)

output, err := xml.MarshalIndent(ctx.report, " ", " ")
if err != nil {
logrus.Errorf("failed to store JUnit xml report: %v\n", err)
}
if ctx.cloudTestConfig.Reporting.JUnitReportFile != "" {
ctx.manager.AddFile(ctx.cloudTestConfig.Reporting.JUnitReportFile, output)
}
if totalFailures > 0 {
return ctx.report, errors.Errorf("there is failed tests %v", totalFailures)
}
return ctx.report, nil
}

func (ctx *executionContext) generateClusterFailuresReportSuite() (time.Duration, int, *reporting.Suite) {
clusterFailuresSuite := &reporting.Suite{
Name: "Cluster failures",
}

clusterFailures := 0
failuresTime := time.Duration(0)
// Check cluster instances
for _, cluster := range ctx.clusters {
availableClusters := 0
for _, inst := range cluster.instances {
if inst.state != clusterNotAvailable {
Expand All @@ -1352,34 +1433,46 @@ func (ctx *executionContext) generateJUnitReportFile() (*reporting.JUnitFile, er
for _, inst := range cluster.instances {
if inst.state == clusterNotAvailable {
for _, exec := range inst.executions {
ctx.generateClusterFailedReportEntry(inst, exec, suite)
failures++
totalTests++
ctx.generateClusterFailedReportEntry(inst, exec, clusterFailuresSuite)
failuresTime += exec.duration
clusterFailures++
break
}
}
}
}

suite.Tests = totalTests
suite.Failures = failures
suite.Time = fmt.Sprintf("%v", totalTime)
totalFailures += failures

ctx.report.Suites = append(ctx.report.Suites, suite)
}
clusterFailuresSuite.Time = fmt.Sprintf("%v", failuresTime.Seconds())
clusterFailuresSuite.TimeComment = fmt.Sprintf(reporting.TimeCommentFormat, failuresTime.Round(time.Second))
return failuresTime, clusterFailures, clusterFailuresSuite
}

output, err := xml.MarshalIndent(ctx.report, " ", " ")
if err != nil {
logrus.Errorf("failed to store JUnit xml report: %v\n", err)
func (ctx *executionContext) generateReportSuiteByTestTasks(suiteName string, tests []*testTask) (int, time.Duration, int, *reporting.Suite) {
failuresCount := 0
testsCount := 0
time := time.Duration(0)
suite := &reporting.Suite{
Name: suiteName,
}
if ctx.cloudTestConfig.Reporting.JUnitReportFile != "" {
ctx.manager.AddFile(ctx.cloudTestConfig.Reporting.JUnitReportFile, output)
for _, test := range tests {
testsCount, time, failuresCount = ctx.generateTestCaseReport(test, testsCount, time, failuresCount, suite)
}
if totalFailures > 0 {
return ctx.report, errors.Errorf("there is failed tests %v", totalFailures)
return testsCount, time, failuresCount, suite
}

func (ctx *executionContext) getAllTestTasksGroupedByExecutions() map[string][]*testTask {
var executionsTests = make(map[string][]*testTask)
for _, cluster := range ctx.clusters {
for _, test := range cluster.tasks {
execName := test.test.ExecutionConfig.Name
executionsTests[execName] = append(executionsTests[execName], test)
}
for _, test := range cluster.completed {
execName := test.test.ExecutionConfig.Name
executionsTests[execName] = append(executionsTests[execName], test)
}
}
return ctx.report, nil
return executionsTests
}

func (ctx *executionContext) generateClusterFailedReportEntry(inst *clusterInstance, exec *clusterOperationRecord, suite *reporting.Suite) {
Expand All @@ -1394,7 +1487,7 @@ func (ctx *executionContext) generateClusterFailedReportEntry(inst *clusterInsta
}
startCase := &reporting.TestCase{
Name: fmt.Sprintf("Startup-%v", inst.id),
Time: fmt.Sprintf("%v", exec.duration),
Time: fmt.Sprintf("%v", exec.duration.Seconds()),
}
startCase.Failure = &reporting.Failure{
Type: "ERROR",
Expand All @@ -1406,8 +1499,9 @@ func (ctx *executionContext) generateClusterFailedReportEntry(inst *clusterInsta

func (ctx *executionContext) generateTestCaseReport(test *testTask, totalTests int, totalTime time.Duration, failures int, suite *reporting.Suite) (int, time.Duration, int) {
testCase := &reporting.TestCase{
Name: test.test.Key,
Time: test.test.Duration.String(),
Name: test.test.Name,
Time: fmt.Sprintf("%v", test.test.Duration.Seconds()),
Cluster: test.clusterTaskID,
}

switch test.test.Status {
Expand Down
28 changes: 21 additions & 7 deletions pkg/reporting/junit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ package reporting

import "encoding/xml"

const (
// TimeCommentFormat is a format for printing readable time suite comment
TimeCommentFormat = "Suite was running for %v"
)

// JUnitFile - JUnitFile
type JUnitFile struct {
XMLName xml.Name `xml:"testsuites"`
Expand All @@ -10,13 +15,21 @@ type JUnitFile struct {

// Suite - Suite
type Suite struct {
XMLName xml.Name `xml:"testsuite"`
Tests int `xml:"tests,attr"`
Failures int `xml:"failures,attr"`
Time string `xml:"time,attr"`
Name string `xml:"name,attr"`
Properties []*Property `xml:"properties>property,omitempty"`
TestCases []*TestCase
XMLName xml.Name `xml:"testsuite"`
Tests int `xml:"tests,attr"`
Failures int `xml:"failures,attr"`
Time string `xml:"time,attr"`
Name string `xml:"name,attr"`
Properties []*Property `xml:"properties>property,omitempty"`
TimeComment string `xml:",comment"`
TestCases []*TestCase
Suites []*Suite
}

// SuiteDetails holds additional information about test suite.
type SuiteDetails struct {
XMLName xml.Name `xml:"details"`
FormattedTime string `xml:"formatted_time,attr"`
}

// TestCase - TestCase
Expand All @@ -25,6 +38,7 @@ type TestCase struct {
Classname string `xml:"classname,attr"`
Name string `xml:"name,attr"`
Time string `xml:"time,attr"`
Cluster string `xml:"cluster_instance,attr"`
SkipMessage *SkipMessage `xml:"skipped,omitempty"`
Failure *Failure `xml:"failure,omitempty"`
}
Expand Down
72 changes: 44 additions & 28 deletions pkg/tests/all_cluster_fail_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,20 @@ func TestClusterInstancesFailed(t *testing.T) {

g.Expect(report).NotTo(BeNil())

g.Expect(len(report.Suites)).To(Equal(2))
g.Expect(report.Suites[0].Failures).To(Equal(1))
g.Expect(report.Suites[0].Tests).To(Equal(3))
g.Expect(len(report.Suites[0].TestCases)).To(Equal(3))
rootSuite := report.Suites[0]

g.Expect(report.Suites[1].Failures).To(Equal(2))
g.Expect(report.Suites[1].Tests).To(Equal(5))
g.Expect(len(report.Suites[1].TestCases)).To(Equal(5))
g.Expect(len(rootSuite.Suites)).To(Equal(2))

g.Expect(len(rootSuite.Suites[0].Suites)).To(Equal(2))

g.Expect(rootSuite.Suites[0].Failures).To(Equal(1))
g.Expect(rootSuite.Suites[0].Tests).To(Equal(6))
g.Expect(len(rootSuite.Suites[0].Suites[0].TestCases)).To(Equal(3))
g.Expect(len(rootSuite.Suites[0].Suites[1].TestCases)).To(Equal(3))

g.Expect(rootSuite.Suites[1].Failures).To(Equal(2))
g.Expect(rootSuite.Suites[1].Tests).To(Equal(2))
g.Expect(len(rootSuite.Suites[1].TestCases)).To(Equal(2))

// Do assertions
}
Expand Down Expand Up @@ -84,24 +90,32 @@ func TestClusterInstancesOnFailGoRunner(t *testing.T) {

g.Expect(report).NotTo(BeNil())

g.Expect(len(report.Suites)).To(Equal(2))
g.Expect(report.Suites[0].Failures).To(Equal(1))
g.Expect(report.Suites[0].Tests).To(Equal(3))
g.Expect(len(report.Suites[0].TestCases)).To(Equal(3))
rootSuite := report.Suites[0]

g.Expect(report.Suites[1].Failures).To(Equal(2))
g.Expect(report.Suites[1].Tests).To(Equal(5))
g.Expect(len(report.Suites[1].TestCases)).To(Equal(5))
g.Expect(len(rootSuite.Suites)).To(Equal(2))

g.Expect(rootSuite.Suites[0].Failures).To(Equal(1))
g.Expect(rootSuite.Suites[0].Tests).To(Equal(6))
g.Expect(len(rootSuite.Suites[0].Suites[0].TestCases)).To(Equal(3))
g.Expect(len(rootSuite.Suites[0].Suites[1].TestCases)).To(Equal(3))

g.Expect(rootSuite.Suites[1].Failures).To(Equal(2))
g.Expect(rootSuite.Suites[1].Tests).To(Equal(2))
g.Expect(len(rootSuite.Suites[1].TestCases)).To(Equal(2))

foundFailTest := false

for _, t := range report.Suites[0].TestCases {
if t.Name == "_TestFail" {
g.Expect(t.Failure).NotTo(Equal(BeNil()))
g.Expect(strings.Contains(t.Failure.Contents, ">>>Running on fail script<<<")).To(Equal(true))
foundFailTest = true
} else {
g.Expect(t.Failure).Should(BeNil())
for _, execSuite := range rootSuite.Suites[0].Suites {
if execSuite.Name == "a_provider" {
for _, testCase := range execSuite.TestCases {
if testCase.Name == "TestFail" {
g.Expect(testCase.Failure).NotTo(BeNil())
g.Expect(strings.Contains(testCase.Failure.Contents, ">>>Running on fail script<<<")).To(Equal(true))
foundFailTest = true
} else {
g.Expect(testCase.Failure).Should(BeNil())
}
}
}
}
g.Expect(foundFailTest).Should(BeTrue())
Expand Down Expand Up @@ -140,13 +154,14 @@ func TestClusterInstancesOnFailShellRunner(t *testing.T) {
g.Expect(err.Error()).To(Equal("there is failed tests 1"))
foundFailTest := false

for _, t := range report.Suites[0].TestCases {
if t.Name == "_fail" {
g.Expect(t.Failure).NotTo(Equal(BeNil()))
g.Expect(strings.Contains(t.Failure.Contents, ">>>Running on fail script<<<")).To(Equal(true))
for _, executionSuite := range report.Suites[0].Suites {
testCase := executionSuite.Suites[0].TestCases[0]
if executionSuite.Name == "fail" {
g.Expect(testCase.Failure).NotTo(BeNil())
g.Expect(strings.Contains(testCase.Failure.Contents, ">>>Running on fail script<<<")).To(Equal(true))
foundFailTest = true
} else {
g.Expect(t.Failure).Should(BeNil())
g.Expect(testCase.Failure).Should(BeNil())
}
}
g.Expect(foundFailTest).Should(BeTrue())
Expand Down Expand Up @@ -195,8 +210,9 @@ func TestClusterInstancesOnFailShellRunnerInterdomain(t *testing.T) {
g.Expect(err.Error()).To(Equal("there is failed tests 1"))
foundFailTest := false

for _, t := range report.Suites[0].TestCases {
if t.Name == "a_provider_b_provider_fail" {
for _, suite := range report.Suites[0].Suites {
t := suite.Suites[0].TestCases[0]
if suite.Name == "fail" {
g.Expect(t.Failure).NotTo(Equal(BeNil()))
g.Expect(strings.Contains(t.Failure.Contents, ">>>Running on fail script with ./.tests/config.a <<<")).To(Equal(true))
g.Expect(strings.Contains(t.Failure.Contents, ">>>Running on fail script with ./.tests/config.b <<<")).To(Equal(true))
Expand Down
Loading

0 comments on commit 3abbc25

Please sign in to comment.