From cd22bfe6bd3a17fd95bcc575313034d569b9b720 Mon Sep 17 00:00:00 2001 From: Christopher Dwyer-Perkins Date: Tue, 28 Jan 2025 11:01:10 -0400 Subject: [PATCH] Bugfix/only annotation not always applied correctly (#308) * Added expected and actual to TestResults class * First pass of MochaTestReporter * Added actual and expected to assertTrue and assertFalse * New string function and more assert messages * Cleaned up message generation * Fixed failing unit test * Cleaned up some messaging output * Do not report empty actual and expected diffs * actual and expected are invalid by default * whitespace clean up * Made temp logs less spammy * More assertion updates * Updates to most aa and array asserts * clean up on displaying types * Updates to assertArrayCount * Updates to assertArrayCount to fix a crash case * Messaging updates * Updated array count assertions * updated assertNotEmpty and assertEmpty * Fixed a actual/expected filting bug * Added tests to sample project * Fixed a bug with assertArrayNotCount * Updates to assertArrayContainsOnlyValuesOfType and assertType * Fixed a bug in assertArrayContainsOnlyValuesOfType with AA's * Removed stop statement * updates to assertArrayNotContainsSubset, assertArrayContainsSubset, and more tests for AssertArrayContains * updates to assertClass and assertSubType * Fixed a bugt in assertArrayNotContainsSubset that would always fail for aa subsets * updated assertNodeCount * More updates * Updates assets to respect custom failure messages when supplied * remove only annotation * Fixed a crash in eqAssocArray * Fixed some unexpected failing tests * Fixed some failure messages for assert true and false * Tweeks to empty multiline aa and array * Fixed a crash reporting node tests due to missing actual and expected results * Fixed a bug related to ignoredFields * Fixed a error log that would print even when there was no error for node tests * Added new test reporter hooks * implimented new hooks and moved logs into reporters * Fixed a falure case that would pass * Include ignored tests but skip their exicution when found * Moved tests that should fail to their own suite * Removed old commited code * Removed an ignore anotation * Fixed some issues with skipped tests * Updated the console test reporter to handle skipped and ignored tests/groups/suites * Fixed a regression in the console test reporter * Fixed some broken tests * Re-enabled catch on crash * Added new colorizeOutput setting * Formatting fixes * linking fixes * replaced short hand prints * Moved diffing logic into an internal namespace to prevent code conflicts * Code clean up * Updated reporters to take event objects * Updated brighterscript to 0.68.3 * Updated brighterscript to 0.68.3 for test workspace * Added a log that will always fire after test reporters * Removed some unnessisary prints from mocha reporter * Doc blocks * Made stacktraces easier to read and added test file links in crashes * Typo * More tests * Removed actual and expected from assertNotEqual * More failure tests * More failure tests for assertArrayNotContains * Cleaned up some tests * Fixed assertAANotHasKey not marking tests as failed in some cases * Fixed crash in assertInvalid * Added actual and expected diff for assertAAHasKeys * Cleaned up some failure messages * More tests and clean up for failed assertions * Updated asMultilineString to print invalid and uninitialized as their types * comment typo * Code clean up and lots more comments in mocha test reporter * Fixed the reports namespace name to be reporter * Moved some logs to a new warning log rather then error * updated assertClass to remove actual and expected diff * Fixed uninitilized not being comparable with eqValues * Fixed mocha results printing * Fixed most failing tests * Update CommonUtils.bs * Update CommonUtils.bs * Update TestGroup.bs * Update CodeCoverageProcessor.spec.ts * added fast glob npm module * updated FileFactory to use flog to find framework files * Updated the framework fold structure to match what will be on device * Fixed build copy command * Code clean up and removed unessisary props following the structure change * Removed options from FileFactory constructor * Moved MochaTestReporter class into source/rooibos * Added a couple passing tests for assertInvalid * Always sort keys in aa when generating diff strings * Added support for reporting slow tests * Fixed failing test * Fixed an issue where identical diffs would attempt to be rendered * Moved back to brighterscript 0.67.4 * Fixed typos in docs * Added doc for slow annotation * Added test with multi levels of only annotations * Started working on a fix onlys not being applied correctly when mixed with suites, groups, and tests * Added tons of tests for passing usecases * Update bsc-plugin/src/lib/rooibos/CodeCoverageProcessor.spec.ts * New line when tests start * Updated the logic around what files are included or not * Added assertNotInvalid test for roInvalid * Updated assertInvalid test for roInvalid * hardened assertNotInvalid and fixed assertInvalid for roInvalid * Fixed some off by oneline number reporting bugs --------- Co-authored-by: Bronley Plumb --- .../src/lib/rooibos/RooibosSessionInfo.ts | 141 +++++------ bsc-plugin/src/lib/rooibos/TestGroup.ts | 18 +- bsc-plugin/src/lib/rooibos/TestSuite.ts | 16 +- bsc-plugin/src/plugin.spec.ts | 219 ++++++++++++++++++ 4 files changed, 278 insertions(+), 116 deletions(-) diff --git a/bsc-plugin/src/lib/rooibos/RooibosSessionInfo.ts b/bsc-plugin/src/lib/rooibos/RooibosSessionInfo.ts index d6d50142..9a3c5fd6 100644 --- a/bsc-plugin/src/lib/rooibos/RooibosSessionInfo.ts +++ b/bsc-plugin/src/lib/rooibos/RooibosSessionInfo.ts @@ -12,9 +12,7 @@ export class SessionInfo { public testSuites = new Map(); public testSuitesByPath = new Map(); public testSuitesToRun: TestSuite[] = []; - public hasSoloSuites = false; - public hasSoloGroups = false; - public hasSoloTests = false; + public shouldRunSolo = false; public testsCount = 0; public suitesCount = 0; public groupsCount = 0; @@ -36,19 +34,9 @@ export class SessionInfo { for (let testSuite of testSuites) { if (testSuite.isValid && !this.isExcludedByTag(testSuite, false)) { this.testSuites.set(testSuite.name, testSuite); - this.addTestSuiteToPath(testSuite); - if (testSuite.isSolo) { - this.hasSoloSuites = !this.hasSoloGroups && !this.hasSoloTests; - } - if (testSuite.hasSoloGroups) { - this.hasSoloGroups = !this.hasSoloTests; - } - if (testSuite.hasSoloTests) { - this.hasSoloTests = true; - this.hasSoloGroups = false; - this.hasSoloSuites = false; - } + + this.shouldRunSolo = this.shouldRunSolo || testSuite.isSolo || testSuite.hasSoloGroups || testSuite.hasSoloTests; } else { this.allTestSuites.add(testSuite); } @@ -70,85 +58,74 @@ export class SessionInfo { public updateInfo() { this.resetCounts(); for (let testSuite of [...this.testSuites.values()]) { - if (this.isExcludedByTag(testSuite, false)) { - testSuite.isIncluded = false; - } else if (this.hasSoloTests && !testSuite.hasSoloTests) { - testSuite.isIncluded = false; - } else if (this.hasSoloSuites && !testSuite.isSolo) { - testSuite.isIncluded = false; - } else if (testSuite.isIgnored) { - testSuite.isIncluded = true; - this.ignoredTestNames.push(testSuite.name + ' [WHOLE SUITE]'); - this.ignoredCount++; + testSuite.isIncluded = false; + testSuite.isIgnored = testSuite.isIgnored || this.isExcludedByTag(testSuite, false); + + if (this.shouldRunSolo) { + if (!testSuite.isIgnored && (testSuite.isSolo || testSuite.hasSoloGroups || testSuite.hasSoloTests)) { + testSuite.isIncluded = true; + } } else { testSuite.isIncluded = true; + if (testSuite.isIgnored) { + this.ignoredTestNames.push(testSuite.name + ' [WHOLE SUITE]'); + this.ignoredCount++; + } } + if (!testSuite.isIncluded) { continue; } //'testSuite ' + testSuite.name); for (let testGroup of testSuite.getTestGroups()) { - if (testSuite.isIgnored) { - testGroup.isIgnored = true; - } + testGroup.isIgnored = testGroup.isIgnored || testSuite.isIgnored || this.isExcludedByTag(testGroup, true); + testGroup.isIncluded = false; - //'GROUP ' + testGroup.name); - if (testGroup.isIgnored) { - this.ignoredCount += testGroup.ignoredTestCases.length; - this.ignoredTestNames.push(testGroup.name + ' [WHOLE GROUP]'); - testGroup.isIncluded = true; - } - - if (testGroup.ignoredTestCases.length > 0) { - this.ignoredTestNames.push(testGroup.name); - this.ignoredCount += testGroup.ignoredTestCases.length; - for (let ignoredTestCase of testGroup.ignoredTestCases) { - if (!ignoredTestCase.isParamTest) { - this.ignoredTestNames.push(ignoredTestCase.name); - } else if (ignoredTestCase.paramTestIndex === 0) { - let testCaseName = ignoredTestCase.name; - if (testCaseName.length > 1 && testCaseName.substr(testCaseName.length - 1) === '0') { - testCaseName = testCaseName.substr(0, testCaseName.length - 1); - } - this.ignoredTestNames.push(testCaseName); + if (this.shouldRunSolo) { + if (!testGroup.isIgnored) { + if (testGroup.hasSoloTests || testGroup.isSolo || (testSuite.isSolo && !testSuite.hasSoloGroups)) { + testGroup.isIncluded = true; } } - } - if (this.isExcludedByTag(testGroup, true)) { - testGroup.isIncluded = false; - } else if (this.hasSoloTests && !testGroup.hasSoloTests) { - testGroup.isIncluded = false; - } else if (this.hasSoloGroups && !testGroup.isSolo) { - testGroup.isIncluded = false; } else { testGroup.isIncluded = true; + if (testGroup.isIgnored) { + this.ignoredTestNames.push(testGroup.name + ' [WHOLE GROUP]'); + this.ignoredCount += testGroup.testCases.size; + } } if (testGroup.isIncluded) { - this.groupsCount++; + if (testGroup.isIncluded) { + this.groupsCount++; + } let testCases = [...testGroup.testCases.values()]; for (let testCase of testCases) { - if (testGroup.isIgnored) { - testCase.isIgnored = true; - } - - if (this.isExcludedByTag(testCase, true)) { - testCase.isIncluded = false; - } else if (testCase.isIgnored) { - testCase.isIncluded = true; - } else if (this.hasSoloTests && !testCase.isSolo) { - testCase.isIncluded = false; - } else { - testCase.isIncluded = testGroup.isIncluded || testCase.isSolo; - } - } - - for (let testCase of testGroup.soloTestCases) { - if (this.isExcludedByTag(testCase, true)) { - testCase.isIncluded = false; + testCase.isIncluded = false; + testCase.isIgnored = testCase.isIgnored || testGroup.isIgnored || this.isExcludedByTag(testCase, true); + + if (this.shouldRunSolo) { + if (!testCase.isIgnored) { + if (testGroup.hasSoloTests) { + testCase.isIncluded = testCase.isSolo; + } else if (testGroup.isSolo || testSuite.isSolo) { + testCase.isIncluded = true; + } + } } else { testCase.isIncluded = true; + if (testCase.isIgnored) { + if (!testCase.isParamTest) { + this.ignoredTestNames.push(testCase.name); + } else if (testCase.paramTestIndex === 0) { + let testCaseName = testCase.name; + if (testCaseName.length > 1 && testCaseName.substr(testCaseName.length - 1) === '0') { + testCaseName = testCaseName.substr(0, testCaseName.length - 1); + } + this.ignoredTestNames.push(testCaseName); + } + } } } @@ -183,25 +160,11 @@ export class SessionInfo { } private resetCounts() { - - this.hasSoloTests = false; - this.hasSoloGroups = false; - this.hasSoloSuites = false; + this.shouldRunSolo = false; for (let testSuite of [...this.testSuites.values()]) { - if (testSuite.isValid && !this.isExcludedByTag(testSuite, false)) { - if (testSuite.isSolo) { - this.hasSoloSuites = !this.hasSoloGroups && !this.hasSoloTests; - } - if (testSuite.hasSoloGroups) { - this.hasSoloGroups = !this.hasSoloTests; - } - if (testSuite.hasSoloTests) { - this.hasSoloTests = true; - this.hasSoloGroups = false; - this.hasSoloSuites = false; - } + this.shouldRunSolo = this.shouldRunSolo || testSuite.isSolo || testSuite.hasSoloGroups || testSuite.hasSoloTests; } } this.suitesCount = this.testSuites.size; diff --git a/bsc-plugin/src/lib/rooibos/TestGroup.ts b/bsc-plugin/src/lib/rooibos/TestGroup.ts index 6775d635..e1d67f10 100644 --- a/bsc-plugin/src/lib/rooibos/TestGroup.ts +++ b/bsc-plugin/src/lib/rooibos/TestGroup.ts @@ -22,24 +22,12 @@ export class TestGroup extends TestBlock { public testSuite: TestSuite; public testCases = new Map(); - public ignoredTestCases: TestCase[] = []; - public soloTestCases: TestCase[] = []; public addTestCase(testCase: TestCase) { - this.testCases.set(testCase.name + (testCase.isParamTest ? testCase.paramTestIndex.toString() : ''), testCase); - - if (testCase.isIgnored) { - this.ignoredTestCases.push(testCase); - this.hasIgnoredTests = true; - this.soloTestCases.push(testCase); - } else if (testCase.isSolo) { - this.hasSoloTests = true; - this.soloTestCases.push(testCase); - this.hasAsyncTests = testCase.isAsync; - } else { - this.hasAsyncTests = testCase.isAsync; - } + this.hasIgnoredTests = this.hasIgnoredTests || testCase.isIgnored; + this.hasSoloTests = this.hasSoloTests || testCase.isSolo; + this.hasAsyncTests = this.hasAsyncTests || testCase.isAsync; } public getTestCases(): TestCase[] { diff --git a/bsc-plugin/src/lib/rooibos/TestSuite.ts b/bsc-plugin/src/lib/rooibos/TestSuite.ts index 528d9709..473d0e7c 100644 --- a/bsc-plugin/src/lib/rooibos/TestSuite.ts +++ b/bsc-plugin/src/lib/rooibos/TestSuite.ts @@ -97,18 +97,10 @@ export class TestSuite extends TestBlock { public addGroup(group: TestGroup) { this.testGroups.set(group.name, group); - if (group.ignoredTestCases.length > 0) { - this.hasIgnoredTests = true; - } - if (group.hasSoloTests) { - this.hasSoloTests = true; - } - if (group.hasAsyncTests) { - this.annotation.isAsync = true; - } - if (group.isSolo) { - this.hasSoloGroups = true; - } + this.hasIgnoredTests = this.hasIgnoredTests || group.hasIgnoredTests; + this.hasSoloTests = this.hasSoloTests || group.hasSoloTests; + this.hasSoloGroups = this.hasSoloGroups || group.isSolo; + this.annotation.isAsync = this.annotation.isAsync || group.hasAsyncTests; this.isValid = true; } diff --git a/bsc-plugin/src/plugin.spec.ts b/bsc-plugin/src/plugin.spec.ts index 4d8dd924..b4c27abb 100644 --- a/bsc-plugin/src/plugin.spec.ts +++ b/bsc-plugin/src/plugin.spec.ts @@ -104,6 +104,225 @@ describe('RooibosPlugin', () => { expect(suite.isSolo).to.be.true; }); + it('finds and combines onlys', () => { + program.setFile('source/test.spec.bs', ` + @only + @suite("named1") + class ATest + @describe("groupA") + + @it("is testA1") + function Test() + end function + + @it("is testA2") + function Test() + end function + + @describe("groupAA") + + @it("is testA1") + function Test() + end function + + @it("is testA2") + function Test() + end function + + end class + + @suite("named2") + class BTest + @only + @describe("groupB") + + @it("is testB1") + function Test() + end function + + @it("is testB2") + function Test() + end function + + @describe("groupBB") + + @it("is testBB1") + function Test() + end function + + @it("is testBB2") + function Test() + end function + + end class + + @suite("named3") + class CTest + @describe("groupC") + + @it("is testC1") + function Test() + end function + + @only + @it("is testC2") + function Test() + end function + + @describe("groupCC") + + @it("is testCC1") + function Test() + end function + + @it("is testCC2") + function Test() + end function + + end class + + @suite("named4") + class DTest + @only + @describe("groupD") + + @it("is testD1") + function Test() + end function + + @only + @it("is testD2") + function Test() + end function + + @only + @describe("groupDD") + + @it("is testDD1") + function Test() + end function + + @it("is testDD2") + function Test() + end function + + end class + + @only + @suite("named5") + class ETest + @only + @describe("groupE") + + @it("is testE1") + function Test() + end function + + @only + @it("is testE2") + function Test() + end function + + end class + + @ignore + @suite("named6") + class FTest + @describe("groupF") + + @it("is testF1") + function Test() + end function + + @it("is testF2") + function Test() + end function + + end class + `); + program.validate(); + expect(program.getDiagnostics()).to.be.empty; + expect(plugin.session.sessionInfo.testSuitesToRun).to.not.be.empty; + expect(plugin.session.sessionInfo.testSuitesToRun).to.be.length(5); + + const testSuitesToRun = plugin.session.sessionInfo.testSuitesToRun; + + let suiteOne = testSuitesToRun[0]; + let suiteOneGroups = [...suiteOne.testGroups.values()]; + let suiteOneTests = []; + for (let group of suiteOneGroups) { + suiteOneTests.push(...group.testCases.values()); + } + expect(suiteOne.name).to.equal('named1'); + expect(suiteOneGroups).to.be.length(2); + expect(suiteOneTests).to.be.length(4); + expect(suiteOneGroups.map(group => group.name)).to.eql(['groupA', 'groupAA']); + expect(suiteOneTests.map(test => test.name)).to.eql(['is testA1', 'is testA2', 'is testA1', 'is testA2']); + expect(suiteOne.isSolo).to.be.true; + expect(suiteOne.hasSoloGroups).to.be.false; + expect(suiteOne.hasSoloTests).to.be.false; + + let suiteTwo = testSuitesToRun[1]; + let suiteTwoGroups = [...suiteTwo.testGroups.values()]; + let suiteTwoTests = []; + for (let group of suiteTwoGroups) { + suiteTwoTests.push(...group.testCases.values()); + } + expect(suiteTwo.name).to.equal('named2'); + expect(suiteTwoGroups).to.be.length(2); + expect(suiteTwoTests).to.be.length(4); + expect(suiteTwoGroups.filter(group => group.isIncluded).map(group => group.name)).to.eql(['groupB']); + expect(suiteTwoTests.filter(test => test.isIncluded).map(test => test.name)).to.eql(['is testB1', 'is testB2']); + expect(suiteTwo.isSolo).to.be.false; + expect(suiteTwo.hasSoloGroups).to.be.true; + expect(suiteTwo.hasSoloTests).to.be.false; + + let suiteThree = testSuitesToRun[2]; + let suiteThreeGroups = [...suiteThree.testGroups.values()]; + let suiteThreeTests = []; + for (let group of suiteThreeGroups) { + suiteThreeTests.push(...group.testCases.values()); + } + expect(suiteThree.name).to.equal('named3'); + expect(suiteThreeGroups).to.be.length(2); + expect(suiteThreeTests).to.be.length(4); + expect(suiteThreeGroups.filter(group => group.isIncluded).map(group => group.name)).to.eql(['groupC']); + expect(suiteThreeTests.filter(test => test.isIncluded).map(test => test.name)).to.eql(['is testC2']); + expect(suiteThree.isSolo).to.be.false; + expect(suiteThree.hasSoloGroups).to.be.false; + expect(suiteThree.hasSoloTests).to.be.true; + + let suiteFour = testSuitesToRun[3]; + let suiteFourGroups = [...suiteFour.testGroups.values()]; + let suiteFourTests = []; + for (let group of suiteFourGroups) { + suiteFourTests.push(...group.testCases.values()); + } + expect(suiteFour.name).to.equal('named4'); + expect(suiteFourGroups).to.be.length(2); + expect(suiteFourTests).to.be.length(4); + expect(suiteFourGroups.filter(group => group.isIncluded).map(group => group.name)).to.eql(['groupD', 'groupDD']); + expect(suiteFourTests.filter(test => test.isIncluded).map(test => test.name)).to.eql(['is testD2', 'is testDD1', 'is testDD2']); + expect(suiteFour.isSolo).to.be.false; + expect(suiteFour.hasSoloGroups).to.be.true; + expect(suiteFour.hasSoloTests).to.be.true; + + let suiteFive = testSuitesToRun[4]; + let suiteFiveGroups = [...suiteFive.testGroups.values()]; + let suiteFiveTests = []; + for (let group of suiteFiveGroups) { + suiteFiveTests.push(...group.testCases.values()); + } + expect(suiteFive.name).to.equal('named5'); + expect(suiteFiveGroups).to.be.length(1); + expect(suiteFiveTests).to.be.length(2); + expect(suiteFiveGroups.filter(group => group.isIncluded).map(group => group.name)).to.eql(['groupE']); + expect(suiteFiveTests.filter(test => test.isIncluded).map(test => test.name)).to.eql(['is testE2']); + expect(suiteFive.isSolo).to.be.true; + expect(suiteFive.hasSoloGroups).to.be.true; + expect(suiteFive.hasSoloTests).to.be.true; + }); + it('finds a @async', () => { program.setFile('source/test.spec.bs', ` @async