Skip to content

Commit

Permalink
[WIP] [BREAKING CHANGE] Spec: Combine the "Test" and "Suite" concepts
Browse files Browse the repository at this point in the history 
This is to accomodate node-tap and tape, which allow for child
tests to be associated directly with other assertion-holding tests
(as opposed to having tests only contain assertions, and suites
contain only tests and other suites).
Ref #126.

It also allows for future compatibility with TAP 14, which currently
has no concept of test groups or test suites, but is considering
the addition of "sub tests".
Ref TestAnything/testanything.github.io#36.

Also:

- Define "Adapter" and "Producer" terms.

- Refer mostly to producers and reporters, instead of frameworks,
  runners, or adapters.

- Remove mention that the spec is for reporting information about
  JavaScript test frameworks, it can report information about any
  kind of test that can be represented in its structure of JSON
  messages.
  Instead, do clarify that the spec defines a JavaScript-based
  API of producers and reporters.

Thought dump:

In aggregation, simplify status to failed/passed only,
if something has only todo or skipped children, don't
propagate this like we did with suites, but cast it down
to only failed/passed, as we did with "run" before.

This is because, with the "suite" concept gone, we can't
assume that test parents only contained other tests, they
may have their own assertions. As such, a parent with only
two skipped children doesn't mean the parent can therefore
be marked as skipped, rather it will be marked as passed,
assuming no errors/failures reported.

This affects the adapters for QUnit/Mocha/Jasmine, but when
frameworks implement this themselves, they can of course have
know if an entire suite was known to have been explicitly skipped
in which case it can mark that accordingly.
  • Loading branch information
@Krinkle
Krinkle committed Jan 17, 2021
1 parent e9411f1 commit 12aa08f
Show file tree
Hide file tree
Showing 11 changed files with 488 additions and 624 deletions.
115 changes: 60 additions & 55 deletions lib/adapters/JasmineAdapter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ module.exports = class JasmineAdapter extends EventEmitter {
// NodeJS or browser
this.env = jasmine.env || jasmine.getEnv();

this.suiteStarts = {};
this.suiteChildren = {};
this.suiteEnds = {};
this.suiteEnds = [];
this.testStarts = {};
this.testEnds = {};

Expand Down Expand Up @@ -56,93 +55,90 @@ module.exports = class JasmineAdapter extends EventEmitter {

return {
name: testStart.name,
suiteName: testStart.suiteName,
parentName: testStart.parentName,
fullName: testStart.fullName.slice(),
status: (result.status === 'pending') ? 'skipped' : result.status,
// TODO: Jasmine 3.4+ has result.duration, use it.
// Note that result.duration uses 0 instead of null for a 'skipped' test.
runtime: (result.status === 'pending') ? null : (new Date() - this.startTime),
errors,
assertions
};
}

/**
* Convert a Jasmine SuiteResult for CRI 'runStart' or 'suiteStart' event data.
* Traverse the Jasmine structured returned by `this.env.topSuite()`
* in order to extract the child-parent relations and full names.
*
* Jasmine provides details about childSuites and tests only in the structure
* returned by "this.env.topSuite()".
*/
createSuiteStart (result, parentNames) {
processSuite (result, parentNames, parentIds) {
const isGlobalSuite = (result.description === 'Jasmine__TopLevel__Suite');

const name = isGlobalSuite ? null : result.description;
const fullName = parentNames.slice();
const tests = [];
const childSuites = [];

if (!isGlobalSuite) {
fullName.push(result.description);
fullName.push(name);
}

parentIds.push(result.id);
this.suiteChildren[result.id] = [];

result.children.forEach((child) => {
this.testStarts[child.id] = {
name: child.description,
parentName: name,
fullName: [...fullName, child.description]
};

if (child.id.indexOf('suite') === 0) {
childSuites.push(this.createSuiteStart(child, fullName));
this.processSuite(child, fullName.slice(), parentIds.slice());
} else {
const testStart = {
name: child.description,
suiteName: name,
fullName: [...fullName, child.description]
};
tests.push(testStart);
this.testStarts[child.id] = testStart;
// Update flat list of test children
parentIds.forEach((id) => {
this.suiteChildren[id].push(child.id);
});
}
});

const helperData = helpers.collectSuiteStartData(tests, childSuites);
const suiteStart = {
name,
fullName,
tests,
childSuites,
testCounts: helperData.testCounts
};
this.suiteStarts[result.id] = suiteStart;
this.suiteChildren[result.id] = result.children.map(child => child.id);
return suiteStart;
}

createSuiteEnd (suiteStart, result) {
const tests = [];
const childSuites = [];
this.suiteChildren[result.id].forEach((childId) => {
if (childId.indexOf('suite') === 0) {
childSuites.push(this.suiteEnds[childId]);
} else {
tests.push(this.testEnds[childId]);
}
});
createSuiteEnd (testStart, result) {
const tests = this.suiteChildren[result.id].map((testId) => this.testEnds[testId]);

const helperData = helpers.collectSuiteEndData(tests, childSuites);
const helperData = helpers.aggregateTests(tests);
return {
name: suiteStart.name,
fullName: suiteStart.fullName,
tests,
childSuites,
name: testStart.name,
parentName: testStart.parentName,
fullName: testStart.fullName,
// Jasmine has result.status, but does not propagate 'todo' or 'skipped'
status: helperData.status,
testCounts: helperData.testCounts,
// Jasmine 3.4+ has result.duration, but uses 0 instead of null
// when 'skipped' is skipped.
runtime: helperData.status === 'skipped' ? null : (result.duration || helperData.runtime)
runtime: result.duration || helperData.runtime,
errors: [],
assertions: []
};
}

onJasmineStarted () {
this.globalSuite = this.createSuiteStart(this.env.topSuite(), []);
this.emit('runStart', this.globalSuite);
this.processSuite(this.env.topSuite(), [], []);

let total = 0;
this.env.topSuite().children.forEach(function countChild (child) {
total++;
if (child.id.indexOf('suite') === 0) {
child.children.forEach(countChild);
}
});

this.emit('runStart', {
name: null,
counts: {
total: total
}
});
}

onSuiteStarted (result) {
this.emit('suiteStart', this.suiteStarts[result.id]);
this.emit('testStart', this.testStarts[result.id]);
}

onSpecStarted (result) {
Expand All @@ -156,11 +152,20 @@ module.exports = class JasmineAdapter extends EventEmitter {
}

onSuiteDone (result) {
this.suiteEnds[result.id] = this.createSuiteEnd(this.suiteStarts[result.id], result);
this.emit('suiteEnd', this.suiteEnds[result.id]);
const suiteEnd = this.createSuiteEnd(this.testStarts[result.id], result);
this.suiteEnds.push(suiteEnd);
this.emit('testEnd', suiteEnd);
}

onJasmineDone (doneInfo) {
this.emit('runEnd', this.createSuiteEnd(this.globalSuite, this.env.topSuite()));
const topSuite = this.env.topSuite();
const tests = this.suiteChildren[topSuite.id].map((testId) => this.testEnds[testId]);
const helperData = helpers.aggregateTests([...tests, ...this.suiteEnds]);
this.emit('runEnd', {
name: null,
status: helperData.status,
counts: helperData.counts,
runtime: helperData.runtime
});
}
};
86 changes: 56 additions & 30 deletions lib/adapters/MochaAdapter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,14 @@ module.exports = class MochaAdapter extends EventEmitter {
super();

this.errors = null;
this.finalRuntime = 0;
this.finalCounts = {
passed: 0,
failed: 0,
skipped: 0,
todo: 0,
total: 0
};

// Mocha will instantiate the given function as a class, even if you only need a callback.
// As such, it can't be an arrow function as those throw TypeError when instantiated.
Expand All @@ -27,40 +35,37 @@ module.exports = class MochaAdapter extends EventEmitter {
convertToSuiteStart (mochaSuite) {
return {
name: mochaSuite.title,
fullName: this.titlePath(mochaSuite),
tests: mochaSuite.tests.map(this.convertTest.bind(this)),
childSuites: mochaSuite.suites.map(this.convertToSuiteStart.bind(this)),
testCounts: {
total: mochaSuite.total()
}
parentName: (mochaSuite.parent && !mochaSuite.parent.root) ? mochaSuite.parent.title : null,
fullName: this.titlePath(mochaSuite)
};
}

convertToSuiteEnd (mochaSuite) {
const tests = mochaSuite.tests.map(this.convertTest.bind(this));
const childSuites = mochaSuite.suites.map(this.convertToSuiteEnd.bind(this));
const helperData = helpers.collectSuiteEndData(tests, childSuites);
const helperData = helpers.aggregateTests([...tests, ...childSuites]);

return {
name: mochaSuite.title,
parentName: (mochaSuite.parent && !mochaSuite.parent.root) ? mochaSuite.parent.title : null,
fullName: this.titlePath(mochaSuite),
tests,
childSuites,
status: helperData.status,
testCounts: helperData.testCounts,
runtime: helperData.runtime
runtime: helperData.runtime,
errors: [],
assertions: []
};
}

convertTest (mochaTest) {
let suiteName;
let parentName;
let fullName;
if (!mochaTest.parent.root) {
suiteName = mochaTest.parent.title;
parentName = mochaTest.parent.title;
fullName = this.titlePath(mochaTest.parent);
// Add also the test name.
fullName.push(mochaTest.title);
} else {
suiteName = null;
parentName = null;
fullName = [mochaTest.title];
}

Expand All @@ -73,21 +78,23 @@ module.exports = class MochaAdapter extends EventEmitter {
message: error.message || error.toString(),
stack: error.stack
}));
const status = (mochaTest.state === undefined) ? 'skipped' : mochaTest.state;
const runtime = (mochaTest.duration === undefined) ? null : mochaTest.duration;

return {
name: mochaTest.title,
suiteName,
parentName,
fullName,
status: (mochaTest.state === undefined) ? 'skipped' : mochaTest.state,
runtime: (mochaTest.duration === undefined) ? null : mochaTest.duration,
status,
runtime,
errors,
assertions: errors
};
} else {
// It is a "test start".
return {
name: mochaTest.title,
suiteName,
parentName,
fullName
};
}
Expand All @@ -112,15 +119,24 @@ module.exports = class MochaAdapter extends EventEmitter {
}

onStart () {
const globalSuiteStart = this.convertToSuiteStart(this.runner.suite);
globalSuiteStart.name = null;

this.emit('runStart', globalSuiteStart);
// total is all tests + all suites
// each suite gets a CRI "test" wrapper
let total = this.runner.suite.total();
this.runner.suite.suites.forEach(function addSuites (suite) {
total++;
suite.suites.forEach(addSuites);
});
this.emit('runStart', {
name: null,
counts: {
total: total
}
});
}

onSuite (mochaSuite) {
if (!mochaSuite.root) {
this.emit('suiteStart', this.convertToSuiteStart(mochaSuite));
this.emit('testStart', this.convertToSuiteStart(mochaSuite));
}
}

Expand Down Expand Up @@ -148,19 +164,29 @@ module.exports = class MochaAdapter extends EventEmitter {
// and status are already attached to the test, but the errors are not.
mochaTest.errors = this.errors;

this.emit('testEnd', this.convertTest(mochaTest));
const testEnd = this.convertTest(mochaTest);
this.emit('testEnd', testEnd);
this.finalCounts.total++;
this.finalCounts[testEnd.status]++;
this.finalRuntime += testEnd.runtime || 0;
}

onSuiteEnd (mochaSuite) {
if (!mochaSuite.root) {
this.emit('suiteEnd', this.convertToSuiteEnd(mochaSuite));
const testEnd = this.convertToSuiteEnd(mochaSuite);
this.emit('testEnd', testEnd);
this.finalCounts.total++;
this.finalCounts[testEnd.status]++;
this.finalRuntime += testEnd.runtime || 0;
}
}

onEnd () {
const globalSuiteEnd = this.convertToSuiteEnd(this.runner.suite);
globalSuiteEnd.name = null;

this.emit('runEnd', globalSuiteEnd);
onEnd (details) {
this.emit('runEnd', {
name: null,
status: this.finalCounts.failed > 0 ? 'failed' : 'passed',
counts: this.finalCounts,
runtime: this.finalRuntime
});
}
};
Loading

0 comments on commit 12aa08f

Please sign in to comment.