diff --git a/README.md b/README.md index c1135f1..69d12ca 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Running tests in hypothetical JavaScript engine "X": ``` cd test262; -test262-harness --hostType=X --hostPath=`which X` test/**/*.js +test262-harness --host-type=X --host-path=`which X` test/**/*.js ``` @@ -40,24 +40,26 @@ test262-harness --hostType=X --hostPath=`which X` test/**/*.js | -- | -- | -- | -- | | `-h`, `--help` | Show help & examples | n/a | n/a | | `-v`, `--version` | Print the current version of test262-harness | n/a | n/a | -| `--hostType` | Type of host to run tests in. See [eshost's supported hosts](https://github.com/bterlson/eshost#supported-hosts) for available options. | No | `node` -| `--hostPath` | Path to the host executable. | Yes, if `hostType` is specified | `process.execPath` -| `--hostArgs` | Any additional arguments to pass to the host when invoking it (eg. `--harmony`, `--es6all`, etc). | No | n/a | +| `--host-type` | Type of host to run tests in. See [eshost's supported hosts](https://github.com/bterlson/eshost#supported-hosts) for available options. | No | `node` +| `--host-path` | Path to the host executable. | Yes, if `host-type` is specified | `process.execPath` +| `--host-args` | Any additional arguments to pass to the host when invoking it (eg. `--harmony`, `--es6all`, etc). | No | n/a | | `-t`, `--threads` | Run this many tests in parallel. Note that the browser runners don't work great with t > 1. | No | 1 | | `-r`, `--reporter` | Format of data written to standard output. Currently either `json` or `simple`. | No | `simple` | -| `--features` | Comma-separated list of [`features`](https://github.com/tc39/test262/blob/master/features.txt) to filter for. Example: `--features="BigInt,Atomics"`. | No | n/a | +| `--features` (deprecated) | Comma-separated list of [`features`](https://github.com/tc39/test262/blob/master/features.txt) to filter for. Example: `--features="BigInt,Atomics"`. | No | n/a | +| `--features-include` | Comma-separated list of [`features`](https://github.com/tc39/test262/blob/master/features.txt) to filter for _inclusion_. Example: `--features-include="BigInt,Atomics"` would run only tests that include the features `BigInt` and `Atomics` | No | n/a | +| `--features-exclude` | Comma-separated list of [`features`](https://github.com/tc39/test262/blob/master/features.txt) to filter for _exclusion_. Example: `--features-exclude="BigInt,Atomics"` would run only tests that do not include the features `BigInt` and `Atomics`. **NOTE** exclusions are resolved _after_ inclusions. | No | n/a | | `--reporter-keys` | Comma-separated list of keys to include in output of `json` reporter. | No | n/a | -| `--test262Dir` | Root test262 directory and is used to locate the includes directory. | No | Relative to test files | -| `--includesDir` | Includes directory. | No | Inferred from `test262Dir` or else detected by walking upward from the first test found. | -| `--tempDir` | Directory that `eshost` will create its temp files in (does not affect location of files created by `--saveCompiledTests` and `--saveOnlyFailed` | No | OS Temp Dir | +| `--test262-dir` | Root test262 directory and is used to locate the includes directory. | No | Relative to test files | +| `--includes-dir` | Includes directory. | No | Inferred from `test262-dir` or else detected by walking upward from the first test found. | +| `--tempDir` | Directory that `eshost` will create its temp files in (does not affect location of files created by `--save-compiled-tests` and `--save-only-failed` | No | OS Temp Dir | | `--prelude` | Path to a file to include before every test (useful for testing polyfills for example); supports multiple `--prelude` parameters | No | n/a | | `--timeout` | Set a custom test timeout in milliseconds | No | `10000` | | `--transformer` | Path to module which exports a code transformer function | No | n/a | | `--preprocessor` | Path to module which exports a map function that operates on each Test262Test object before it executed. | No | n/a | -| `--acceptVersion` | Execute tests from a version of Test262 that differs from the versions supported by this utility. This may cause the utility to report invalid test results. | No | Inferred from `test262Dir/package.json` | -| `--saveCompiledTests` | Write the compiled version of `path/to/test.js` as `path/to/test.js...` so that it can be easily re-run under that host. Run `test262-harness --help` for examples. | No | n/a -| `--saveOnlyFailed` | Only save the compiled version of the test if it failed, to help easily repro failed tests (implies `--saveCompiledTests`). | No | n/a -| `--errorForFailures` | Return a non-zero exit code if one or more tests fail. | No | n/a +| `--accept-version` | Execute tests from a version of Test262 that differs from the versions supported by this utility. This may cause the utility to report invalid test results. | No | Inferred from `test262Dir/package.json` | +| `--save-compiled-tests` | Write the compiled version of `path/to/test.js` as `path/to/test.js...` so that it can be easily re-run under that host. Run `test262-harness --help` for examples. | No | n/a +| `--save-only-failed` | Only save the compiled version of the test if it failed, to help easily repro failed tests (implies `--save-compiled-tests`). | No | n/a +| `--error-for-failures` | Return a non-zero exit code if one or more tests fail. | No | n/a ### Preprocessor diff --git a/bin/run.js b/bin/run.js index d83806a..d9c46d2 100755 --- a/bin/run.js +++ b/bin/run.js @@ -75,7 +75,8 @@ if (argv.prelude) { // If using default hostType, hostPath defaults to the current node executable location. let hostType; let hostPath; -let features; +let featuresInclude; +let featuresExclude; if (argv.hostType) { hostType = argv.hostType; @@ -119,8 +120,12 @@ if (argv.preprocessor) { preprocessor = require(preprocessorPath); } -if (argv.features) { - features = argv.features.split(',').map(feature => feature.trim()); +if (argv.features || argv.featuresInclude) { + featuresInclude = (argv.features || argv.featuresInclude).split(',').map(feature => feature.trim()); +} + +if (argv.featuresExclude) { + featuresExclude = argv.featuresExclude.split(',').map(feature => feature.trim()); } // Show help if no arguments provided @@ -141,6 +146,7 @@ if (!test262Dir) { reporterOpts.test262Dir = test262Dir; const remove = path.relative(process.cwd(), test262Dir); + argv._ = argv._.map(p => path.relative(remove, p)); let test262Version; @@ -160,7 +166,7 @@ if (!acceptVersion) { const stream = new TestStream(test262Dir, includesDir, acceptVersion, argv._); -let tests = stream.pipe(filter(hasFeatures)).pipe(map(insertPrelude)); +let tests = stream.pipe(filter(filterByFeatureInclude)).pipe(filter(filterByFeatureExclude)).pipe(map(insertPrelude)); if (preprocessor) { tests = tests.pipe(filter(preprocessor)); @@ -207,9 +213,16 @@ function insertPrelude(test) { return test; } -function hasFeatures(test) { - if (!features) { +function filterByFeatureInclude(test) { + if (!featuresInclude) { + return true; + } + return featuresInclude.filter(feature => (test.attrs.features || []).includes(feature)).length > 0; +} + +function filterByFeatureExclude(test) { + if (!featuresExclude) { return true; } - return features.filter(feature => (test.attrs.features || []).includes(feature)).length > 0; + return featuresExclude.filter(feature => !(test.attrs.features || []).includes(feature)).length > 0; } diff --git a/lib/cli.js b/lib/cli.js index a232682..bc4f685 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -3,12 +3,12 @@ const yargs = require('yargs'); const yargv = yargs .strict() .usage('Usage: test262-harness [options] ') - .describe('hostType', `eshost host type (${supportedHosts.join(", ")}, etc.)`) - .describe('hostPath', 'path to eshost host binary') - .describe('hostArgs', 'command-line arguments to pass to eshost host') - .describe('test262Dir', 'test262 root directory') - .describe('includesDir', 'directory where helper files are found') - .describe('tempDir', 'directory that eshost will create its temp files in (does not affect location of files created by --saveCompiledTests and --saveOnlyFailed') + .describe('host-type', `eshost host type (${supportedHosts.join(", ")}, etc.)`) + .describe('host-path', 'path to eshost host binary') + .describe('host-args', 'command-line arguments to pass to eshost host') + .describe('test262-dir', 'test262 root directory') + .describe('includes-dir', 'directory where helper files are found') + .describe('temp-dir', 'directory that eshost will create its temp files in (does not affect location of files created by --saveCompiledTests and --saveOnlyFailed') .describe('threads', 'number of threads to use') .describe('prelude', 'content to include above each test; supports multiple --prelude parameters') .describe('version', 'print version of test262-harness') @@ -16,6 +16,12 @@ const yargv = yargs .describe('features', 'comma-separated list of features to filter for') .nargs('features', 1) .alias('features', 'f') + .describe('features-exclude', 'comma-separated list of features to filter for exclusion') + .nargs('features-exclude', 1) + .alias('features-exclude', 'fe') + .describe('features-include', 'comma-separated list of features to filter for inclusion') + .nargs('features-include', 1) + .alias('features-include', 'fi') .describe('transformer', 'path to module which exports a code transformer function') .describe('preprocessor', 'path to module which exports a map function that operates on each Test262Test object before it executed') .nargs('prelude', 1) @@ -32,16 +38,16 @@ const yargv = yargs .alias('help', 'h') .describe('timeout', 'test timeout (in ms, default 10000)') .nargs('timeout', 1) - .describe('acceptVersion', 'override for supported Test262 version') - .boolean('saveCompiledTests') - .describe('saveCompiledTests', 'Write the compiled version of path/to/test.js as path/to/test.js... so that it can be easily re-run under that host') - .boolean('saveOnlyFailed') - .describe('saveOnlyFailed', 'Only save the compiled version of the test if it failed, to help easily repro failed tests (implies --saveCompiledTests)') - .describe('errorForFailures', 'Return a non-zero exit code if one or more tests fail') + .describe('accept-version', 'override for supported Test262 version') + .boolean('save-compiled-tests') + .describe('save-compiled-tests', 'Write the compiled version of path/to/test.js as path/to/test.js... so that it can be easily re-run under that host') + .boolean('save-only-failed') + .describe('save-only-failed', 'Only save the compiled version of the test if it failed, to help easily repro failed tests (implies --saveCompiledTests)') + .describe('error-for-failures', 'Return a non-zero exit code if one or more tests fail') .example('test262-harness path/to/test.js') - .example('test262-harness --hostType ch --hostPath path/to/host path/to/test262/test/folder/**/*.js') - .example('test262-harness --hostType ch --hostPath path/to/host --saveCompiledTests path/to/test262/test/folder/**/*.js') - .example('test262-harness --hostType ch --hostPath path/to/host --saveOnlyFailed path/to/test262/test/folder/**/*.js') + .example('test262-harness --host-type ch --host-Path path/to/host path/to/test262/test/folder/**/*.js') + .example('test262-harness --host-type ch --host-Path path/to/host --save-compiled-tests path/to/test262/test/folder/**/*.js') + .example('test262-harness --host-type ch --host-Path path/to/host --save-only-failed path/to/test262/test/folder/**/*.js') .fail((msg, err) => { if (err) { console.error(err.stack); diff --git a/test/collateral/harness/.gitkeep b/test/collateral/harness/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/test/collateral/harness/assert.js b/test/collateral/harness/assert.js new file mode 100644 index 0000000..5fbc122 --- /dev/null +++ b/test/collateral/harness/assert.js @@ -0,0 +1,95 @@ +// Copyright (C) 2017 Ecma International. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +description: | + Collection of assertion functions used throughout test262 +---*/ + +function assert(mustBeTrue, message) { + if (mustBeTrue === true) { + return; + } + + if (message === undefined) { + message = 'Expected true but got ' + String(mustBeTrue); + } + $ERROR(message); +} + +assert._isSameValue = function (a, b) { + if (a === b) { + // Handle +/-0 vs. -/+0 + return a !== 0 || 1 / a === 1 / b; + } + + // Handle NaN vs. NaN + return a !== a && b !== b; +}; + +assert.sameValue = function (actual, expected, message) { + if (assert._isSameValue(actual, expected)) { + return; + } + + if (message === undefined) { + message = ''; + } else { + message += ' '; + } + + message += 'Expected SameValue(«' + String(actual) + '», «' + String(expected) + '») to be true'; + + $ERROR(message); +}; + +assert.notSameValue = function (actual, unexpected, message) { + if (!assert._isSameValue(actual, unexpected)) { + return; + } + + if (message === undefined) { + message = ''; + } else { + message += ' '; + } + + message += 'Expected SameValue(«' + String(actual) + '», «' + String(unexpected) + '») to be false'; + + $ERROR(message); +}; + +assert.throws = function (expectedErrorConstructor, func, message) { + if (typeof func !== "function") { + $ERROR('assert.throws requires two arguments: the error constructor ' + + 'and a function to run'); + return; + } + if (message === undefined) { + message = ''; + } else { + message += ' '; + } + + try { + func(); + } catch (thrown) { + if (typeof thrown !== 'object' || thrown === null) { + message += 'Thrown value was not an object!'; + $ERROR(message); + } else if (thrown.constructor !== expectedErrorConstructor) { + message += 'Expected a ' + expectedErrorConstructor.name + ' but got a ' + thrown.constructor.name; + $ERROR(message); + } + return; + } + + message += 'Expected a ' + expectedErrorConstructor.name + ' to be thrown but no exception was thrown at all'; + $ERROR(message); +}; + +assert.throws.early = function(err, code) { + var wrappedCode = 'function wrapperFn() { ' + code + ' }'; + var ieval = eval; + + assert.throws(err, function() { Function(wrappedCode); }, 'Function: ' + code); +}; diff --git a/test/collateral/harness/doneprintHandle.js b/test/collateral/harness/doneprintHandle.js new file mode 100644 index 0000000..a6ea799 --- /dev/null +++ b/test/collateral/harness/doneprintHandle.js @@ -0,0 +1,22 @@ +// Copyright (C) 2017 Ecma International. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +description: | + +---*/ + +function __consolePrintHandle__(msg) { + print(msg); +} + +function $DONE(error) { + if (error) { + if(typeof error === 'object' && error !== null && 'name' in error) { + __consolePrintHandle__('Test262:AsyncTestFailure:' + error.name + ': ' + error.message); + } else { + __consolePrintHandle__('Test262:AsyncTestFailure:Test262Error: ' + error); + } + } else { + __consolePrintHandle__('Test262:AsyncTestComplete'); + } +} diff --git a/test/collateral/harness/sta.js b/test/collateral/harness/sta.js new file mode 100644 index 0000000..341c21e --- /dev/null +++ b/test/collateral/harness/sta.js @@ -0,0 +1,23 @@ +// Copyright (c) 2012 Ecma International. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +description: | + Provides both: + + - An error class to avoid false positives when testing for thrown exceptions + - A function to explicitly throw an exception using the Test262Error class +---*/ + + +function Test262Error(message) { + this.message = message || ""; +} + +Test262Error.prototype.toString = function () { + return "Test262Error: " + this.message; +}; + +var $ERROR; +$ERROR = function $ERROR(message) { + throw new Test262Error(message); +}; diff --git a/test/collateral/test/async.js b/test/collateral/test/async.js index b0f1d0c..ba06fd3 100644 --- a/test/collateral/test/async.js +++ b/test/collateral/test/async.js @@ -2,6 +2,7 @@ description: Async test expected: pass: true +features: [A] flags: [async] ---*/ diff --git a/test/collateral/test/asyncNegative.js b/test/collateral/test/asyncNegative.js index bd884a7..b1d2c79 100644 --- a/test/collateral/test/asyncNegative.js +++ b/test/collateral/test/asyncNegative.js @@ -5,6 +5,7 @@ negative: type: RangeError expected: pass: true +features: [A,B] flags: [async] ---*/ diff --git a/test/collateral/test/bothStrict.js b/test/collateral/test/bothStrict.js index f069b24..8140cef 100644 --- a/test/collateral/test/bothStrict.js +++ b/test/collateral/test/bothStrict.js @@ -7,6 +7,7 @@ negative: type: ReferenceError expected: pass: true +features: [A] ---*/ var strict; try { x = 1; strict = false;} catch(e) { strict = true } diff --git a/test/collateral/test/negativeMessage.js b/test/collateral/test/negativeMessage.js index 9c60658..df188c5 100644 --- a/test/collateral/test/negativeMessage.js +++ b/test/collateral/test/negativeMessage.js @@ -3,4 +3,5 @@ description: Should report the expected error indicated by the "negative" frontm negative: ExpectedError expected: pass: false +features: [B] ---*/ diff --git a/test/test.js b/test/test.js index 1bfe6e3..8d36a61 100644 --- a/test/test.js +++ b/test/test.js @@ -133,6 +133,42 @@ const tests = [ ], { exitCode: 1 }, ], + [ + [ + '--features-include=A', + 'collateral/test/**/*.js', + ], + { cwd: 'test', recordsCount: 6 }, + ], + [ + [ + '--features-exclude=B', + 'collateral/test/**/*.js', + ], + { cwd: 'test', recordsCount: 14 }, + ], + [ + [ + '--features-include=A,B', + 'collateral/test/**/*.js', + ], + { cwd: 'test', recordsCount: 8 }, + ], + [ + [ + '--features-exclude=A,B', + 'collateral/test/**/*.js', + ], + { cwd: 'test', recordsCount: 16 }, + ], + [ + [ + '--features-include=A', + '--features-exclude=B', + 'collateral/test/**/*.js', + ], + { cwd: 'test', recordsCount: 4 }, + ], ].reduce((accum, a) => { let b = a.slice(); @@ -167,56 +203,63 @@ function validate({ args, records, exitCode, options }) { }); if (options.reporter === 'json') { - records.forEach(record => { - const prelude = options && options.prelude; - const description = prelude ? - `${record.attrs.description} with prelude` : - record.attrs.description; + if (options.recordsCount) { + tap.test('--features-include/exclude', assert => { + assert.equal(records.length, options.recordsCount, 'records.length matches expected records count'); + assert.end(); + }); + } else { + records.forEach(record => { + const prelude = options && options.prelude; + const description = prelude ? + `${record.attrs.description} with prelude` : + record.attrs.description; - tap.test(description, assert => { + tap.test(description, assert => { - if (typeof record.scenario !== 'undefined') { - if (record.contents.startsWith('"use strict"')) { - assert.equal(record.scenario, 'strict mode'); - } else { - assert.equal(record.scenario, 'default'); + if (typeof record.scenario !== 'undefined') { + if (record.contents.startsWith('"use strict"')) { + assert.equal(record.scenario, 'strict mode'); + } else { + assert.equal(record.scenario, 'default'); + } } - } - assert.notEqual(record.attrs.expected, undefined, - 'Test has an "expected" frontmatter'); + assert.notEqual(record.attrs.expected, undefined, + 'Test has an "expected" frontmatter'); - if (!record.attrs.expected) { - // can't do anything else - assert.end(); - return; - } + if (!record.attrs.expected) { + // can't do anything else + assert.end(); + return; + } - assert.equal(record.result.pass, record.attrs.expected.pass, - 'Test passes or fails as expected'); + assert.equal(record.result.pass, record.attrs.expected.pass, + 'Test passes or fails as expected'); - if (record.attrs.expected.message) { - assert.equal(record.result.message, record.attrs.expected.message, - 'Test fails with appropriate message'); - } + if (record.attrs.expected.message) { + assert.equal(record.result.message, record.attrs.expected.message, + 'Test fails with appropriate message'); + } - if (prelude) { - assert.ok(record.rawResult.stdout.includes('prelude a!'), - 'Has prelude-a content'); + if (prelude) { + assert.ok(record.rawResult.stdout.includes('prelude a!'), + 'Has prelude-a content'); - assert.ok(record.rawResult.stdout.includes('prelude b!'), - 'Has prelude-b content'); - } + assert.ok(record.rawResult.stdout.includes('prelude b!'), + 'Has prelude-b content'); + } - if (options.noRawResult) { - assert.equal(record.rawResult, undefined); - } else { - assert.equal(typeof record.rawResult, 'object'); - } + if (options.noRawResult) { + assert.equal(record.rawResult, undefined); + } else { + assert.equal(typeof record.rawResult, 'object'); + } - assert.end(); + assert.end(); + }); }); - }); + } } if (options.reporter === 'simple') {