From cc26951180e629d131a4dd2211b96781c1af18cf Mon Sep 17 00:00:00 2001 From: cjihrig Date: Sat, 13 Jul 2024 11:10:59 -0400 Subject: [PATCH] test_runner: support running tests in process This commit introduces a new --experimental-test-isolation flag that, when set to 'none', causes the test runner to execute all tests in the same process. By default, this is the main test runner process, but if watch mode is enabled, it spawns a separate process that runs all of the tests. The default value of the new flag is 'process', which uses the existing behavior of running each test file in its own child process. It is worth noting that when the isolation mode is 'none', globals and all other top level logic (such as top level before() and after() hooks) is shared among all files. Co-authored-by: Moshe Atlow PR-URL: https://github.com/nodejs/node/pull/53927 Fixes: https://github.com/nodejs/node/issues/51548 Reviewed-By: Benjamin Gruenbaum Reviewed-By: Matteo Collina --- doc/api/cli.md | 21 +- doc/api/test.md | 40 ++- doc/node.1 | 3 + lib/internal/main/test_runner.js | 2 +- lib/internal/test_runner/harness.js | 1 + lib/internal/test_runner/runner.js | 196 ++++++++++--- lib/internal/test_runner/utils.js | 31 +- src/env-inl.h | 3 +- src/node_options.cc | 19 +- src/node_options.h | 1 + .../test-runner/no-isolation/one.test.js | 32 +++ .../test-runner/no-isolation/two.test.js | 30 ++ test/fixtures/test-runner/snapshots/unit-2.js | 11 + test/parallel/test-runner-cli-concurrency.js | 14 + test/parallel/test-runner-cli-timeout.js | 8 + test/parallel/test-runner-cli.js | 271 ++++++++++-------- test/parallel/test-runner-coverage.js | 38 +++ .../test-runner-extraneous-async-activity.js | 18 ++ .../test-runner-force-exit-failure.js | 23 +- .../test-runner-no-isolation-filtering.js | 69 +++++ test/parallel/test-runner-no-isolation.mjs | 47 +++ test/parallel/test-runner-snapshot-tests.js | 72 +++++ 22 files changed, 745 insertions(+), 205 deletions(-) create mode 100644 test/fixtures/test-runner/no-isolation/one.test.js create mode 100644 test/fixtures/test-runner/no-isolation/two.test.js create mode 100644 test/fixtures/test-runner/snapshots/unit-2.js create mode 100644 test/parallel/test-runner-no-isolation-filtering.js create mode 100644 test/parallel/test-runner-no-isolation.mjs diff --git a/doc/api/cli.md b/doc/api/cli.md index 0d2e28d22ec511..755b961fdeb100 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -1091,6 +1091,20 @@ generated as part of the test runner output. If no tests are run, a coverage report is not generated. See the documentation on [collecting code coverage from tests][] for more details. +### `--experimental-test-isolation=mode` + + + +> Stability: 1.0 - Early development + +Configures the type of test isolation used in the test runner. When `mode` is +`'process'`, each test file is run in a separate child process. When `mode` is +`'none'`, all test files run in the same process as the test runner. The default +isolation mode is `'process'`. This flag is ignored if the `--test` flag is not +present. See the [test runner execution model][] section for more information. + ### `--experimental-test-module-mocks` The maximum number of test files that the test runner CLI will execute -concurrently. The default value is `os.availableParallelism() - 1`. +concurrently. If `--experimental-test-isolation` is set to `'none'`, this flag +is ignored and concurrency is one. Otherwise, concurrency defaults to +`os.availableParallelism() - 1`. ### `--test-coverage-exclude` @@ -2361,7 +2377,7 @@ added: v22.3.0 > Stability: 1.0 - Early development -Regenerates the snapshot file used by the test runner for [snapshot testing][]. +Regenerates the snapshot files used by the test runner for [snapshot testing][]. Node.js must be started with the `--experimental-test-snapshots` flag in order to use this functionality. @@ -3534,6 +3550,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12 [snapshot testing]: test.md#snapshot-testing [syntax detection]: packages.md#syntax-detection [test reporters]: test.md#test-reporters +[test runner execution model]: test.md#test-runner-execution-model [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones [tracking issue for user-land snapshots]: https://github.com/nodejs/node/issues/44014 [ways that `TZ` is handled in other environments]: https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html diff --git a/doc/api/test.md b/doc/api/test.md index 60eeb51c954f2c..cbfc9db94bb58c 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -445,18 +445,26 @@ in the [test runner execution model][] section. ### Test runner execution model -Each matching test file is executed in a separate child process. The maximum -number of child processes running at any time is controlled by the -[`--test-concurrency`][] flag. If the child process finishes with an exit code -of 0, the test is considered passing. Otherwise, the test is considered to be a -failure. Test files must be executable by Node.js, but are not required to use -the `node:test` module internally. +When process-level test isolation is enabled, each matching test file is +executed in a separate child process. The maximum number of child processes +running at any time is controlled by the [`--test-concurrency`][] flag. If the +child process finishes with an exit code of 0, the test is considered passing. +Otherwise, the test is considered to be a failure. Test files must be executable +by Node.js, but are not required to use the `node:test` module internally. Each test file is executed as if it was a regular script. That is, if the test file itself uses `node:test` to define tests, all of those tests will be executed within a single application thread, regardless of the value of the `concurrency` option of [`test()`][]. +When process-level test isolation is disabled, each matching test file is +imported into the test runner process. Once all test files have been loaded, the +top level tests are executed with a concurrency of one. Because the test files +are all run within the same context, it is possible for tests to interact with +each other in ways that are not possible when isolation is enabled. For example, +if a test relies on global state, it is possible for that state to be modified +by a test originating from another file. + ## Collecting code coverage > Stability: 1 - Experimental @@ -933,7 +941,7 @@ the [`--experimental-test-snapshots`][] command-line flag. Snapshot files are generated by starting Node.js with the [`--test-update-snapshots`][] command-line flag. A separate snapshot file is generated for each test file. By default, the snapshot file has the same name -as `process.argv[1]` with a `.snapshot` file extension. This behavior can be +as the test file with a `.snapshot` file extension. This behavior can be configured using the `snapshot.setResolveSnapshotPath()` function. Each snapshot assertion corresponds to an export in the snapshot file. @@ -1239,6 +1247,9 @@ added: - v18.9.0 - v16.19.0 changes: + - version: REPLACEME + pr-url: https://github.com/nodejs/node/pull/53927 + description: Added the `isolation` option. - version: v22.6.0 pr-url: https://github.com/nodejs/node/pull/53866 description: Added the `globPatterns` option. @@ -1274,8 +1285,13 @@ changes: * `inspectPort` {number|Function} Sets inspector port of test child process. This can be a number, or a function that takes no arguments and returns a number. If a nullish value is provided, each process gets its own port, - incremented from the primary's `process.debugPort`. - **Default:** `undefined`. + incremented from the primary's `process.debugPort`. This option is ignored + if the `isolation` option is set to `'none'` as no child processes are + spawned. **Default:** `undefined`. + * `isolation` {string} Configures the type of test isolation. If set to + `'process'`, each test file is run in a separate child process. If set to + `'none'`, all test files run in the current process. **Default:** + `'process'`. * `only`: {boolean} If truthy, the test context will only run tests that have the `only` option set * `setup` {Function} A function that accepts the `TestsStream` instance @@ -1727,9 +1743,9 @@ added: v22.3.0 * `fn` {Function} A function used to compute the location of the snapshot file. The function receives the path of the test file as its only argument. If the - `process.argv[1]` is not associated with a file (for example in the REPL), - the input is undefined. `fn()` must return a string specifying the location of - the snapshot file. + test is not associated with a file (for example in the REPL), the input is + undefined. `fn()` must return a string specifying the location of the snapshot + snapshot file. This function is used to customize the location of the snapshot file used for snapshot testing. By default, the snapshot filename is the same as the entry diff --git a/doc/node.1 b/doc/node.1 index 5e809896321641..6f333727ecf194 100644 --- a/doc/node.1 +++ b/doc/node.1 @@ -185,6 +185,9 @@ Enable the experimental node:sqlite module. .It Fl -experimental-test-coverage Enable code coverage in the test runner. . +.It Fl -experimental-test-isolation Ns = Ns Ar mode +Configures the type of test isolation used in the test runner. +. .It Fl -experimental-test-module-mocks Enable module mocking in the test runner. . diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js index cc853da7388821..b1f69b07771ac6 100644 --- a/lib/internal/main/test_runner.js +++ b/lib/internal/main/test_runner.js @@ -21,7 +21,7 @@ markBootstrapComplete(); const options = parseCommandLine(); -if (isUsingInspector()) { +if (isUsingInspector() && options.isolation === 'process') { process.emitWarning('Using the inspector with --test forces running at a concurrency of 1. ' + 'Use the inspectPort option to run with concurrency'); options.concurrency = 1; diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 3c56820cf4e247..1bc6cddabd41a0 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -334,4 +334,5 @@ module.exports = { after: hook('after'), beforeEach: hook('beforeEach'), afterEach: hook('afterEach'), + startSubtestAfterBootstrap, }; diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index a4874d5caead91..b5431221b4ebd9 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -11,6 +11,7 @@ const { ArrayPrototypeMap, ArrayPrototypePush, ArrayPrototypeShift, + ArrayPrototypeSlice, ArrayPrototypeSome, ArrayPrototypeSort, ObjectAssign, @@ -24,6 +25,7 @@ const { StringPrototypeIndexOf, StringPrototypeSlice, StringPrototypeStartsWith, + Symbol, TypedArrayPrototypeGetLength, TypedArrayPrototypeSubarray, } = primordials; @@ -44,18 +46,28 @@ const { ERR_TEST_FAILURE, }, } = require('internal/errors'); +const esmLoader = require('internal/modules/esm/loader'); const { validateArray, validateBoolean, validateFunction, validateObject, + validateOneOf, validateInteger, } = require('internal/validators'); const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector'); const { isRegExp } = require('internal/util/types'); -const { kEmptyObject } = require('internal/util'); +const { pathToFileURL } = require('internal/url'); +const { + createDeferredPromise, + getCWDURL, + kEmptyObject, +} = require('internal/util'); const { kEmitMessage } = require('internal/test_runner/tests_stream'); -const { createTestTree } = require('internal/test_runner/harness'); +const { + createTestTree, + startSubtestAfterBootstrap, +} = require('internal/test_runner/harness'); const { kAborted, kCancelledByParent, @@ -77,7 +89,11 @@ const { triggerUncaughtException, exitCodes: { kGenericUserError }, } = internalBinding('errors'); +let debug = require('internal/util/debuglog').debuglog('test_runner', (fn) => { + debug = fn; +}); +const kIsolatedProcessName = Symbol('kIsolatedProcessName'); const kFilterArgs = ['--test', '--experimental-test-coverage', '--watch']; const kFilterArgValues = ['--test-reporter', '--test-reporter-destination']; const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms']; @@ -130,7 +146,12 @@ function getRunArgs(path, { forceExit, inspectPort, testNamePatterns, testSkipPa if (only === true) { ArrayPrototypePush(argv, '--test-only'); } - ArrayPrototypePush(argv, path); + + if (path === kIsolatedProcessName) { + ArrayPrototypePush(argv, '--test', ...ArrayPrototypeSlice(process.argv, 1)); + } else { + ArrayPrototypePush(argv, path); + } return argv; } @@ -326,7 +347,9 @@ class FileTest extends Test { function runTestFile(path, filesWatcher, opts) { const watchMode = filesWatcher != null; - const subtest = opts.root.createSubtest(FileTest, path, { __proto__: null, signal: opts.signal }, async (t) => { + const testPath = path === kIsolatedProcessName ? '' : path; + const testOpts = { __proto__: null, signal: opts.signal }; + const subtest = opts.root.createSubtest(FileTest, testPath, testOpts, async (t) => { const args = getRunArgs(path, opts); const stdio = ['pipe', 'pipe', 'pipe']; const env = { __proto__: null, ...process.env, NODE_TEST_CONTEXT: 'child-v8' }; @@ -418,10 +441,23 @@ function watchFiles(testFiles, opts) { const filesWatcher = { __proto__: null, watcher, runningProcesses, runningSubtests }; opts.root.harness.watching = true; + async function restartTestFile(file) { + const runningProcess = runningProcesses.get(file); + if (runningProcess) { + runningProcess.kill(); + await once(runningProcess, 'exit'); + } + if (!runningSubtests.size) { + // Reset the topLevel counter + opts.root.harness.counters.topLevel = 0; + } + await runningSubtests.get(file); + runningSubtests.set(file, runTestFile(file, filesWatcher, opts)); + } + watcher.on('changed', ({ owners, eventType }) => { if (!opts.hasFiles && eventType === 'rename') { const updatedTestFiles = createTestFileList(opts.globPatterns); - const newFileName = ArrayPrototypeFind(updatedTestFiles, (x) => !ArrayPrototypeIncludes(testFiles, x)); const previousFileName = ArrayPrototypeFind(testFiles, (x) => !ArrayPrototypeIncludes(updatedTestFiles, x)); @@ -439,25 +475,22 @@ function watchFiles(testFiles, opts) { } - watcher.unfilterFilesOwnedBy(owners); - PromisePrototypeThen(SafePromiseAllReturnVoid(testFiles, async (file) => { - if (!owners.has(file)) { - return; - } - const runningProcess = runningProcesses.get(file); - if (runningProcess) { - runningProcess.kill(); - await once(runningProcess, 'exit'); - } - if (!runningSubtests.size) { - // Reset the topLevel counter - opts.root.harness.counters.topLevel = 0; - } - await runningSubtests.get(file); - runningSubtests.set(file, runTestFile(file, filesWatcher, opts)); - }, undefined, (error) => { - triggerUncaughtException(error, true /* fromPromise */); - })); + if (opts.isolation === 'none') { + PromisePrototypeThen(restartTestFile(kIsolatedProcessName), undefined, (error) => { + triggerUncaughtException(error, true /* fromPromise */); + }); + } else { + watcher.unfilterFilesOwnedBy(owners); + PromisePrototypeThen(SafePromiseAllReturnVoid(testFiles, async (file) => { + if (!owners.has(file)) { + return; + } + + await restartTestFile(file); + }, undefined, (error) => { + triggerUncaughtException(error, true /* fromPromise */); + })); + } }); if (opts.signal) { kResistStopPropagation ??= require('internal/event_target').kResistStopPropagation; @@ -485,6 +518,7 @@ function run(options = kEmptyObject) { files, forceExit, inspectPort, + isolation = 'process', watch, setup, only, @@ -566,6 +600,7 @@ function run(options = kEmptyObject) { throw new ERR_INVALID_ARG_TYPE(name, ['string', 'RegExp'], value); }); } + validateOneOf(isolation, 'options.isolation', ['process', 'none']); const rootTestOptions = { __proto__: null, concurrency, timeout, signal }; const globalOptions = { @@ -576,21 +611,16 @@ function run(options = kEmptyObject) { setup, // This line can be removed when parseCommandLine() is removed here. }; const root = createTestTree(rootTestOptions, globalOptions); - - if (process.env.NODE_TEST_CONTEXT !== undefined) { - process.emitWarning('node:test run() is being called recursively within a test file. skipping running files.'); - root.postRun(); - return root.reporter; - } let testFiles = files ?? createTestFileList(globPatterns); if (shard) { testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1); } - let postRun = () => root.postRun(); - let teardown = () => root.harness.teardown(); + let teardown; + let postRun; let filesWatcher; + let runFiles; const opts = { __proto__: null, root, @@ -602,21 +632,95 @@ function run(options = kEmptyObject) { globPatterns, only, forceExit, + isolation, }; - if (watch) { - filesWatcher = watchFiles(testFiles, opts); - postRun = undefined; - teardown = undefined; - } - const runFiles = () => { - root.harness.bootstrapPromise = null; - root.harness.buildPromise = null; - return SafePromiseAllSettledReturnVoid(testFiles, (path) => { - const subtest = runTestFile(path, filesWatcher, opts); - filesWatcher?.runningSubtests.set(path, subtest); - return subtest; - }); - }; + + if (isolation === 'process') { + if (process.env.NODE_TEST_CONTEXT !== undefined) { + process.emitWarning('node:test run() is being called recursively within a test file. skipping running files.'); + root.postRun(); + return root.reporter; + } + + if (watch) { + filesWatcher = watchFiles(testFiles, opts); + } else { + postRun = () => root.postRun(); + teardown = () => root.harness.teardown(); + } + + runFiles = () => { + root.harness.bootstrapPromise = null; + root.harness.buildPromise = null; + return SafePromiseAllSettledReturnVoid(testFiles, (path) => { + const subtest = runTestFile(path, filesWatcher, opts); + filesWatcher?.runningSubtests.set(path, subtest); + return subtest; + }); + }; + } else if (isolation === 'none') { + if (watch) { + filesWatcher = watchFiles(testFiles, opts); + runFiles = async () => { + root.harness.bootstrapPromise = null; + root.harness.buildPromise = null; + const subtest = runTestFile(kIsolatedProcessName, filesWatcher, opts); + filesWatcher?.runningSubtests.set(kIsolatedProcessName, subtest); + return subtest; + }; + } else { + runFiles = async () => { + const { promise, resolve: finishBootstrap } = createDeferredPromise(); + + await root.runInAsyncScope(async () => { + const parentURL = getCWDURL().href; + const cascadedLoader = esmLoader.getOrInitializeCascadedLoader(); + let topLevelTestCount = 0; + + root.harness.bootstrapPromise = promise; + + for (let i = 0; i < testFiles.length; ++i) { + const testFile = testFiles[i]; + const fileURL = pathToFileURL(testFile); + const parent = i === 0 ? undefined : parentURL; + let threw = false; + let importError; + + root.entryFile = resolve(testFile); + debug('loading test file:', fileURL.href); + try { + await cascadedLoader.import(fileURL, parent, { __proto__: null }); + } catch (err) { + threw = true; + importError = err; + } + + debug( + 'loaded "%s": top level test count before = %d and after = %d', + testFile, + topLevelTestCount, + root.subtests.length, + ); + if (topLevelTestCount === root.subtests.length) { + // This file had no tests in it. Add the placeholder test. + const subtest = root.createSubtest(Test, testFile); + if (threw) { + subtest.fail(importError); + } + startSubtestAfterBootstrap(subtest); + } + + topLevelTestCount = root.subtests.length; + } + }); + + debug('beginning test execution'); + root.entryFile = null; + finishBootstrap(); + root.processPendingSubtests(); + }; + } + } const setupPromise = PromiseResolve(setup?.(root.reporter)); PromisePrototypeThen(PromisePrototypeThen(PromisePrototypeThen(setupPromise, runFiles), postRun), teardown); diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index e6c421ff870bbd..882eb50aa5db47 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -195,11 +195,12 @@ function parseCommandLine() { let coverageExcludeGlobs; let coverageIncludeGlobs; let destinations; - let only; + let isolation; + let only = getOptionValue('--test-only'); let reporters; let shard; - let testNamePatterns; - let testSkipPatterns; + let testNamePatterns = mapPatternFlagToRegExArray('--test-name-pattern'); + let testSkipPatterns = mapPatternFlagToRegExArray('--test-skip-pattern'); let timeout; if (isChildProcessV8) { @@ -230,10 +231,17 @@ function parseCommandLine() { } if (isTestRunner) { + isolation = getOptionValue('--experimental-test-isolation'); timeout = getOptionValue('--test-timeout') || Infinity; - concurrency = getOptionValue('--test-concurrency') || true; - only = false; - testNamePatterns = null; + + if (isolation === 'none') { + concurrency = 1; + } else { + concurrency = getOptionValue('--test-concurrency') || true; + only = false; + testNamePatterns = null; + testSkipPatterns = null; + } const shardOption = getOptionValue('--test-shard'); if (shardOption) { @@ -290,6 +298,7 @@ function parseCommandLine() { coverageIncludeGlobs, destinations, forceExit, + isolation, only, reporters, setup, @@ -305,6 +314,16 @@ function parseCommandLine() { return globalTestOptions; } +function mapPatternFlagToRegExArray(flagName) { + const patterns = getOptionValue(flagName); + + if (patterns?.length > 0) { + return ArrayPrototypeMap(patterns, (re) => convertStringToRegExp(re, flagName)); + } + + return null; +} + function countCompletedTest(test, harness = test.root.harness) { if (test.nesting === 0) { harness.counters.topLevel++; diff --git a/src/env-inl.h b/src/env-inl.h index 203841a25c1160..08fe98e10b7716 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -645,7 +645,8 @@ inline bool Environment::owns_inspector() const { inline bool Environment::should_create_inspector() const { return (flags_ & EnvironmentFlags::kNoCreateInspector) == 0 && - !options_->test_runner && !options_->watch_mode; + !(options_->test_runner && options_->test_isolation == "process") && + !options_->watch_mode; } inline bool Environment::should_wait_for_inspector_frontend() const { diff --git a/src/node_options.cc b/src/node_options.cc index 6c1ee44d9583bb..f88495d0bbbe94 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -143,6 +143,18 @@ void EnvironmentOptions::CheckOptions(std::vector* errors, } if (test_runner) { + if (test_isolation == "none") { + debug_options_.allow_attaching_debugger = true; + } else { + if (test_isolation != "process") { + errors->push_back("invalid value for --experimental-test-isolation"); + } + +#ifndef ALLOW_ATTACHING_DEBUGGER_IN_TEST_RUNNER + debug_options_.allow_attaching_debugger = false; +#endif + } + if (syntax_check_only) { errors->push_back("either --test or --check can be used, not both"); } @@ -159,10 +171,6 @@ void EnvironmentOptions::CheckOptions(std::vector* errors, errors->push_back( "--watch-path cannot be used in combination with --test"); } - -#ifndef ALLOW_ATTACHING_DEBUGGER_IN_TEST_RUNNER - debug_options_.allow_attaching_debugger = false; -#endif } if (watch_mode) { @@ -650,6 +658,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { AddOption("--experimental-test-coverage", "enable code coverage in the test runner", &EnvironmentOptions::test_runner_coverage); + AddOption("--experimental-test-isolation", + "configures the type of test isolation used in the test runner", + &EnvironmentOptions::test_isolation); AddOption("--experimental-test-module-mocks", "enable module mocking in the test runner", &EnvironmentOptions::test_runner_module_mocks); diff --git a/src/node_options.h b/src/node_options.h index 52d57610ed103a..e166b3c5fdf7da 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -191,6 +191,7 @@ class EnvironmentOptions : public Options { std::vector test_reporter_destination; bool test_only = false; bool test_udp_no_try_send = false; + std::string test_isolation = "process"; std::string test_shard; std::vector test_skip_pattern; std::vector coverage_include_pattern; diff --git a/test/fixtures/test-runner/no-isolation/one.test.js b/test/fixtures/test-runner/no-isolation/one.test.js new file mode 100644 index 00000000000000..69e0485a37127b --- /dev/null +++ b/test/fixtures/test-runner/no-isolation/one.test.js @@ -0,0 +1,32 @@ +'use strict'; +const { before, beforeEach, after, afterEach, test, suite } = require('node:test'); + +globalThis.GLOBAL_ORDER = []; + +before(function() { + GLOBAL_ORDER.push(`before one: ${this.name}`); +}); + +beforeEach(function() { + GLOBAL_ORDER.push(`beforeEach one: ${this.name}`); +}); + +after(function() { + GLOBAL_ORDER.push(`after one: ${this.name}`); +}); + +afterEach(function() { + GLOBAL_ORDER.push(`afterEach one: ${this.name}`); +}); + +suite('suite one', function() { + GLOBAL_ORDER.push(this.name); + + test('suite one - test', { only: true }, function() { + GLOBAL_ORDER.push(this.name); + }); +}); + +test('test one', function() { + GLOBAL_ORDER.push(this.name); +}); diff --git a/test/fixtures/test-runner/no-isolation/two.test.js b/test/fixtures/test-runner/no-isolation/two.test.js new file mode 100644 index 00000000000000..50ae6541ce156d --- /dev/null +++ b/test/fixtures/test-runner/no-isolation/two.test.js @@ -0,0 +1,30 @@ +'use strict'; +const { before, beforeEach, after, afterEach, test, suite } = require('node:test'); + +before(function() { + GLOBAL_ORDER.push(`before two: ${this.name}`); +}); + +beforeEach(function() { + GLOBAL_ORDER.push(`beforeEach two: ${this.name}`); +}); + +after(function() { + GLOBAL_ORDER.push(`after two: ${this.name}`); +}); + +afterEach(function() { + GLOBAL_ORDER.push(`afterEach two: ${this.name}`); +}); + +suite('suite two', function() { + GLOBAL_ORDER.push(this.name); + + before(function() { + GLOBAL_ORDER.push(`before suite two: ${this.name}`); + }); + + test('suite two - test', { only: true }, function() { + GLOBAL_ORDER.push(this.name); + }); +}); diff --git a/test/fixtures/test-runner/snapshots/unit-2.js b/test/fixtures/test-runner/snapshots/unit-2.js new file mode 100644 index 00000000000000..311378b2810136 --- /dev/null +++ b/test/fixtures/test-runner/snapshots/unit-2.js @@ -0,0 +1,11 @@ +'use strict'; +const { snapshot, test } = require('node:test'); +const { basename, join } = require('node:path'); + +snapshot.setResolveSnapshotPath((testFile) => { + return join(process.cwd(), `${basename(testFile)}.snapshot`); +}); + +test('has a snapshot', (t) => { + t.assert.snapshot('a snapshot from ' + __filename); +}); diff --git a/test/parallel/test-runner-cli-concurrency.js b/test/parallel/test-runner-cli-concurrency.js index fbabaf08e27279..b2aa0ac6c3c6c5 100644 --- a/test/parallel/test-runner-cli-concurrency.js +++ b/test/parallel/test-runner-cli-concurrency.js @@ -24,3 +24,17 @@ test('concurrency of two', async () => { const cp = spawnSync(process.execPath, args, { cwd, env }); assert.match(cp.stderr.toString(), /concurrency: 2,/); }); + +test('isolation=none uses a concurrency of one', async () => { + const args = ['--test', '--experimental-test-isolation=none']; + const cp = spawnSync(process.execPath, args, { cwd, env }); + assert.match(cp.stderr.toString(), /concurrency: 1,/); +}); + +test('isolation=none overrides --test-concurrency', async () => { + const args = [ + '--test', '--experimental-test-isolation=none', '--test-concurrency=2', + ]; + const cp = spawnSync(process.execPath, args, { cwd, env }); + assert.match(cp.stderr.toString(), /concurrency: 1,/); +}); diff --git a/test/parallel/test-runner-cli-timeout.js b/test/parallel/test-runner-cli-timeout.js index b8998d397fa12c..53a3e4ce7ea48e 100644 --- a/test/parallel/test-runner-cli-timeout.js +++ b/test/parallel/test-runner-cli-timeout.js @@ -18,3 +18,11 @@ test('timeout of 10ms', async () => { const cp = spawnSync(process.execPath, args, { cwd, env }); assert.match(cp.stderr.toString(), /timeout: 10,/); }); + +test('isolation=none uses the --test-timeout flag', async () => { + const args = [ + '--test', '--experimental-test-isolation=none', '--test-timeout=10', + ]; + const cp = spawnSync(process.execPath, args, { cwd, env }); + assert.match(cp.stderr.toString(), /timeout: 10,/); +}); diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js index f165a509c995cc..d2d2eea8809404 100644 --- a/test/parallel/test-runner-cli.js +++ b/test/parallel/test-runner-cli.js @@ -7,92 +7,162 @@ const { join } = require('path'); const fixtures = require('../common/fixtures'); const testFixtures = fixtures.path('test-runner'); -{ - // File not found. - const args = ['--test', 'a-random-file-that-does-not-exist.js']; - const child = spawnSync(process.execPath, args); +for (const isolation of ['none', 'process']) { + { + // File not found. + const args = [ + '--test', + `--experimental-test-isolation=${isolation}`, + 'a-random-file-that-does-not-exist.js', + ]; + const child = spawnSync(process.execPath, args); - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stdout.toString(), ''); - assert.match(child.stderr.toString(), /^Could not find/); -} + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stdout.toString(), ''); + assert.match(child.stderr.toString(), /^Could not find/); + } -{ - // Default behavior. node_modules is ignored. Files that don't match the - // pattern are ignored except in test/ directories. - const args = ['--test']; - const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') }); + { + // Default behavior. node_modules is ignored. Files that don't match the + // pattern are ignored except in test/ directories. + const args = ['--test', `--experimental-test-isolation=${isolation}`]; + const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') }); - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /ok 1 - this should pass/); - assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); - assert.match(stdout, /ok 4 - this should pass/); - assert.match(stdout, /ok 5 - this should be skipped/); - assert.match(stdout, /ok 6 - this should be executed/); -} + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, /not ok 2 - this should fail/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); + assert.match(stdout, /ok 4 - this should pass/); + assert.match(stdout, /ok 5 - this should be skipped/); + assert.match(stdout, /ok 6 - this should be executed/); + } -{ - // Same but with a prototype mutation in require scripts. - const args = ['--require', join(testFixtures, 'protoMutation.js'), '--test']; - const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') }); + { + // Same but with a prototype mutation in require scripts. + const args = [ + '--require', join(testFixtures, 'protoMutation.js'), + '--test', + `--experimental-test-isolation=${isolation}`, + ]; + const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') }); + + const stdout = child.stdout.toString(); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, /not ok 2 - this should fail/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); + assert.match(stdout, /ok 4 - this should pass/); + assert.match(stdout, /ok 5 - this should be skipped/); + assert.match(stdout, /ok 6 - this should be executed/); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + } - const stdout = child.stdout.toString(); - assert.match(stdout, /ok 1 - this should pass/); - assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); - assert.match(stdout, /ok 4 - this should pass/); - assert.match(stdout, /ok 5 - this should be skipped/); - assert.match(stdout, /ok 6 - this should be executed/); - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); -} + { + // User specified files that don't match the pattern are still run. + const args = [ + '--test', + `--experimental-test-isolation=${isolation}`, + join(testFixtures, 'index.js'), + ]; + const child = spawnSync(process.execPath, args, { cwd: testFixtures }); -{ - // User specified files that don't match the pattern are still run. - const args = ['--test', join(testFixtures, 'index.js')]; - const child = spawnSync(process.execPath, args, { cwd: testFixtures }); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert.match(stdout, /not ok 1 - .+index\.js/); + } - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /not ok 1 - .+index\.js/); -} + { + // Searches node_modules if specified. + const args = [ + '--test', + `--experimental-test-isolation=${isolation}`, + join(testFixtures, 'default-behavior/node_modules/*.js'), + ]; + const child = spawnSync(process.execPath, args); -{ - // Searches node_modules if specified. - const args = ['--test', join(testFixtures, 'default-behavior/node_modules/*.js')]; - const child = spawnSync(process.execPath, args); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert.match(stdout, /not ok 1 - .+test-nm\.js/); + } - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /not ok 1 - .+test-nm\.js/); -} + { + // The current directory is used by default. + const args = ['--test', `--experimental-test-isolation=${isolation}`]; + const options = { cwd: join(testFixtures, 'default-behavior') }; + const child = spawnSync(process.execPath, args, options); -{ - // The current directory is used by default. - const args = ['--test']; - const options = { cwd: join(testFixtures, 'default-behavior') }; - const child = spawnSync(process.execPath, args, options); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, /not ok 2 - this should fail/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); + assert.match(stdout, /ok 4 - this should pass/); + assert.match(stdout, /ok 5 - this should be skipped/); + assert.match(stdout, /ok 6 - this should be executed/); + } - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /ok 1 - this should pass/); - assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); - assert.match(stdout, /ok 4 - this should pass/); - assert.match(stdout, /ok 5 - this should be skipped/); - assert.match(stdout, /ok 6 - this should be executed/); + { + // Test combined stream outputs + const args = [ + '--test', + `--experimental-test-isolation=${isolation}`, + 'test/fixtures/test-runner/default-behavior/index.test.js', + 'test/fixtures/test-runner/nested.js', + 'test/fixtures/test-runner/invalid-tap.js', + ]; + const child = spawnSync(process.execPath, args); + + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert.match(stdout, /# Subtest: this should pass/); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, / {2}---/); + assert.match(stdout, / {2}duration_ms: .*/); + assert.match(stdout, / {2}\.\.\./); + + assert.match(stdout, /# Subtest: .+invalid-tap\.js/); + assert.match(stdout, /invalid tap output/); + assert.match(stdout, /ok 2 - .+invalid-tap\.js/); + + assert.match(stdout, /# Subtest: level 0a/); + assert.match(stdout, / {4}# Subtest: level 1a/); + assert.match(stdout, / {4}ok 1 - level 1a/); + assert.match(stdout, / {4}# Subtest: level 1b/); + assert.match(stdout, / {4}not ok 2 - level 1b/); + assert.match(stdout, / {6}code: 'ERR_TEST_FAILURE'/); + assert.match(stdout, / {6}stack: |-'/); + assert.match(stdout, / {8}TestContext\. .*/); + assert.match(stdout, / {4}# Subtest: level 1c/); + assert.match(stdout, / {4}ok 3 - level 1c # SKIP aaa/); + assert.match(stdout, / {4}# Subtest: level 1d/); + assert.match(stdout, / {4}ok 4 - level 1d/); + assert.match(stdout, /not ok 3 - level 0a/); + assert.match(stdout, / {2}error: '1 subtest failed'/); + assert.match(stdout, /# Subtest: level 0b/); + assert.match(stdout, /not ok 4 - level 0b/); + assert.match(stdout, / {2}error: 'level 0b error'/); + assert.match(stdout, /# tests 8/); + assert.match(stdout, /# suites 0/); + assert.match(stdout, /# pass 4/); + assert.match(stdout, /# fail 3/); + assert.match(stdout, /# cancelled 0/); + assert.match(stdout, /# skipped 1/); + assert.match(stdout, /# todo 0/); + } } { @@ -115,57 +185,6 @@ const testFixtures = fixtures.path('test-runner'); } } -{ - // Test combined stream outputs - const args = [ - '--test', - 'test/fixtures/test-runner/default-behavior/index.test.js', - 'test/fixtures/test-runner/nested.js', - 'test/fixtures/test-runner/invalid-tap.js', - ]; - const child = spawnSync(process.execPath, args); - - - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /# Subtest: this should pass/); - assert.match(stdout, /ok 1 - this should pass/); - assert.match(stdout, / {2}---/); - assert.match(stdout, / {2}duration_ms: .*/); - assert.match(stdout, / {2}\.\.\./); - - assert.match(stdout, /# Subtest: .+invalid-tap\.js/); - assert.match(stdout, /# invalid tap output/); - assert.match(stdout, /ok 2 - .+invalid-tap\.js/); - - assert.match(stdout, /# Subtest: level 0a/); - assert.match(stdout, / {4}# Subtest: level 1a/); - assert.match(stdout, / {4}ok 1 - level 1a/); - assert.match(stdout, / {4}# Subtest: level 1b/); - assert.match(stdout, / {4}not ok 2 - level 1b/); - assert.match(stdout, / {6}code: 'ERR_TEST_FAILURE'/); - assert.match(stdout, / {6}stack: |-'/); - assert.match(stdout, / {8}TestContext\. .*/); - assert.match(stdout, / {4}# Subtest: level 1c/); - assert.match(stdout, / {4}ok 3 - level 1c # SKIP aaa/); - assert.match(stdout, / {4}# Subtest: level 1d/); - assert.match(stdout, / {4}ok 4 - level 1d/); - assert.match(stdout, /not ok 3 - level 0a/); - assert.match(stdout, / {2}error: '1 subtest failed'/); - assert.match(stdout, /# Subtest: level 0b/); - assert.match(stdout, /not ok 4 - level 0b/); - assert.match(stdout, / {2}error: 'level 0b error'/); - assert.match(stdout, /# tests 8/); - assert.match(stdout, /# suites 0/); - assert.match(stdout, /# pass 4/); - assert.match(stdout, /# fail 3/); - assert.match(stdout, /# cancelled 0/); - assert.match(stdout, /# skipped 1/); - assert.match(stdout, /# todo 0/); -} - { // Test user logging in tests. const args = [ diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index 8a6cb392de2585..6cda6d2d1e090f 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -187,6 +187,44 @@ test('coverage is combined for multiple processes', skipIfNoInspector, () => { assert.strictEqual(result.status, 0); }); +test('coverage works with isolation=none', skipIfNoInspector, () => { + // There is a bug in coverage calculation. The branch % in the common.js + // fixture is different depending on the test isolation mode. The 'none' mode + // is closer to what c8 reports here, so the bug is likely in the code that + // merges reports from different processes. + let report = [ + '# start of coverage report', + '# -------------------------------------------------------------------', + '# file | line % | branch % | funcs % | uncovered lines', + '# -------------------------------------------------------------------', + '# common.js | 89.86 | 68.42 | 100.00 | 8 13-14 18 34-35 53', + '# first.test.js | 83.33 | 100.00 | 50.00 | 5-6', + '# second.test.js | 100.00 | 100.00 | 100.00 | ', + '# third.test.js | 100.00 | 100.00 | 100.00 | ', + '# -------------------------------------------------------------------', + '# all files | 92.11 | 76.00 | 88.89 |', + '# -------------------------------------------------------------------', + '# end of coverage report', + ].join('\n'); + + if (common.isWindows) { + report = report.replaceAll('/', '\\'); + } + + const fixture = fixtures.path('v8-coverage', 'combined_coverage'); + const args = [ + '--test', '--experimental-test-coverage', '--test-reporter', 'tap', '--experimental-test-isolation=none', + ]; + const result = spawnSync(process.execPath, args, { + env: { ...process.env, NODE_TEST_TMPDIR: tmpdir.path }, + cwd: fixture, + }); + + assert.strictEqual(result.stderr.toString(), ''); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.status, 0); +}); + test('coverage reports on lines, functions, and branches', skipIfNoInspector, async (t) => { const fixture = fixtures.path('test-runner', 'coverage.js'); const child = spawnSync(process.execPath, diff --git a/test/parallel/test-runner-extraneous-async-activity.js b/test/parallel/test-runner-extraneous-async-activity.js index 68db109b292f15..23f3194e02f106 100644 --- a/test/parallel/test-runner-extraneous-async-activity.js +++ b/test/parallel/test-runner-extraneous-async-activity.js @@ -48,3 +48,21 @@ const { spawnSync } = require('child_process'); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); } + +{ + const child = spawnSync(process.execPath, [ + '--test', + '--experimental-test-isolation=none', + fixtures.path('test-runner', 'async-error-in-test-hook.mjs'), + ]); + const stdout = child.stdout.toString(); + assert.match(stdout, /^# Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# pass 1$/m); + assert.match(stdout, /^# fail 0$/m); + assert.match(stdout, /^# cancelled 0$/m); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); +} diff --git a/test/parallel/test-runner-force-exit-failure.js b/test/parallel/test-runner-force-exit-failure.js index 1fff8f30d7e038..ce1f3208c5b4e6 100644 --- a/test/parallel/test-runner-force-exit-failure.js +++ b/test/parallel/test-runner-force-exit-failure.js @@ -4,12 +4,21 @@ const { match, doesNotMatch, strictEqual } = require('node:assert'); const { spawnSync } = require('node:child_process'); const fixtures = require('../common/fixtures'); const fixture = fixtures.path('test-runner/throws_sync_and_async.js'); -const r = spawnSync(process.execPath, ['--test', '--test-force-exit', fixture]); -strictEqual(r.status, 1); -strictEqual(r.signal, null); -strictEqual(r.stderr.toString(), ''); +for (const isolation of ['none', 'process']) { + const args = [ + '--test', + '--test-force-exit', + `--experimental-test-isolation=${isolation}`, + fixture, + ]; + const r = spawnSync(process.execPath, args); -const stdout = r.stdout.toString(); -match(stdout, /error: 'fails'/); -doesNotMatch(stdout, /this should not have a chance to be thrown/); + strictEqual(r.status, 1); + strictEqual(r.signal, null); + strictEqual(r.stderr.toString(), ''); + + const stdout = r.stdout.toString(); + match(stdout, /error: 'fails'/); + doesNotMatch(stdout, /this should not have a chance to be thrown/); +} diff --git a/test/parallel/test-runner-no-isolation-filtering.js b/test/parallel/test-runner-no-isolation-filtering.js new file mode 100644 index 00000000000000..f8fba1cbfffbef --- /dev/null +++ b/test/parallel/test-runner-no-isolation-filtering.js @@ -0,0 +1,69 @@ +'use strict'; +require('../common'); +const fixtures = require('../common/fixtures'); +const assert = require('node:assert'); +const { spawnSync } = require('node:child_process'); +const { test } = require('node:test'); + +const fixture1 = fixtures.path('test-runner', 'no-isolation', 'one.test.js'); +const fixture2 = fixtures.path('test-runner', 'no-isolation', 'two.test.js'); + +test('works with --test-only', () => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--test-only', + fixture1, + fixture2, + ]; + const child = spawnSync(process.execPath, args); + const stdout = child.stdout.toString(); + + assert.strictEqual(child.status, 0); + assert.strictEqual(child.signal, null); + assert.match(stdout, /# tests 2/); + assert.match(stdout, /# suites 2/); + assert.match(stdout, /# pass 2/); + assert.match(stdout, /ok 1 - suite one/); + assert.match(stdout, /ok 1 - suite one - test/); + assert.match(stdout, /ok 2 - suite two/); + assert.match(stdout, /ok 1 - suite two - test/); +}); + +test('works with --test-name-pattern', () => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--test-name-pattern=/test one/', + fixture1, + fixture2, + ]; + const child = spawnSync(process.execPath, args); + const stdout = child.stdout.toString(); + + assert.strictEqual(child.status, 0); + assert.strictEqual(child.signal, null); + assert.match(stdout, /# tests 1/); + assert.match(stdout, /# suites 0/); + assert.match(stdout, /# pass 1/); + assert.match(stdout, /ok 1 - test one/); +}); + +test('works with --test-skip-pattern', () => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--test-skip-pattern=/one/', + fixture1, + fixture2, + ]; + const child = spawnSync(process.execPath, args); + const stdout = child.stdout.toString(); + + assert.strictEqual(child.status, 0); + assert.strictEqual(child.signal, null); + assert.match(stdout, /# tests 1/); + assert.match(stdout, /# suites 1/); + assert.match(stdout, /# pass 1/); + assert.match(stdout, /ok 1 - suite two - test/); +}); diff --git a/test/parallel/test-runner-no-isolation.mjs b/test/parallel/test-runner-no-isolation.mjs new file mode 100644 index 00000000000000..60b0c962e6779b --- /dev/null +++ b/test/parallel/test-runner-no-isolation.mjs @@ -0,0 +1,47 @@ +import { allowGlobals, mustCall, mustNotCall } from '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import { deepStrictEqual } from 'node:assert'; +import { run } from 'node:test'; + +const stream = run({ + files: [ + fixtures.path('test-runner', 'no-isolation', 'one.test.js'), + fixtures.path('test-runner', 'no-isolation', 'two.test.js'), + ], + isolation: 'none', +}); + +stream.on('test:fail', mustNotCall()); +stream.on('test:pass', mustCall(5)); +// eslint-disable-next-line no-unused-vars +for await (const _ of stream); +allowGlobals(globalThis.GLOBAL_ORDER); +deepStrictEqual(globalThis.GLOBAL_ORDER, [ + 'before one: ', + 'suite one', + 'before two: ', + 'suite two', + + 'beforeEach one: suite one - test', + 'beforeEach two: suite one - test', + 'suite one - test', + 'afterEach one: suite one - test', + 'afterEach two: suite one - test', + + 'beforeEach one: test one', + 'beforeEach two: test one', + 'test one', + 'afterEach one: test one', + 'afterEach two: test one', + + 'before suite two: suite two', + + 'beforeEach one: suite two - test', + 'beforeEach two: suite two - test', + 'suite two - test', + 'afterEach one: suite two - test', + 'afterEach two: suite two - test', + + 'after one: ', + 'after two: ', +]); diff --git a/test/parallel/test-runner-snapshot-tests.js b/test/parallel/test-runner-snapshot-tests.js index e00019ef49d4f6..62ebdd3cade2fb 100644 --- a/test/parallel/test-runner-snapshot-tests.js +++ b/test/parallel/test-runner-snapshot-tests.js @@ -339,3 +339,75 @@ test('t.assert.snapshot()', async (t) => { t.assert.match(child.stdout, /fail 0/); }); }); + +test('snapshots from multiple files (isolation=none)', async (t) => { + tmpdir.refresh(); + + const fixture = fixtures.path('test-runner', 'snapshots', 'unit.js'); + const fixture2 = fixtures.path('test-runner', 'snapshots', 'unit-2.js'); + + await t.test('fails prior to snapshot generation', async (t) => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--experimental-test-snapshots', + fixture, + fixture2, + ]; + const child = await common.spawnPromisified( + process.execPath, + args, + { cwd: tmpdir.path }, + ); + + t.assert.strictEqual(child.code, 1); + t.assert.strictEqual(child.signal, null); + t.assert.match(child.stdout, /# tests 6/); + t.assert.match(child.stdout, /# pass 0/); + t.assert.match(child.stdout, /# fail 6/); + t.assert.match(child.stdout, /Missing snapshots/); + }); + + await t.test('passes when regenerating snapshots', async (t) => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--experimental-test-snapshots', + '--test-update-snapshots', + fixture, + fixture2, + ]; + const child = await common.spawnPromisified( + process.execPath, + args, + { cwd: tmpdir.path }, + ); + + t.assert.strictEqual(child.code, 0); + t.assert.strictEqual(child.signal, null); + t.assert.match(child.stdout, /tests 6/); + t.assert.match(child.stdout, /pass 6/); + t.assert.match(child.stdout, /fail 0/); + }); + + await t.test('passes when snapshots exist', async (t) => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--experimental-test-snapshots', + fixture, + fixture2, + ]; + const child = await common.spawnPromisified( + process.execPath, + args, + { cwd: tmpdir.path }, + ); + + t.assert.strictEqual(child.code, 0); + t.assert.strictEqual(child.signal, null); + t.assert.match(child.stdout, /tests 6/); + t.assert.match(child.stdout, /pass 6/); + t.assert.match(child.stdout, /fail 0/); + }); +});