diff --git a/benchmark/test_runner/multi-files.js b/benchmark/test_runner/multi-files.js new file mode 100644 index 00000000000000..83c8f11b460804 --- /dev/null +++ b/benchmark/test_runner/multi-files.js @@ -0,0 +1,27 @@ +'use strict'; +const common = require('../common'); + +const bench = common.createBenchmark(main, { + patterns: ['test/fixtures/test-runner/**/*.?(c|m)js', 'test/parallel/test-runner-*'], + concurrency: ['yes', 'no'], +}, { + flags: ['--expose-internals'], +}); + + +function main({ patterns, concurrency }) { + const { run } = require('node:test'); + const { Glob } = require('internal/fs/glob'); + const glob = new Glob([patterns]); + const files = glob.globSync().filter((f) => !f.includes('never_ending') && !f.includes('watch-mode')); + concurrency = concurrency === 'yes'; + let tests = 0; + + bench.start(); + (async function() { + const stream = run({ concurrency, files }); + for await (const { type } of stream) { + if (type === 'test:start') tests++; + } + })().then(() => bench.end(tests)); +} diff --git a/benchmark/test_runner/suite-tests.js b/benchmark/test_runner/suite-tests.js index 88b2aa1c74b19e..5d4a4b1205beed 100644 --- a/benchmark/test_runner/suite-tests.js +++ b/benchmark/test_runner/suite-tests.js @@ -1,14 +1,11 @@ 'use strict'; const common = require('../common'); -const { finished } = require('node:stream/promises'); - -const reporter = require('../fixtures/empty-test-reporter'); const { describe, it } = require('node:test'); const bench = common.createBenchmark(main, { numberOfSuites: [10, 100], - testsPerSuite: [10, 100, 1000], + testsPerSuite: [10, 100], testType: ['sync', 'async'], concurrency: ['yes', 'no'], }, { @@ -52,8 +49,6 @@ async function run({ numberOfSuites, testsPerSuite, testType, concurrency }) { } } - await finished(reporter); - return numberOfSuites * testsPerSuite; } diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 357347627fcc2b..bb8910ba402f13 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -2,7 +2,6 @@ const { ArrayPrototypeForEach, FunctionPrototypeBind, - PromiseResolve, SafeMap, } = primordials; const { getCallerLocation } = internalBinding('util'); @@ -23,12 +22,13 @@ const { parseCommandLine, setupTestReporters, } = require('internal/test_runner/utils'); +const { setImmediate } = require('timers'); const { bigint: hrtime } = process.hrtime; const testResources = new SafeMap(); function createTestTree(options = kEmptyObject) { - return setup(new Test({ __proto__: null, ...options, name: '' })); + return setup(new Suite({ __proto__: null, ...options, name: '' })); } function createProcessEventHandler(eventName, rootTest) { @@ -198,24 +198,24 @@ function getGlobalRoot() { } }); reportersSetup = setupTestReporters(globalRoot); + setImmediate(() => globalRoot.run()); } return globalRoot; } -async function startSubtest(subtest) { +async function startSubtest(subtest, parent) { await reportersSetup; getGlobalRoot().harness.bootstrapComplete = true; - await subtest.start(); + if (!(parent instanceof Suite) || (parent.parent === null && parent.endTime !== null)) { + await subtest.start(); + } } function runInParentContext(Factory) { function run(name, options, fn, overrides) { const parent = testResources.get(executionAsyncId()) || getGlobalRoot(); const subtest = parent.createSubtest(Factory, name, options, fn, overrides); - if (!(parent instanceof Suite)) { - return startSubtest(subtest); - } - return PromiseResolve(); + return startSubtest(subtest, parent); } const test = (name, options, fn) => { diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index 51d01114696030..da375b8a873f20 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -16,7 +16,6 @@ const { PromisePrototypeThen, SafePromiseAll, SafePromiseAllReturnVoid, - SafePromiseAllSettledReturnVoid, PromiseResolve, SafeMap, SafeSet, @@ -378,7 +377,9 @@ function runTestFile(path, root, inspectPort, filesWatcher, testNamePatterns) { throw err; } }); - return subtest.start(); + if (filesWatcher) { + return subtest.start(); + } } function watchFiles(testFiles, root, inspectPort, signal, testNamePatterns) { @@ -476,22 +477,27 @@ function run(options) { testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1); } - let postRun = () => root.postRun(); let filesWatcher; if (watch) { filesWatcher = watchFiles(testFiles, root, inspectPort, signal, testNamePatterns); - postRun = undefined; } + const runFiles = () => { root.harness.bootstrapComplete = true; - return SafePromiseAllSettledReturnVoid(testFiles, (path) => { - const subtest = runTestFile(path, root, inspectPort, filesWatcher, testNamePatterns); - filesWatcher?.runningSubtests.set(path, subtest); - return subtest; - }); + root.buildPhaseFinished = false; + for (let i = 0; i < testFiles.length; i++) { + const path = testFiles[i]; + const enqueued = runTestFile(path, root, inspectPort, filesWatcher, testNamePatterns); + filesWatcher?.runningSubtests.set(path, enqueued); + } + root.buildPhaseFinished = true; + if (filesWatcher) { + return root.processPendingSubtests(); + } + return PromisePrototypeThen(root.run(), () => root.postRun()); }; - PromisePrototypeThen(PromisePrototypeThen(PromiseResolve(setup?.(root)), runFiles), postRun); + PromisePrototypeThen(PromiseResolve(setup?.(root)), runFiles); return root.reporter; } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 975ad4ac08b41f..7b5c00905eefbe 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -198,6 +198,10 @@ class SuiteContext { get name() { return this.#suite.name; } + + diagnostic(message) { + this.#suite.diagnostic(message); + } } class Test extends AsyncResource { @@ -226,9 +230,8 @@ class Test extends AsyncResource { if (parent === null) { this.concurrency = 1; this.nesting = 0; - this.only = testOnlyFlag; this.reporter = new TestsStream(); - this.runOnlySubtests = this.only; + this.runOnlySubtests = testOnlyFlag; this.testNumber = 0; this.timeout = kDefaultTimeout; this.root = this; @@ -245,9 +248,8 @@ class Test extends AsyncResource { this.concurrency = parent.concurrency; this.nesting = nesting; - this.only = only ?? !parent.runOnlySubtests; this.reporter = parent.reporter; - this.runOnlySubtests = !this.only; + this.runOnlySubtests = false; this.testNumber = parent.subtests.length + 1; this.timeout = parent.timeout; this.root = parent.root; @@ -292,10 +294,6 @@ class Test extends AsyncResource { skip = 'test name does not match pattern'; } - if (testOnlyFlag && !this.only) { - skip = '\'only\' option not set'; - } - if (skip) { fn = noop; } @@ -334,10 +332,11 @@ class Test extends AsyncResource { this.subtests = []; this.waitingOn = 0; this.finished = false; + this.only = testOnlyFlag || parent?.runOnlySubtests ? only : undefined; - if (!testOnlyFlag && (only || this.runOnlySubtests)) { - const warning = - "'only' and 'runOnly' require the --test-only command-line option."; + + if (!testOnlyFlag && only && !parent.runOnlySubtests) { + const warning = "'only' requires the --test-only command-line option."; this.diagnostic(warning); } @@ -351,6 +350,18 @@ class Test extends AsyncResource { file: loc[2], }; } + + if (this.only && parent !== null) { + parent.markOnly(); + } + } + + markOnly() { + if (this.runOnlySubtests) { + return; + } + this.runOnlySubtests = true; + this.parent?.markOnly(); } matchesTestNamePatterns() { @@ -366,13 +377,11 @@ class Test extends AsyncResource { ArrayPrototypePush(this.pendingSubtests, deferred); } - async processPendingSubtests() { + processPendingSubtests() { while (this.pendingSubtests.length > 0 && this.hasConcurrency()) { const deferred = ArrayPrototypeShift(this.pendingSubtests); const test = deferred.test; - this.reporter.dequeue(test.nesting, test.loc, test.name); - await test.run(); - deferred.resolve(); + PromisePrototypeThen(test.dequeue(), deferred.resolve); } } @@ -432,7 +441,7 @@ class Test extends AsyncResource { // If this test has already ended, attach this test to the root test so // that the error can be properly reported. - const preventAddingSubtests = this.finished || this.buildPhaseFinished; + const preventAddingSubtests = (this.finished || this.buildPhaseFinished) && parent.parent !== null; if (preventAddingSubtests) { while (parent.parent !== null) { parent = parent.parent; @@ -525,6 +534,12 @@ class Test extends AsyncResource { ArrayPrototypePush(this.diagnostics, message); } + dequeue() { + this.reporter.dequeue(this.nesting, this.loc, this.name); + this.parent.activeSubtests++; + return this.run(); + } + start() { // If there is enough available concurrency to run the test now, then do // it. Otherwise, return a Promise to the caller and mark the test as @@ -537,9 +552,7 @@ class Test extends AsyncResource { this.parent.addPendingSubtest(deferred); return deferred.promise; } - - this.reporter.dequeue(this.nesting, this.loc, this.name); - return this.run(); + return this.dequeue(); } [kShouldAbort]() { @@ -574,12 +587,18 @@ class Test extends AsyncResource { } } + get runOnlySibling() { + return this.parent?.runOnlySubtests && !this.only && !this.runOnlySubtests; + } + async run() { - if (this.parent !== null) { - this.parent.activeSubtests++; - } this.startTime = hrtime(); + if (this.runOnlySibling || this.only === false) { + this.fn = noop; + this.skip('\'only\' option not set'); + } + if (this[kShouldAbort]()) { this.postRun(); return; @@ -890,7 +909,6 @@ class Suite extends Test { this.fn = options.fn || this.fn; this.skipped = false; } - this.runOnlySubtests = testOnlyFlag; try { const { ctx, args } = this.getRunArgs(); @@ -912,7 +930,7 @@ class Suite extends Test { this.buildPhaseFinished = true; } - this.fn = () => {}; + this.fn = noop; } getRunArgs() { @@ -920,12 +938,19 @@ class Suite extends Test { return { __proto__: null, ctx, args: [ctx] }; } + hasConcurrency() { + if (this.runOnlySubtests && !this.buildPhaseFinished) { + return false; + } + return super.hasConcurrency(); + } + async run() { const hookArgs = this.getRunArgs(); + this.runOnlySubtests ||= this.runOnlySibling; let stopPromise; try { - this.parent.activeSubtests++; await this.buildSuite; this.startTime = hrtime(); @@ -935,12 +960,13 @@ class Suite extends Test { return; } - if (this.parent.hooks.before.length > 0) { + if (this.parent?.hooks.before.length > 0) { await this.parent.runHook('before', this.parent.getRunArgs()); } await this.runHook('before', hookArgs); + this.processPendingSubtests(); stopPromise = stopTest(this.timeout, this.signal); const subtests = this.skipped || this.error ? [] : this.subtests; const promise = SafePromiseAll(subtests, (subtests) => subtests.start()); @@ -959,7 +985,9 @@ class Suite extends Test { stopPromise?.[SymbolDispose](); } - this.postRun(); + if (this.parent !== null) { + this.postRun(); + } } } diff --git a/test/fixtures/test-runner/output/abort.js b/test/fixtures/test-runner/output/abort.js index eba48d9ec58718..a5624a4f1c4246 100644 --- a/test/fixtures/test-runner/output/abort.js +++ b/test/fixtures/test-runner/output/abort.js @@ -2,7 +2,7 @@ require('../../../common'); const test = require('node:test'); -test('promise timeout signal', { signal: AbortSignal.timeout(1) }, async (t) => { +test('promise timeout signal', { signal: AbortSignal.timeout(100) }, async (t) => { await Promise.all([ t.test('ok 1', async () => {}), t.test('ok 2', () => {}), @@ -22,7 +22,7 @@ test('promise abort signal', { signal: AbortSignal.abort() }, async (t) => { await t.test('should not appear', () => {}); }); -test('callback timeout signal', { signal: AbortSignal.timeout(1) }, (t, done) => { +test('callback timeout signal', { signal: AbortSignal.timeout(100) }, (t, done) => { t.test('ok 1', async () => {}); t.test('ok 2', () => {}); t.test('ok 3', { signal: t.signal }, async () => {}); @@ -40,7 +40,7 @@ test('callback abort signal', { signal: AbortSignal.abort() }, (t, done) => { t.test('should not appear', done); }); -// AbortSignal.timeout(1) doesn't prevent process from closing +// AbortSignal.timeout doesn't prevent process from closing // thus we have to keep the process open to prevent cancelation // of the entire test tree setTimeout(() => {}, 1000); diff --git a/test/fixtures/test-runner/output/abort.snapshot b/test/fixtures/test-runner/output/abort.snapshot index ceca09da14bfb1..16fd9f993b7b90 100644 --- a/test/fixtures/test-runner/output/abort.snapshot +++ b/test/fixtures/test-runner/output/abort.snapshot @@ -28,6 +28,8 @@ TAP version 13 failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 0) ... # Subtest: not ok 2 not ok 6 - not ok 2 @@ -37,6 +39,8 @@ TAP version 13 failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 0) ... # Subtest: not ok 3 not ok 7 - not ok 3 @@ -165,6 +169,8 @@ not ok 2 - promise abort signal failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 2) ... # Subtest: not ok 2 not ok 6 - not ok 2 @@ -174,6 +180,8 @@ not ok 2 - promise abort signal failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 2) ... # Subtest: not ok 3 not ok 7 - not ok 3 diff --git a/test/fixtures/test-runner/output/abort_hooks.snapshot b/test/fixtures/test-runner/output/abort_hooks.snapshot index d0b567bb6a22cd..b793ffd5b7dd26 100644 --- a/test/fixtures/test-runner/output/abort_hooks.snapshot +++ b/test/fixtures/test-runner/output/abort_hooks.snapshot @@ -1,3 +1,4 @@ +TAP version 13 before 2.1 2.2 @@ -6,7 +7,6 @@ beforeEach 4.1 afterEach 4.2 -TAP version 13 # Subtest: 1 before describe # Subtest: test 1 not ok 1 - test 1 @@ -16,6 +16,8 @@ TAP version 13 failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 0) ... # Subtest: test 2 not ok 2 - test 2 @@ -25,6 +27,8 @@ TAP version 13 failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 0) ... 1..2 not ok 1 - 1 before describe @@ -133,6 +137,8 @@ not ok 3 - 3 beforeEach describe failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 2) ... # Subtest: 4 afterEach describe # Subtest: test 1 @@ -186,6 +192,8 @@ not ok 4 - 4 afterEach describe failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 3) ... 1..4 # tests 8 diff --git a/test/fixtures/test-runner/output/abort_suite.js b/test/fixtures/test-runner/output/abort_suite.js index 8a2a2c05458fd4..96da777a0e880f 100644 --- a/test/fixtures/test-runner/output/abort_suite.js +++ b/test/fixtures/test-runner/output/abort_suite.js @@ -2,7 +2,7 @@ require('../../../common'); const { describe, it } = require('node:test'); -describe('describe timeout signal', { signal: AbortSignal.timeout(1) }, (t) => { +describe('describe timeout signal', { signal: AbortSignal.timeout(100) }, (t) => { it('ok 1', async () => {}); it('ok 2', () => {}); it('ok 3', { signal: t.signal }, async () => {}); @@ -20,7 +20,7 @@ describe('describe abort signal', { signal: AbortSignal.abort() }, () => { it('should not appear', () => {}); }); -// AbortSignal.timeout(1) doesn't prevent process from closing +// AbortSignal.timeout doesn't prevent process from closing // thus we have to keep the process open to prevent cancelation // of the entire test tree setTimeout(() => {}, 1000); diff --git a/test/fixtures/test-runner/output/abort_suite.snapshot b/test/fixtures/test-runner/output/abort_suite.snapshot index e7e8c4f4e2360f..e9413913869ae0 100644 --- a/test/fixtures/test-runner/output/abort_suite.snapshot +++ b/test/fixtures/test-runner/output/abort_suite.snapshot @@ -28,6 +28,8 @@ TAP version 13 failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 0) ... # Subtest: not ok 2 not ok 6 - not ok 2 @@ -37,6 +39,8 @@ TAP version 13 failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 0) ... # Subtest: not ok 3 not ok 7 - not ok 3 diff --git a/test/fixtures/test-runner/output/default_output.snapshot b/test/fixtures/test-runner/output/default_output.snapshot index b003f9299c4418..457a916cc17204 100644 --- a/test/fixtures/test-runner/output/default_output.snapshot +++ b/test/fixtures/test-runner/output/default_output.snapshot @@ -7,6 +7,8 @@ *[39m *[39m *[39m + *[39m + at async Promise.all (index 0) *[39m [90m﹣ should skip [90m(*ms)[39m # SKIP[39m @@ -18,6 +20,7 @@ *[39m *[39m *[39m + *[39m [31m✖ should pass but parent fail [90m(*ms)[39m[39m [32m'test did not finish before its parent and was cancelled'[39m @@ -44,6 +47,8 @@ *[39m *[39m *[39m + *[39m + at async Promise.all (index 0) *[39m * @@ -54,6 +59,7 @@ *[39m *[39m *[39m + *[39m * [31m✖ should pass but parent fail [90m(*ms)[39m[39m diff --git a/test/fixtures/test-runner/output/describe_it.snapshot b/test/fixtures/test-runner/output/describe_it.snapshot index be345f11575c8d..78a8688f6762d1 100644 --- a/test/fixtures/test-runner/output/describe_it.snapshot +++ b/test/fixtures/test-runner/output/describe_it.snapshot @@ -25,6 +25,9 @@ not ok 3 - sync todo # TODO * * * + * + * + async Promise.all (index 1) ... # Subtest: sync todo with message not ok 4 - sync todo with message # TODO this is a failing todo @@ -42,6 +45,9 @@ not ok 4 - sync todo with message # TODO this is a failing todo * * * + * + * + async Promise.all (index 2) ... # Subtest: sync skip pass ok 5 - sync skip pass # SKIP @@ -74,6 +80,9 @@ not ok 8 - sync throw fail * * * + * + * + async Promise.all (index 6) ... # Subtest: async skip pass ok 9 - async skip pass # SKIP @@ -106,6 +115,9 @@ not ok 12 - async throw fail * * * + * + * + async Promise.all (index 10) ... # Subtest: async skip fail not ok 13 - async skip fail # SKIP @@ -115,6 +127,8 @@ not ok 13 - async skip fail # SKIP failureType: 'callbackAndPromisePresent' error: 'passed a callback but also returned a Promise' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 11) ... # Subtest: async assertion fail not ok 14 - async assertion fail @@ -140,6 +154,9 @@ not ok 14 - async assertion fail * * * + * + * + async Promise.all (index 12) ... # Subtest: resolve pass ok 15 - resolve pass @@ -162,6 +179,9 @@ not ok 16 - reject fail * * * + * + * + async Promise.all (index 14) ... # Subtest: unhandled rejection - passes but warns ok 17 - unhandled rejection - passes but warns @@ -204,10 +224,10 @@ ok 21 - immediate resolve pass * * * + * new Promise () * * - Array.map () ... # Subtest: mixing describe/it and test should work ok 2 - mixing describe/it and test should work @@ -223,6 +243,8 @@ not ok 22 - subtest sync throw fail failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 21) ... # Subtest: sync throw non-error fail not ok 23 - sync throw non-error fail @@ -232,6 +254,8 @@ not ok 23 - sync throw non-error fail failureType: 'testCodeFailure' error: 'Symbol(thrown symbol from sync throw non-error fail)' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 22) ... # Subtest: level 0a # Subtest: level 1a @@ -292,6 +316,8 @@ not ok 28 - sync skip option is false fail * * * + async Promise.all (index 26) + * ... # Subtest: ok 29 - @@ -373,6 +399,8 @@ not ok 42 - callback also returns a Promise failureType: 'callbackAndPromisePresent' error: 'passed a callback but also returned a Promise' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 40) ... # Subtest: callback throw not ok 43 - callback throw @@ -390,6 +418,8 @@ not ok 43 - callback throw * * * + async Promise.all (index 41) + * ... # Subtest: callback called twice not ok 44 - callback called twice @@ -402,6 +432,7 @@ not ok 44 - callback called twice stack: |- * * + async Promise.all (index 42) ... # Subtest: callback called twice in different ticks ok 45 - callback called twice in different ticks @@ -444,6 +475,8 @@ not ok 49 - custom inspect symbol fail failureType: 'testCodeFailure' error: 'customized' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 48) ... # Subtest: custom inspect symbol that throws fail not ok 50 - custom inspect symbol that throws fail @@ -457,6 +490,8 @@ not ok 50 - custom inspect symbol that throws fail [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] } code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 49) ... # Subtest: subtest sync throw fails # Subtest: sync throw fails at first @@ -474,10 +509,10 @@ not ok 50 - custom inspect symbol that throws fail * * * + * new Promise () * * - Array.map () ... # Subtest: sync throw fails at second not ok 2 - sync throw fails at second @@ -497,7 +532,7 @@ not ok 50 - custom inspect symbol that throws fail * * * - async Promise.all (index 0) + * ... 1..2 not ok 51 - subtest sync throw fails @@ -508,6 +543,8 @@ not ok 51 - subtest sync throw fails failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 50) ... # Subtest: describe sync throw fails # Subtest: should not run @@ -518,6 +555,8 @@ not ok 51 - subtest sync throw fails failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 51) ... 1..1 not ok 52 - describe sync throw fails @@ -549,6 +588,8 @@ not ok 52 - describe sync throw fails failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 52) ... 1..1 not ok 53 - describe async throw fails @@ -582,6 +623,7 @@ not ok 53 - describe async throw fails code: 'ERR_TEST_FAILURE' stack: |- async Promise.all (index 0) + async Promise.all (index 53) ... # Subtest: timed out callback test not ok 2 - timed out callback test @@ -591,6 +633,9 @@ not ok 53 - describe async throw fails failureType: 'testTimeoutFailure' error: 'test timed out after 5ms' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 1) + async Promise.all (index 53) ... # Subtest: large timeout async test is ok ok 3 - large timeout async test is ok @@ -611,6 +656,8 @@ not ok 54 - timeouts failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 53) ... # Subtest: successful thenable # Subtest: successful thenable @@ -639,6 +686,8 @@ not ok 55 - successful thenable failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 54) ... # Subtest: rejected thenable not ok 56 - rejected thenable diff --git a/test/fixtures/test-runner/output/global_after_should_fail_the_test.snapshot b/test/fixtures/test-runner/output/global_after_should_fail_the_test.snapshot index 3196f377b3d4bf..e385d2dc65d8ff 100644 --- a/test/fixtures/test-runner/output/global_after_should_fail_the_test.snapshot +++ b/test/fixtures/test-runner/output/global_after_should_fail_the_test.snapshot @@ -1,5 +1,5 @@ -this is a test TAP version 13 +this is a test # Subtest: this is a test ok 1 - this is a test --- @@ -21,7 +21,6 @@ not ok 2 - /test/fixtures/test-runner/output/global_after_should_fail_the_test.j * * * - * ... 1..1 # tests 1 diff --git a/test/fixtures/test-runner/output/hooks.snapshot b/test/fixtures/test-runner/output/hooks.snapshot index 5afe398ed3d0ea..a6c220911b5f73 100644 --- a/test/fixtures/test-runner/output/hooks.snapshot +++ b/test/fixtures/test-runner/output/hooks.snapshot @@ -42,6 +42,8 @@ ok 1 - describe hooks failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 1) ... # Subtest: 2 not ok 2 - 2 @@ -51,6 +53,8 @@ ok 1 - describe hooks failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 1) ... 1..2 not ok 2 - before throws @@ -70,6 +74,7 @@ not ok 2 - before throws * * * + async Promise.all (index 1) * ... # Subtest: after throws @@ -101,6 +106,7 @@ not ok 3 - after throws * * * + async Promise.all (index 2) * ... # Subtest: beforeEach throws @@ -122,7 +128,7 @@ not ok 3 - after throws * async Promise.all (index 0) * - * + async Promise.all (index 3) ... # Subtest: 2 not ok 2 - 2 @@ -140,7 +146,9 @@ not ok 3 - after throws * * * + async Promise.all (index 1) * + async Promise.all (index 3) ... 1..2 not ok 4 - beforeEach throws @@ -151,6 +159,8 @@ not ok 4 - beforeEach throws failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 3) ... # Subtest: afterEach throws # Subtest: 1 @@ -190,6 +200,7 @@ not ok 4 - beforeEach throws * * * + async Promise.all (index 1) * ... 1..2 @@ -201,6 +212,8 @@ not ok 5 - afterEach throws failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 4) ... # Subtest: afterEach when test fails # Subtest: 1 @@ -218,10 +231,10 @@ not ok 5 - afterEach throws * * * + * new Promise () * * - Array.map () ... # Subtest: 2 ok 2 - 2 @@ -237,6 +250,8 @@ not ok 6 - afterEach when test fails failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 5) ... # Subtest: afterEach throws and test fails # Subtest: 1 @@ -254,10 +269,10 @@ not ok 6 - afterEach when test fails * * * + * new Promise () * * - Array.map () ... # Subtest: 2 not ok 2 - 2 @@ -276,6 +291,7 @@ not ok 6 - afterEach when test fails * * * + async Promise.all (index 1) * ... 1..2 @@ -287,6 +303,8 @@ not ok 7 - afterEach throws and test fails failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 6) ... # Subtest: test hooks # Subtest: 1 @@ -369,6 +387,8 @@ not ok 9 - t.before throws failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 8) ... # Subtest: t.beforeEach throws # Subtest: 1 @@ -389,7 +409,7 @@ not ok 9 - t.before throws * * * - * + async Promise.all (index 9) ... # Subtest: 2 not ok 2 - 2 @@ -409,7 +429,7 @@ not ok 9 - t.before throws * * * - * + async Promise.all (index 9) ... 1..2 not ok 10 - t.beforeEach throws @@ -419,6 +439,8 @@ not ok 10 - t.beforeEach throws failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 9) ... # Subtest: t.afterEach throws # Subtest: 1 @@ -469,6 +491,8 @@ not ok 11 - t.afterEach throws failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 10) ... # Subtest: afterEach when test fails # Subtest: 1 @@ -489,6 +513,7 @@ not ok 11 - t.afterEach throws * * * + async Promise.all (index 11) ... # Subtest: 2 ok 2 - 2 @@ -503,6 +528,8 @@ not ok 12 - afterEach when test fails failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 11) ... # Subtest: afterEach throws and test fails # Subtest: 1 @@ -523,6 +550,7 @@ not ok 12 - afterEach when test fails * * * + async Promise.all (index 12) ... # Subtest: 2 not ok 2 - 2 @@ -552,6 +580,8 @@ not ok 13 - afterEach throws and test fails failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 12) ... # Subtest: t.after() is called if test body throws not ok 14 - t.after() is called if test body throws @@ -565,6 +595,7 @@ not ok 14 - t.after() is called if test body throws * * * + async Promise.all (index 13) * ... # - after() called diff --git a/test/fixtures/test-runner/output/only_tests.js b/test/fixtures/test-runner/output/only_tests.js index 26266b524454b7..cac152578dfc6b 100644 --- a/test/fixtures/test-runner/output/only_tests.js +++ b/test/fixtures/test-runner/output/only_tests.js @@ -1,100 +1,96 @@ // Flags: --test-only 'use strict'; -require('../../../common'); +const common = require('../../../common'); const { test, describe, it } = require('node:test'); // These tests should be skipped based on the 'only' option. -test('only = undefined'); -test('only = undefined, skip = string', { skip: 'skip message' }); -test('only = undefined, skip = true', { skip: true }); -test('only = undefined, skip = false', { skip: false }); -test('only = false', { only: false }); -test('only = false, skip = string', { only: false, skip: 'skip message' }); -test('only = false, skip = true', { only: false, skip: true }); -test('only = false, skip = false', { only: false, skip: false }); +test('only = undefined', common.mustNotCall()); +test('only = undefined, skip = string', { skip: 'skip message' }, common.mustNotCall()); +test('only = undefined, skip = true', { skip: true }, common.mustNotCall()); +test('only = undefined, skip = false', { skip: false }, common.mustNotCall()); +test('only = false', { only: false }, common.mustNotCall()); +test('only = false, skip = string', { only: false, skip: 'skip message' }, common.mustNotCall()); +test('only = false, skip = true', { only: false, skip: true }, common.mustNotCall()); +test('only = false, skip = false', { only: false, skip: false }, common.mustNotCall()); // These tests should be skipped based on the 'skip' option. -test('only = true, skip = string', { only: true, skip: 'skip message' }); -test('only = true, skip = true', { only: true, skip: true }); +test('only = true, skip = string', { only: true, skip: 'skip message' }, common.mustNotCall()); +test('only = true, skip = true', { only: true, skip: true }, common.mustNotCall()); // An 'only' test with subtests. -test('only = true, with subtests', { only: true }, async (t) => { +test('only = true, with subtests', { only: true }, common.mustCall(async (t) => { // These subtests should run. - await t.test('running subtest 1'); - await t.test('running subtest 2'); + await t.test('running subtest 1', common.mustCall()); + await t.test('running subtest 2', common.mustCall()); // Switch the context to only execute 'only' tests. t.runOnly(true); - await t.test('skipped subtest 1'); - await t.test('skipped subtest 2'); - await t.test('running subtest 3', { only: true }); + await t.test('skipped subtest 1', common.mustNotCall()); + await t.test('skipped subtest 2'), common.mustNotCall(); + await t.test('running subtest 3', { only: true }, common.mustCall()); // Switch the context back to execute all tests. t.runOnly(false); - await t.test('running subtest 4', async (t) => { + await t.test('running subtest 4', common.mustCall(async (t) => { // These subtests should run. - await t.test('running sub-subtest 1'); - await t.test('running sub-subtest 2'); + await t.test('running sub-subtest 1', common.mustCall()); + await t.test('running sub-subtest 2', common.mustCall()); // Switch the context to only execute 'only' tests. t.runOnly(true); - await t.test('skipped sub-subtest 1'); - await t.test('skipped sub-subtest 2'); - }); + await t.test('skipped sub-subtest 1', common.mustNotCall()); + await t.test('skipped sub-subtest 2', common.mustNotCall()); + })); // Explicitly do not run these tests. - await t.test('skipped subtest 3', { only: false }); - await t.test('skipped subtest 4', { skip: true }); -}); + await t.test('skipped subtest 3', { only: false }, common.mustNotCall()); + await t.test('skipped subtest 4', { skip: true }, common.mustNotCall()); +})); -describe.only('describe only = true, with subtests', () => { - it.only('`it` subtest 1 should run', () => {}); +describe.only('describe only = true, with subtests', common.mustCall(() => { + it.only('`it` subtest 1 should run', common.mustCall()); - it('`it` subtest 2 should not run', async () => {}); -}); + it('`it` subtest 2 should not run', common.mustNotCall()); +})); -describe.only('describe only = true, with a mixture of subtests', () => { - it.only('`it` subtest 1', () => {}); +describe.only('describe only = true, with a mixture of subtests', common.mustCall(() => { + it.only('`it` subtest 1', common.mustCall()); - it.only('`it` async subtest 1', async () => {}); + it.only('`it` async subtest 1', common.mustCall(async () => {})); - it('`it` subtest 2 only=true', { only: true }); + it('`it` subtest 2 only=true', { only: true }, common.mustCall()); - it('`it` subtest 2 only=false', { only: false }, () => { - throw new Error('This should not run'); - }); + it('`it` subtest 2 only=false', { only: false }, common.mustNotCall()); - it.skip('`it` subtest 3 skip', () => { - throw new Error('This should not run'); - }); + it.skip('`it` subtest 3 skip', common.mustNotCall()); - it.todo('`it` subtest 4 todo', { only: false }, () => { - throw new Error('This should not run'); - }); + it.todo('`it` subtest 4 todo', { only: false }, common.mustNotCall()); - test.only('`test` subtest 1', () => {}); + test.only('`test` subtest 1', common.mustCall()); - test.only('`test` async subtest 1', async () => {}); + test.only('`test` async subtest 1', common.mustCall(async () => {})); - test('`test` subtest 2 only=true', { only: true }); + test('`test` subtest 2 only=true', { only: true }, common.mustCall()); - test('`test` subtest 2 only=false', { only: false }, () => { - throw new Error('This should not run'); - }); + test('`test` subtest 2 only=false', { only: false }, common.mustNotCall()); - test.skip('`test` subtest 3 skip', () => { - throw new Error('This should not run'); - }); + test.skip('`test` subtest 3 skip', common.mustNotCall()); - test.todo('`test` subtest 4 todo', { only: false }, () => { - throw new Error('This should not run'); - }); -}); + test.todo('`test` subtest 4 todo', { only: false }, common.mustNotCall()); +})); -describe.only('describe only = true, with subtests', () => { - test.only('subtest should run', () => {}); +describe.only('describe only = true, with subtests', common.mustCall(() => { + test.only('subtest should run', common.mustCall()); - test('async subtest should not run', async () => {}); + test('async subtest should not run', common.mustNotCall()); - test('subtest should be skipped', { only: false }, () => {}); -}); + test('subtest should be skipped', { only: false }, common.mustNotCall()); +})); + +describe('describe only = undefined, with subtests', common.mustCall(() => { + test('async subtest should not run', common.mustNotCall()); +})); + +describe('describe only = false, with subtests', { only: false }, common.mustCall(() => { + test('async subtest should not run', common.mustNotCall()); +})); diff --git a/test/fixtures/test-runner/output/only_tests.snapshot b/test/fixtures/test-runner/output/only_tests.snapshot index ded19f3bec4c6a..de6251f29d9ef2 100644 --- a/test/fixtures/test-runner/output/only_tests.snapshot +++ b/test/fixtures/test-runner/output/only_tests.snapshot @@ -222,12 +222,36 @@ ok 14 - describe only = true, with subtests duration_ms: * type: 'suite' ... -1..14 -# tests 40 -# suites 3 +# Subtest: describe only = undefined, with subtests + # Subtest: async subtest should not run + ok 1 - async subtest should not run # SKIP 'only' option not set + --- + duration_ms: * + ... + 1..1 +ok 15 - describe only = undefined, with subtests + --- + duration_ms: * + type: 'suite' + ... +# Subtest: describe only = false, with subtests + # Subtest: async subtest should not run + ok 1 - async subtest should not run # SKIP 'only' option not set + --- + duration_ms: * + ... + 1..1 +ok 16 - describe only = false, with subtests + --- + duration_ms: * + type: 'suite' + ... +1..16 +# tests 42 +# suites 5 # pass 15 # fail 0 # cancelled 0 -# skipped 25 +# skipped 27 # todo 0 # duration_ms * diff --git a/test/fixtures/test-runner/output/output.js b/test/fixtures/test-runner/output/output.js index f37d3495030950..de7c220d7be595 100644 --- a/test/fixtures/test-runner/output/output.js +++ b/test/fixtures/test-runner/output/output.js @@ -274,11 +274,11 @@ test('callback async throw after done', (t, done) => { done(); }); -test('only is set but not in only mode', { only: true }, async (t) => { - // All of these subtests should run. +test('runOnly is set', async (t) => { + // Subtests should run only outside of a runOnly block, unless they have only: true. await t.test('running subtest 1'); t.runOnly(true); - await t.test('running subtest 2'); + await t.test('skipped subtest 2'); await t.test('running subtest 3', { only: true }); t.runOnly(false); await t.test('running subtest 4'); diff --git a/test/fixtures/test-runner/output/output.snapshot b/test/fixtures/test-runner/output/output.snapshot index 18f030dab361ab..cbd2ced1235f05 100644 --- a/test/fixtures/test-runner/output/output.snapshot +++ b/test/fixtures/test-runner/output/output.snapshot @@ -25,6 +25,9 @@ not ok 3 - sync fail todo # TODO * * * + * + * + async Promise.all (index 1) ... # Subtest: sync fail todo with message not ok 4 - sync fail todo with message # TODO this is a failing todo @@ -42,6 +45,9 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo * * * + * + * + async Promise.all (index 2) ... # Subtest: sync skip pass ok 5 - sync skip pass # SKIP @@ -75,6 +81,9 @@ not ok 8 - sync throw fail * * * + * + * + async Promise.all (index 6) ... # Subtest: async skip pass ok 9 - async skip pass # SKIP @@ -102,6 +111,9 @@ not ok 11 - async throw fail * * * + * + * + async Promise.all (index 9) ... # Subtest: async skip fail not ok 12 - async skip fail # SKIP @@ -119,6 +131,9 @@ not ok 12 - async skip fail # SKIP * * * + * + * + async Promise.all (index 10) ... # Subtest: async assertion fail not ok 13 - async assertion fail @@ -144,6 +159,9 @@ not ok 13 - async assertion fail * * * + * + * + async Promise.all (index 11) ... # Subtest: resolve pass ok 14 - resolve pass @@ -166,6 +184,9 @@ not ok 15 - reject fail * * * + * + * + async Promise.all (index 13) ... # Subtest: unhandled rejection - passes but warns ok 16 - unhandled rejection - passes but warns @@ -222,6 +243,8 @@ not ok 21 - subtest sync throw fail failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 20) ... # Subtest: sync throw non-error fail not ok 22 - sync throw non-error fail @@ -231,6 +254,8 @@ not ok 22 - sync throw non-error fail failureType: 'testCodeFailure' error: 'Symbol(thrown symbol from sync throw non-error fail)' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 21) ... # Subtest: level 0a # Subtest: level 1a @@ -267,6 +292,8 @@ ok 23 - level 0a failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 23) ... # Subtest: +short running # Subtest: ++short running @@ -287,6 +314,8 @@ not ok 24 - top level failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 23) ... # Subtest: invalid subtest - pass but subtest fails ok 25 - invalid subtest - pass but subtest fails @@ -319,6 +348,8 @@ not ok 28 - sync skip option is false fail * * * + async Promise.all (index 26) + * ... # Subtest: ok 29 - @@ -400,6 +431,8 @@ not ok 42 - callback also returns a Promise failureType: 'callbackAndPromisePresent' error: 'passed a callback but also returned a Promise' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 40) ... # Subtest: callback throw not ok 43 - callback throw @@ -417,6 +450,8 @@ not ok 43 - callback throw * * * + async Promise.all (index 41) + * ... # Subtest: callback called twice not ok 44 - callback called twice @@ -429,6 +464,7 @@ not ok 44 - callback called twice stack: |- * * + async Promise.all (index 42) ... # Subtest: callback called twice in different ticks ok 45 - callback called twice in different ticks @@ -463,35 +499,32 @@ ok 48 - callback async throw after done --- duration_ms: * ... -# Subtest: only is set but not in only mode +# Subtest: runOnly is set # Subtest: running subtest 1 ok 1 - running subtest 1 --- duration_ms: * ... - # Subtest: running subtest 2 - ok 2 - running subtest 2 + # Subtest: skipped subtest 2 + ok 2 - skipped subtest 2 # SKIP 'only' option not set --- duration_ms: * ... - # 'only' and 'runOnly' require the --test-only command-line option. # Subtest: running subtest 3 ok 3 - running subtest 3 --- duration_ms: * ... - # 'only' and 'runOnly' require the --test-only command-line option. # Subtest: running subtest 4 ok 4 - running subtest 4 --- duration_ms: * ... 1..4 -ok 49 - only is set but not in only mode +ok 49 - runOnly is set --- duration_ms: * ... -# 'only' and 'runOnly' require the --test-only command-line option. # Subtest: custom inspect symbol fail not ok 50 - custom inspect symbol fail --- @@ -500,6 +533,8 @@ not ok 50 - custom inspect symbol fail failureType: 'testCodeFailure' error: 'customized' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 49) ... # Subtest: custom inspect symbol that throws fail not ok 51 - custom inspect symbol that throws fail @@ -513,6 +548,8 @@ not ok 51 - custom inspect symbol that throws fail [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] } code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 50) ... # Subtest: subtest sync throw fails # Subtest: sync throw fails at first @@ -563,6 +600,8 @@ not ok 52 - subtest sync throw fails failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 51) ... # Subtest: timed out async test not ok 53 - timed out async test @@ -572,6 +611,8 @@ not ok 53 - timed out async test failureType: 'testTimeoutFailure' error: 'test timed out after 5ms' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 52) ... # Subtest: timed out callback test not ok 54 - timed out callback test @@ -581,6 +622,8 @@ not ok 54 - timed out callback test failureType: 'testTimeoutFailure' error: 'test timed out after 5ms' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 53) ... # Subtest: large timeout async test is ok ok 55 - large timeout async test is ok @@ -605,6 +648,8 @@ not ok 58 - rejected thenable failureType: 'testCodeFailure' error: 'custom error' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 57) ... # Subtest: unfinished test with uncaughtException not ok 59 - unfinished test with uncaughtException @@ -685,9 +730,9 @@ not ok 62 - invalid subtest fail # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. # tests 76 # suites 0 -# pass 35 +# pass 34 # fail 25 # cancelled 3 -# skipped 9 +# skipped 10 # todo 4 # duration_ms * diff --git a/test/fixtures/test-runner/output/output_cli.snapshot b/test/fixtures/test-runner/output/output_cli.snapshot index 3cef8f29b253b9..e602dda9416590 100644 --- a/test/fixtures/test-runner/output/output_cli.snapshot +++ b/test/fixtures/test-runner/output/output_cli.snapshot @@ -25,6 +25,9 @@ not ok 3 - sync fail todo # TODO * * * + * + * + async Promise.all (index 1) ... # Subtest: sync fail todo with message not ok 4 - sync fail todo with message # TODO this is a failing todo @@ -42,6 +45,9 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo * * * + * + * + async Promise.all (index 2) ... # Subtest: sync skip pass ok 5 - sync skip pass # SKIP @@ -75,6 +81,9 @@ not ok 8 - sync throw fail * * * + * + * + async Promise.all (index 6) ... # Subtest: async skip pass ok 9 - async skip pass # SKIP @@ -102,6 +111,9 @@ not ok 11 - async throw fail * * * + * + * + async Promise.all (index 9) ... # Subtest: async skip fail not ok 12 - async skip fail # SKIP @@ -119,6 +131,9 @@ not ok 12 - async skip fail # SKIP * * * + * + * + async Promise.all (index 10) ... # Subtest: async assertion fail not ok 13 - async assertion fail @@ -144,6 +159,9 @@ not ok 13 - async assertion fail * * * + * + * + async Promise.all (index 11) ... # Subtest: resolve pass ok 14 - resolve pass @@ -166,6 +184,9 @@ not ok 15 - reject fail * * * + * + * + async Promise.all (index 13) ... # Subtest: unhandled rejection - passes but warns ok 16 - unhandled rejection - passes but warns @@ -222,6 +243,8 @@ not ok 21 - subtest sync throw fail failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 20) ... # Subtest: sync throw non-error fail not ok 22 - sync throw non-error fail @@ -231,6 +254,8 @@ not ok 22 - sync throw non-error fail failureType: 'testCodeFailure' error: 'Symbol(thrown symbol from sync throw non-error fail)' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 21) ... # Subtest: level 0a # Subtest: level 1a @@ -267,6 +292,8 @@ ok 23 - level 0a failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 23) ... # Subtest: +short running # Subtest: ++short running @@ -287,6 +314,8 @@ not ok 24 - top level failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 23) ... # Subtest: invalid subtest - pass but subtest fails ok 25 - invalid subtest - pass but subtest fails @@ -319,6 +348,8 @@ not ok 28 - sync skip option is false fail * * * + async Promise.all (index 26) + * ... # Subtest: ok 29 - @@ -400,6 +431,8 @@ not ok 42 - callback also returns a Promise failureType: 'callbackAndPromisePresent' error: 'passed a callback but also returned a Promise' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 40) ... # Subtest: callback throw not ok 43 - callback throw @@ -417,6 +450,8 @@ not ok 43 - callback throw * * * + async Promise.all (index 41) + * ... # Subtest: callback called twice not ok 44 - callback called twice @@ -429,6 +464,7 @@ not ok 44 - callback called twice stack: |- * * + async Promise.all (index 42) ... # Subtest: callback called twice in different ticks ok 45 - callback called twice in different ticks @@ -463,35 +499,32 @@ ok 48 - callback async throw after done --- duration_ms: * ... -# Subtest: only is set but not in only mode +# Subtest: runOnly is set # Subtest: running subtest 1 ok 1 - running subtest 1 --- duration_ms: * ... - # Subtest: running subtest 2 - ok 2 - running subtest 2 + # Subtest: skipped subtest 2 + ok 2 - skipped subtest 2 # SKIP 'only' option not set --- duration_ms: * ... - # 'only' and 'runOnly' require the --test-only command-line option. # Subtest: running subtest 3 ok 3 - running subtest 3 --- duration_ms: * ... - # 'only' and 'runOnly' require the --test-only command-line option. # Subtest: running subtest 4 ok 4 - running subtest 4 --- duration_ms: * ... 1..4 -ok 49 - only is set but not in only mode +ok 49 - runOnly is set --- duration_ms: * ... -# 'only' and 'runOnly' require the --test-only command-line option. # Subtest: custom inspect symbol fail not ok 50 - custom inspect symbol fail --- @@ -500,6 +533,8 @@ not ok 50 - custom inspect symbol fail failureType: 'testCodeFailure' error: 'customized' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 49) ... # Subtest: custom inspect symbol that throws fail not ok 51 - custom inspect symbol that throws fail @@ -513,6 +548,8 @@ not ok 51 - custom inspect symbol that throws fail [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] } code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 50) ... # Subtest: subtest sync throw fails # Subtest: sync throw fails at first @@ -563,6 +600,8 @@ not ok 52 - subtest sync throw fails failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 51) ... # Subtest: timed out async test not ok 53 - timed out async test @@ -572,6 +611,8 @@ not ok 53 - timed out async test failureType: 'testTimeoutFailure' error: 'test timed out after 5ms' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 52) ... # Subtest: timed out callback test not ok 54 - timed out callback test @@ -581,6 +622,8 @@ not ok 54 - timed out callback test failureType: 'testTimeoutFailure' error: 'test timed out after 5ms' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 53) ... # Subtest: large timeout async test is ok ok 55 - large timeout async test is ok @@ -605,6 +648,8 @@ not ok 58 - rejected thenable failureType: 'testCodeFailure' error: 'custom error' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 57) ... # Subtest: unfinished test with uncaughtException not ok 59 - unfinished test with uncaughtException @@ -690,9 +735,9 @@ ok 63 - last test 1..63 # tests 77 # suites 0 -# pass 36 +# pass 35 # fail 25 # cancelled 3 -# skipped 9 +# skipped 10 # todo 4 # duration_ms * diff --git a/test/fixtures/test-runner/output/spec_reporter.snapshot b/test/fixtures/test-runner/output/spec_reporter.snapshot index 5dc05d5b43c12d..4511fa5b434243 100644 --- a/test/fixtures/test-runner/output/spec_reporter.snapshot +++ b/test/fixtures/test-runner/output/spec_reporter.snapshot @@ -9,6 +9,9 @@ * * * + * + * + at async Promise.all (index 1) sync fail todo with message (*ms) # this is a failing todo Error: thrown from sync fail todo with message @@ -19,6 +22,9 @@ * * * + * + * + at async Promise.all (index 2) sync skip pass (*ms) # SKIP sync skip pass with message (*ms) # this is skipped @@ -33,6 +39,9 @@ * * * + * + * + at async Promise.all (index 6) async skip pass (*ms) # SKIP async pass (*ms) @@ -45,6 +54,9 @@ * * * + * + * + at async Promise.all (index 9) async skip fail (*ms) # SKIP Error: thrown from async throw fail @@ -55,6 +67,9 @@ * * * + * + * + at async Promise.all (index 10) async assertion fail (*ms) AssertionError [ERR_ASSERTION]: Expected values to be strictly equal: @@ -67,7 +82,10 @@ * * * - * { + * + * + * + at async Promise.all (index 11) { generatedMessage: true, code: 'ERR_ASSERTION', actual: true, @@ -85,6 +103,9 @@ * * * + * + * + at async Promise.all (index 13) unhandled rejection - passes but warns (*ms) async unhandled rejection - passes but warns (*ms) @@ -140,6 +161,8 @@ * * * + at async Promise.all (index 26) + * (*ms) functionOnly (*ms) @@ -170,6 +193,8 @@ * * * + at async Promise.all (index 41) + * callback called twice (*ms) 'callback invoked multiple times' @@ -189,16 +214,13 @@ * callback async throw after done (*ms) - only is set but not in only mode + runOnly is set running subtest 1 (*ms) - running subtest 2 (*ms) - 'only' and 'runOnly' require the --test-only command-line option. + skipped subtest 2 (*ms) # 'only' option not set running subtest 3 (*ms) - 'only' and 'runOnly' require the --test-only command-line option. running subtest 4 (*ms) - only is set but not in only mode (*ms) + runOnly is set (*ms) - 'only' and 'runOnly' require the --test-only command-line option. custom inspect symbol fail (*ms) customized @@ -292,10 +314,10 @@ Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. tests 76 suites 0 - pass 35 + pass 34 fail 25 cancelled 3 - skipped 9 + skipped 10 todo 4 duration_ms * @@ -311,6 +333,9 @@ * * * + * + * + at async Promise.all (index 1) * sync fail todo with message (*ms) # this is a failing todo @@ -322,6 +347,9 @@ * * * + * + * + at async Promise.all (index 2) * sync throw fail (*ms) @@ -333,6 +361,9 @@ * * * + * + * + at async Promise.all (index 6) * async throw fail (*ms) @@ -344,6 +375,9 @@ * * * + * + * + at async Promise.all (index 9) * async skip fail (*ms) # SKIP @@ -355,6 +389,9 @@ * * * + * + * + at async Promise.all (index 10) * async assertion fail (*ms) @@ -368,7 +405,10 @@ * * * - * { + * + * + * + at async Promise.all (index 11) { generatedMessage: true, code: 'ERR_ASSERTION', actual: true, @@ -386,6 +426,9 @@ * * * + * + * + at async Promise.all (index 13) * +sync throw fail (*ms) @@ -419,6 +462,8 @@ * * * + at async Promise.all (index 26) + * * callback fail (*ms) @@ -440,6 +485,8 @@ * * * + at async Promise.all (index 41) + * * callback called twice (*ms) diff --git a/test/fixtures/test-runner/output/spec_reporter_cli.snapshot b/test/fixtures/test-runner/output/spec_reporter_cli.snapshot index 25c22069c3b8e7..e78b1cf1a0df96 100644 --- a/test/fixtures/test-runner/output/spec_reporter_cli.snapshot +++ b/test/fixtures/test-runner/output/spec_reporter_cli.snapshot @@ -9,6 +9,9 @@ * * * + * + * + at async Promise.all (index 1) sync fail todo with message (*ms) # this is a failing todo Error: thrown from sync fail todo with message @@ -19,6 +22,9 @@ * * * + * + * + at async Promise.all (index 2) sync skip pass (*ms) # SKIP sync skip pass with message (*ms) # this is skipped @@ -33,6 +39,9 @@ * * * + * + * + at async Promise.all (index 6) async skip pass (*ms) # SKIP async pass (*ms) @@ -45,6 +54,9 @@ * * * + * + * + at async Promise.all (index 9) async skip fail (*ms) # SKIP Error: thrown from async throw fail @@ -55,6 +67,9 @@ * * * + * + * + at async Promise.all (index 10) async assertion fail (*ms) AssertionError [ERR_ASSERTION]: Expected values to be strictly equal: @@ -67,7 +82,10 @@ * * * - * { + * + * + * + at async Promise.all (index 11) { generatedMessage: true, code: 'ERR_ASSERTION', actual: true, @@ -85,6 +103,9 @@ * * * + * + * + at async Promise.all (index 13) unhandled rejection - passes but warns (*ms) async unhandled rejection - passes but warns (*ms) @@ -140,6 +161,8 @@ * * * + at async Promise.all (index 26) + * (*ms) functionOnly (*ms) @@ -170,6 +193,8 @@ * * * + at async Promise.all (index 41) + * callback called twice (*ms) 'callback invoked multiple times' @@ -189,16 +214,13 @@ * callback async throw after done (*ms) - only is set but not in only mode + runOnly is set running subtest 1 (*ms) - running subtest 2 (*ms) - 'only' and 'runOnly' require the --test-only command-line option. + skipped subtest 2 (*ms) # 'only' option not set running subtest 3 (*ms) - 'only' and 'runOnly' require the --test-only command-line option. running subtest 4 (*ms) - only is set but not in only mode (*ms) + runOnly is set (*ms) - 'only' and 'runOnly' require the --test-only command-line option. custom inspect symbol fail (*ms) customized @@ -292,10 +314,10 @@ Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. tests 76 suites 0 - pass 35 + pass 34 fail 25 cancelled 3 - skipped 9 + skipped 10 todo 4 duration_ms * @@ -311,6 +333,9 @@ * * * + * + * + at async Promise.all (index 1) * sync fail todo with message (*ms) # this is a failing todo @@ -322,6 +347,9 @@ * * * + * + * + at async Promise.all (index 2) * sync throw fail (*ms) @@ -333,6 +361,9 @@ * * * + * + * + at async Promise.all (index 6) * async throw fail (*ms) @@ -344,6 +375,9 @@ * * * + * + * + at async Promise.all (index 9) * async skip fail (*ms) # SKIP @@ -355,6 +389,9 @@ * * * + * + * + at async Promise.all (index 10) * async assertion fail (*ms) @@ -368,7 +405,10 @@ * * * - * { + * + * + * + at async Promise.all (index 11) { generatedMessage: true, code: 'ERR_ASSERTION', actual: true, @@ -386,6 +426,9 @@ * * * + * + * + at async Promise.all (index 13) * +sync throw fail (*ms) @@ -419,6 +462,8 @@ * * * + at async Promise.all (index 26) + * * callback fail (*ms) @@ -440,6 +485,8 @@ * * * + at async Promise.all (index 41) + * * callback called twice (*ms) diff --git a/test/fixtures/test-runner/output/timeout_in_before_each_should_not_affect_further_tests.snapshot b/test/fixtures/test-runner/output/timeout_in_before_each_should_not_affect_further_tests.snapshot index b3579da789470b..68cc276808e27b 100644 --- a/test/fixtures/test-runner/output/timeout_in_before_each_should_not_affect_further_tests.snapshot +++ b/test/fixtures/test-runner/output/timeout_in_before_each_should_not_affect_further_tests.snapshot @@ -1,5 +1,5 @@ -gonna timeout TAP version 13 +gonna timeout not gonna timeout before each test second 2 after each test first 0 @@ -15,6 +15,7 @@ gonna timeout code: 'ERR_TEST_FAILURE' stack: |- async Promise.all (index 0) + async Promise.all (index 0) ... # Subtest: first describe second test ok 2 - first describe second test @@ -30,6 +31,8 @@ not ok 1 - before each timeout failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 0) ... after each test second 1 not gonna timeout @@ -44,6 +47,7 @@ not gonna timeout code: 'ERR_TEST_FAILURE' stack: |- async Promise.all (index 0) + async Promise.all (index 1) ... # Subtest: second describe second test ok 2 - second describe second test @@ -59,6 +63,8 @@ not ok 2 - after each timeout failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' + stack: |- + async Promise.all (index 1) ... 1..2 # tests 4