fix(snapshot): fail test when snapshot assertion is used with test.fails

packages/runner/src/errors.ts

@@ -43,3 +43,19 @@ export class AroundHookTeardownError extends Error {
 export class AroundHookMultipleCallsError extends Error {
   public name = 'AroundHookMultipleCallsError'
 }
+
+// `test.fails` doesn't flip the test result when this error is thrown
+export class TestSyntaxError extends Error {
+  public name = 'TestSyntaxError'
+
+  constructor(message: string) {
+    super(message)
+    // use custom property so this survives when the error
+    // is serialized on `packages/expect` side (e.g. for `expect.soft`)
+    // and `packages/runner` can still detect it during `test.fails` handling
+    Object.defineProperty(this, '__vitest_test_syntax_error__', {
+      value: true,
+      enumerable: false,
+    })
+  }
+}

packages/runner/src/index.ts

@@ -1,4 +1,5 @@
 export { recordArtifact } from './artifact'
+export { TestSyntaxError } from './errors'
 export {
   afterAll,
   afterEach,

packages/runner/src/run.ts

@@ -761,14 +761,14 @@ export async function runTest(test: Test, runner: VitestRunner): Promise<void> {
     }
   }
 
-  // if test is marked to be failed, flip the result
+  // if test is marked to be failed, flip the result unless `TestSyntaxError` is present
   if (test.fails) {
     if (test.result.state === 'pass') {
       const error = processError(new Error('Expect test to fail'))
       test.result.state = 'fail'
       test.result.errors = [error]
     }
-    else {
+    else if (!test.result.errors?.some(e => e.__vitest_test_syntax_error__)) {
       test.result.state = 'pass'
       test.result.errors = undefined
     }

packages/vitest/src/integrations/snapshot/chai.ts

@@ -1,6 +1,7 @@
 import type { AsyncExpectationResult, ChaiPlugin, MatcherState, SyncExpectationResult } from '@vitest/expect'
 import type { Test } from '@vitest/runner'
 import { chai, createAssertionMessage, equals, iterableEquality, recordAsyncExpect, subsetEquality, wrapAssertion } from '@vitest/expect'
+import { TestSyntaxError } from '@vitest/runner'
 import { getNames } from '@vitest/runner/utils'
 import {
   addSerializer,
@@ -61,11 +62,14 @@ function getAssertionName(assertion: Chai.Assertion): string {
 }
 
 function getTest(obj: Chai.Assertion) {
-  const test = chai.util.flag(obj, 'vitest-test')
+  const test = chai.util.flag(obj, 'vitest-test') as Test | undefined
   if (!test) {
     throw new Error(`'${getAssertionName(obj)}' cannot be used without test context`)
   }
-  return test as Test
+  if (test.fails) {
+    throw new TestSyntaxError(`'${getAssertionName(obj)}' cannot be used with 'test.fails'`)
+  }
+  return test
 }
 
 function validateAssertion(assertion: Chai.Assertion): void {

packages/vitest/src/node/printError.ts

@@ -284,6 +284,7 @@ const skipErrorProperties = new Set([
   'VITEST_TEST_PATH',
   '__vitest_rollup_error__',
   '__vitest_error_context__',
+  '__vitest_test_syntax_error__',
   ...Object.getOwnPropertyNames(Error.prototype),
   ...Object.getOwnPropertyNames(Object.prototype),
 ])

test/core/test/snapshot.test.ts

@@ -92,19 +92,6 @@ test('properties snapshot', () => {
   })
 })
 
-test.fails('properties snapshot fails', () => {
-  const user = {
-    createdAt: new Date(),
-    id: Math.floor(Math.random() * 20),
-    name: 'LeBron James',
-  }
-
-  expect(user).toMatchSnapshot({
-    createdAt: expect.any(Date),
-    id: expect.any(String),
-  })
-})
-
 test('renders mock snapshot', () => {
   const fn = vi.fn()
   expect(fn).toMatchSnapshot()

test/snapshots/test/snapshots.test.ts

@@ -1,6 +1,6 @@
 import { expect, test } from 'vitest'
 
-import { editFile, runVitest } from '../../test-utils'
+import { editFile, runInlineTests, runVitest } from '../../test-utils'
 
 test('non default snapshot format', () => {
   expect({ foo: ['bar'] }).toMatchInlineSnapshot(`
@@ -27,3 +27,94 @@ test('--update works for workspace project', async () => {
   expect.soft(stdout).include('Snapshots  1 updated')
   expect.soft(exitCode).toBe(0)
 })
+
+test('test.fails fails snapshot', async () => {
+  const result = await runInlineTests({
+    'basic.test.ts': `
+import { expect, test } from 'vitest'
+
+test.fails('file', () => {
+  expect('a').toMatchSnapshot()
+})
+
+test.fails('inline', () => {
+  expect('b').toMatchInlineSnapshot()
+})
+
+test.fails('soft', () => {
+  expect.soft('c').toMatchSnapshot()
+  expect.soft('d').toMatchInlineSnapshot()
+})
+`,
+  })
+  expect(result.stderr).toMatchInlineSnapshot(`
+    "
+    ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 3 ⎯⎯⎯⎯⎯⎯⎯
+
+     FAIL  basic.test.ts > file
+    TestSyntaxError: 'toMatchSnapshot' cannot be used with 'test.fails'
+     ❯ basic.test.ts:5:15
+          3|
+          4| test.fails('file', () => {
+          5|   expect('a').toMatchSnapshot()
+           |               ^
+          6| })
+          7|
+
+    ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/4]⎯
+
+     FAIL  basic.test.ts > inline
+    TestSyntaxError: 'toMatchInlineSnapshot' cannot be used with 'test.fails'
+     ❯ basic.test.ts:9:15
+          7|
+          8| test.fails('inline', () => {
+          9|   expect('b').toMatchInlineSnapshot()
+           |               ^
+         10| })
+         11|
+
+    ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/4]⎯
+
+     FAIL  basic.test.ts > soft
+    TestSyntaxError: 'toMatchSnapshot' cannot be used with 'test.fails'
+     ❯ basic.test.ts:13:20
+         11|
+         12| test.fails('soft', () => {
+         13|   expect.soft('c').toMatchSnapshot()
+           |                    ^
+         14|   expect.soft('d').toMatchInlineSnapshot()
+         15| })
+
+    ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/4]⎯
+
+     FAIL  basic.test.ts > soft
+    TestSyntaxError: 'toMatchInlineSnapshot' cannot be used with 'test.fails'
+     ❯ basic.test.ts:14:20
+         12| test.fails('soft', () => {
+         13|   expect.soft('c').toMatchSnapshot()
+         14|   expect.soft('d').toMatchInlineSnapshot()
+           |                    ^
+         15| })
+         16|
+
+    ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/4]⎯
+
+    "
+  `)
+  expect(result.errorTree()).toMatchInlineSnapshot(`
+    Object {
+      "basic.test.ts": Object {
+        "file": Array [
+          "'toMatchSnapshot' cannot be used with 'test.fails'",
+        ],
+        "inline": Array [
+          "'toMatchInlineSnapshot' cannot be used with 'test.fails'",
+        ],
+        "soft": Array [
+          "'toMatchSnapshot' cannot be used with 'test.fails'",
+          "'toMatchInlineSnapshot' cannot be used with 'test.fails'",
+        ],
+      },
+    }
+  `)
+})