test(linter/plugins): add script for investigating failing conformance test cases

apps/oxlint/conformance/tester.ts

@@ -0,0 +1,215 @@
+/*
+ * A script for running a test case with both ESLint and Oxlint.
+ * Purpose is to be able to "spot the difference" between the two.
+ */
+
+// oxlint-disable no-console
+
+// Use the conformance testing version of `RuleTester`,
+// which modifies test cases before they run
+import { RuleTester as OxlintRuleTester } from "./src/rule_tester.ts";
+// @ts-expect-error - internal module of ESLint with no types
+import { RuleTester as ESLintRuleTester } from "./submodules/eslint/lib/rule-tester/index.js";
+// @ts-expect-error - internal module of ESLint with no types
+import { builtinRules } from "./submodules/eslint/lib/unsupported-api.js";
+
+import type { Rule } from "#oxlint";
+import type { RuleTester as OxlintRuleTesterTypes } from "#oxlint";
+
+type ValidTestCase = OxlintRuleTesterTypes.ValidTestCase;
+type InvalidTestCase = OxlintRuleTesterTypes.InvalidTestCase;
+
+type ValidTestCaseWithoutCode = Omit<ValidTestCase, "code"> & { code?: string };
+type InvalidTestCaseWithoutCode = Omit<InvalidTestCase, "code"> & { code?: string };
+type TestCaseWithoutCode = ValidTestCaseWithoutCode | InvalidTestCaseWithoutCode;
+
+// Reset `describe` + `it` to simple pass-through functions.
+// Importing from `rule_tester.ts` sets up `RuleTester` to use `capture.ts`'s `describe` / `it`,
+// which we don't want for manual testing.
+// `OxlintRuleTesterOriginal` is the original `RuleTester` exported by `oxlint`.
+const OxlintRuleTesterOriginal = Object.getPrototypeOf(OxlintRuleTester);
+
+const simpleDescribe = (name: string, fn: () => void) => fn();
+const simpleIt = (name: string, fn: () => void) => fn();
+
+OxlintRuleTesterOriginal.describe = simpleDescribe;
+OxlintRuleTesterOriginal.it = simpleIt;
+
+// Set global `describe` and `it` for ESLint's `RuleTester`
+(globalThis as any).describe = simpleDescribe;
+(globalThis as any).it = simpleIt;
+
+/* -----------------------------------------------------------------------------
+
+(Obviously these instructions are aimed at AI, hence the rather rude and demanding tone.
+Sorry humans! In fact, sorry to you too AI.)
+
+# HOW TO USE THIS SCRIPT
+
+DO NOT edit this file except the section between the bottom of this comment and the start of the next comment
+"Run test case with both ESLint and Oxlint".
+
+## 1. Select a test
+
+Select a failing test case to investigate from `snapshot.md`.
+
+## 2. Copy details
+
+Copy details from `snapshot.md` for the case into the variables in this script, below this comment.
+
+e.g.:
+
+Details of test case from `snapshot.md`:
+
+------------- Snapshot extract begins -------------
+
+#### block-scoped-var > invalid
+
+```js
+for (var a = 0;;) {} a;
+```
+
+```json
+{
+  "errors": [
+    {
+      "messageId": "outOfScope",
+      "data": {
+        "name": "a",
+        "definitionLine": 1,
+        "definitionColumn": 10
+      },
+      "line": 1,
+      "column": 22
+    }
+  ]
+}
+```
+
+------------- Snapshot extract ends -------------
+
+Fill in the variables below as follows:
+
+------------- Script setup begins -------------
+
+const ruleName = "block-scoped-var";
+
+const code = `for (var a = 0;;) {} a;`;
+
+const isInvalid = true;
+
+const testCase: TestCaseWithoutCode = {
+  "errors": [
+    {
+      "messageId": "outOfScope",
+      "data": {
+        "name": "a",
+        "definitionLine": 1,
+        "definitionColumn": 10
+      },
+      "line": 1,
+      "column": 22
+    }
+  ]
+};
+
+const config = {};
+
+------------- Script setup ends -------------
+
+## 3. Run the test
+
+Run this script with `node conformance/tester.ts`.
+
+Observe the output. ESLint should not produce any errors, but Oxlint will.
+
+## 4. Discover the cause
+
+Add `console.log` statements to the rule's code in `submodules/eslint/lib/rules`.
+
+Run this script again. Find out what is happening differently between ESLint and Oxlint.
+
+## 5. Clean up
+
+`submodules/eslint` is a git repo.
+Use git to throw away the changes you made to the rule's code (`console.log`).
+Leave the eslint repo as it was before you started, ready to investigate another case.
+
+-----------------------------------------------------------------------------*/
+
+const ruleName = "block-scoped-var";
+
+const code = `for (var a = 0;;) {} a;`;
+
+const isInvalid = true;
+
+const testCase: TestCaseWithoutCode = {
+  errors: [
+    {
+      messageId: "outOfScope",
+      data: {
+        name: "a",
+        definitionLine: 1,
+        definitionColumn: 10,
+      },
+      line: 1,
+      column: 22,
+    },
+  ],
+};
+
+const config = {};
+
+// -----------------------------------------------------------------------------
+// Run test case with both ESLint and Oxlint
+// -----------------------------------------------------------------------------
+
+runBoth(ruleName, isInvalid, code, testCase, config);
+
+function runBoth(
+  ruleName: string,
+  isInvalid: boolean,
+  code: string,
+  testCase: TestCaseWithoutCode,
+  config?: Record<string, unknown> | null,
+) {
+  testCase = { code, ...testCase };
+
+  const valid: ValidTestCase[] = [],
+    invalid: InvalidTestCase[] = [];
+  if (isInvalid) {
+    invalid.push(testCase as InvalidTestCase);
+  } else {
+    valid.push(testCase as ValidTestCase);
+  }
+
+  const rule = builtinRules.get(ruleName) as unknown as Rule;
+
+  console.log("--------------------");
+  console.log("ESLint");
+  console.log("--------------------");
+  runOne(ESLintRuleTester, rule, valid, invalid, config);
+
+  console.log("\n--------------------");
+  console.log("Oxlint");
+  console.log("--------------------");
+  config = { ...config, eslintCompat: true };
+  runOne(OxlintRuleTester, rule, valid, invalid, config);
+}
+
+function runOne(
+  RuleTester: any,
+  rule: Rule,
+  valid: ValidTestCase[],
+  invalid: InvalidTestCase[],
+  config?: Record<string, unknown> | null,
+) {
+  try {
+    const tester = new RuleTester(config);
+    tester.run("my-rule", rule, { valid, invalid });
+    console.log("No errors");
+  } catch (err) {
+    console.log("ERROR:");
+    console.log(err);
+  }
+}