Add live regression canaries for folder-bind and mismatch states

Author: ferologics

.github/workflows/ci.yml

@@ -33,6 +33,9 @@ jobs:
       - name: Setup mise
         uses: jdx/mise-action@v2
 
+      - name: Install sccache
+        run: brew install sccache
+
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
@@ -66,8 +69,8 @@ jobs:
       - name: Setup mise
         uses: jdx/mise-action@v2
 
-      - name: Install qemu
-        run: brew install qemu
+      - name: Install runtime deps
+        run: brew install qemu sccache
 
       - name: Install pi CLI for runtime build
         run: npm install -g @mariozechner/pi-coding-agent

TODO.md

@@ -7,7 +7,7 @@
 - [ ] **Testing sanity gate (P0)** — define and ship a real automated regression suite for the liftoff path (Vitest + Rust), keep daily checks fast (`mise run check`) while enforcing live regressions in `mise run check-full`, and stop relying on ad-hoc/manual shell-harness runs for core correctness.
   - [x] Add a machine-readable `state_snapshot` contract for deterministic assertions (no log-grep testing).
   - [x] Add first gated Vitest regression: reopen cwd correctness (`/mnt/workdir...`) on folder-bound task reopen.
-  - [ ] Add remaining gated Vitest regressions: folder-bind continuity (no UI reset), working-folder panel refresh on folder change, and runtime-mismatch badge rules.
+  - [x] Add remaining gated Vitest regressions: folder-bind continuity (no UI reset), working-folder panel refresh on folder change, and runtime-mismatch badge rules.
   - [ ] Add one sequential live journey canary (messages + models + workdir + artifacts + reopen), while keeping focused canaries for isolated invariants.
   - [x] Add CI enforcement for both gates (`check` on PRs, `check-full` required before merge).
   - [x] Remove low-signal/noise tests and stale assertions (smoke test, legacy session-file assumptions, noisy debug logs).

docs/testing-strategy.md

@@ -59,6 +59,13 @@ The regression gate should make these guarantees explicit:
 4. **Runtime mismatch badge semantics**
    - Badge only appears for real mismatches, not transient boot/reconfigure or legacy sentinel state.
 
+Implemented live canaries (integration):
+
+- `reopen-cwd.integration.test.ts`
+- `folder-bind-continuity.integration.test.ts`
+- `working-folder-panel-refresh.integration.test.ts`
+- `runtime-mismatch-badge.integration.test.ts`
+
 ## Live integration strategy (hybrid)
 
 - Keep **one sequential journey canary** that covers a realistic end-to-end flow (messages, models, working folder, artifacts, reopen).

mise.toml

@@ -73,7 +73,7 @@ run = "mise run test-vite && mise run test-rust"
 
 [tasks.test-regressions]
 description = "Run runtime regression integration tests"
-run = "pnpm exec vitest run src/lib/__tests__/integration/*.integration.test.ts"
+run = "pnpm exec vitest run --minWorkers=1 --maxWorkers=1 src/lib/__tests__/integration/*.integration.test.ts"
 
 [tasks.test-rust]
 description = "Run Rust tests"

src/lib/__tests__/integration/folder-bind-continuity.integration.test.ts

@@ -0,0 +1,86 @@
+/* @vitest-environment node */
+
+import path from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { IntegrationHarness } from "./harness";
+
+describe.sequential("folder bind continuity regression", () => {
+    const harness = new IntegrationHarness();
+    const prefix = `regression-folder-bind-${Date.now()}`;
+
+    const workingFolder = path.resolve(process.cwd());
+
+    beforeAll(async () => {
+        await harness.start();
+        await harness.deleteTasksByPrefix(prefix);
+    }, 240_000);
+
+    afterAll(async () => {
+        await harness.deleteTasksByPrefix(prefix).catch(() => undefined);
+        await harness.stop();
+    }, 90_000);
+
+    it("does not reset conversation state when binding first working folder", async () => {
+        const title = `${prefix}-task`;
+        await harness.createTask(title, null);
+
+        const task = await harness.waitForTaskByTitle(title, 30_000);
+        await harness.setTask(task.id);
+        await harness.waitForTaskSettled(task.id, 90_000);
+
+        const initial = await harness.waitForSnapshot((snapshot) => {
+            if (snapshot.task.currentTaskId !== task.id) {
+                return null;
+            }
+
+            if (!snapshot.runtime.rpcConnected || snapshot.runtime.taskSwitching) {
+                return null;
+            }
+
+            return snapshot;
+        }, 60_000);
+
+        expect(initial.ui.quickStartVisible).toBe(true);
+
+        await harness.prompt("continuity probe");
+
+        const withMessage = await harness.waitForSnapshot((snapshot) => {
+            if (snapshot.task.currentTaskId !== task.id) {
+                return null;
+            }
+
+            if (snapshot.conversation.messageCount < 1) {
+                return null;
+            }
+
+            return snapshot;
+        }, 90_000);
+
+        const messageCountBeforeBind = withMessage.conversation.messageCount;
+        expect(messageCountBeforeBind).toBeGreaterThan(0);
+        expect(withMessage.ui.quickStartVisible).toBe(false);
+
+        await harness.setFolder(workingFolder);
+
+        const afterBind = await harness.waitForSnapshot((snapshot) => {
+            if (snapshot.task.currentTaskId !== task.id) {
+                return null;
+            }
+
+            if (!snapshot.runtime.rpcConnected || snapshot.runtime.taskSwitching) {
+                return null;
+            }
+
+            if (snapshot.task.currentWorkingFolder !== workingFolder) {
+                return null;
+            }
+
+            return snapshot;
+        }, 120_000);
+
+        expect(afterBind.conversation.messageCount).toBeGreaterThanOrEqual(messageCountBeforeBind);
+        expect(afterBind.conversation.messageCount).toBeGreaterThan(0);
+        expect(afterBind.ui.bootScreenVisible).toBe(false);
+        expect(afterBind.ui.quickStartVisible).toBe(false);
+    }, 240_000);
+});

src/lib/__tests__/integration/harness.ts

@@ -20,6 +20,10 @@ interface TaskSummary {
     workingFolder: string | null;
 }
 
+function isOkResponse(response: string): boolean {
+    return response === "OK";
+}
+
 export interface StateSnapshot {
     schemaVersion: number;
     timestamp: string;
@@ -62,6 +66,11 @@ export interface StateSnapshot {
         workingFolderRelative: string | null;
         mismatchVisible: boolean;
     };
+    panels: {
+        workingFolderFileRowCount: number;
+        workingFolderLoadingVisible: boolean;
+        workingFolderEmptyVisible: boolean;
+    };
 }
 
 function sleep(ms: number): Promise<void> {
@@ -271,12 +280,14 @@ export class IntegrationHarness {
     ): Promise<T> {
         return await waitFor(
             async () => {
+                let snapshot: StateSnapshot;
                 try {
-                    const snapshot = await this.snapshot();
-                    return predicate(snapshot);
+                    snapshot = await this.snapshot();
                 } catch {
                     return null;
                 }
+
+                return predicate(snapshot);
             },
             timeoutMs,
             250,
@@ -291,11 +302,25 @@ export class IntegrationHarness {
             workingFolder,
         });
 
-        if (response !== "OK") {
+        if (!isOkResponse(response)) {
             throw new Error(`create_task failed: ${response}`);
         }
     }
 
+    async prompt(message: string): Promise<void> {
+        const response = await this.sendCommand({ cmd: "prompt", message });
+        if (!isOkResponse(response)) {
+            throw new Error(`prompt failed: ${response}`);
+        }
+    }
+
+    async setFolder(folder: string): Promise<void> {
+        const response = await this.sendCommand({ cmd: "set_folder", folder });
+        if (!isOkResponse(response)) {
+            throw new Error(`set_folder failed: ${response}`);
+        }
+    }
+
     async listTasks(): Promise<TaskSummary[]> {
         return await this.sendJson<TaskSummary[]>({ cmd: "task_list" });
     }
@@ -319,14 +344,14 @@ export class IntegrationHarness {
 
     async setTask(taskId: string): Promise<void> {
         const response = await this.sendCommand({ cmd: "set_task", taskId });
-        if (response !== "OK") {
+        if (!isOkResponse(response)) {
             throw new Error(`set_task failed: ${response}`);
         }
     }
 
     async deleteTask(taskId: string): Promise<void> {
         const response = await this.sendCommand({ cmd: "delete_task", taskId });
-        if (response !== "OK") {
+        if (!isOkResponse(response)) {
             throw new Error(`delete_task failed: ${response}`);
         }
     }
@@ -340,6 +365,20 @@ export class IntegrationHarness {
         }
     }
 
+    async waitForTaskSettled(taskId: string, timeoutMs = 120_000): Promise<StateSnapshot> {
+        return await this.waitForSnapshot((snapshot) => {
+            if (!snapshot.runtime.rpcConnected || snapshot.runtime.taskSwitching) {
+                return null;
+            }
+
+            if (snapshot.task.currentTaskId !== taskId) {
+                return null;
+            }
+
+            return snapshot;
+        }, timeoutMs);
+    }
+
     logPath(): string {
         return LOG_PATH;
     }

src/lib/__tests__/integration/runtime-mismatch-badge.integration.test.ts

@@ -0,0 +1,61 @@
+/* @vitest-environment node */
+
+import path from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { IntegrationHarness } from "./harness";
+
+describe.sequential("runtime mismatch badge semantics regression", () => {
+    const harness = new IntegrationHarness();
+    const prefix = `regression-mismatch-${Date.now()}`;
+
+    const workingFolder = path.resolve(process.cwd());
+
+    beforeAll(async () => {
+        await harness.start();
+        await harness.deleteTasksByPrefix(prefix);
+    }, 240_000);
+
+    afterAll(async () => {
+        await harness.deleteTasksByPrefix(prefix).catch(() => undefined);
+        await harness.stop();
+    }, 90_000);
+
+    it("does not show runtime mismatch badge during transient reconfigure states", async () => {
+        const title = `${prefix}-task`;
+        await harness.createTask(title, null);
+
+        const task = await harness.waitForTaskByTitle(title, 30_000);
+        await harness.setTask(task.id);
+
+        const settledBeforeBind = await harness.waitForTaskSettled(task.id, 90_000);
+        expect(settledBeforeBind.runtimeDebug.mismatchVisible).toBe(false);
+
+        await harness.setFolder(workingFolder);
+
+        const settledAfterBind = await harness.waitForSnapshot((snapshot) => {
+            if (snapshot.task.currentTaskId !== task.id) {
+                return null;
+            }
+
+            const transient =
+                snapshot.runtime.taskSwitching ||
+                !snapshot.runtime.rpcConnected ||
+                snapshot.ui.reconfigureBannerVisible;
+
+            if (transient) {
+                if (snapshot.runtimeDebug.mismatchVisible) {
+                    throw new Error("runtime mismatch badge became visible during transient reconfigure state");
+                }
+                return null;
+            }
+
+            if (snapshot.task.currentWorkingFolder !== workingFolder) {
+                return null;
+            }
+
+            return snapshot;
+        }, 120_000);
+
+        expect(settledAfterBind.runtimeDebug.mismatchVisible).toBe(false);
+    }, 240_000);
+});

src/lib/__tests__/integration/working-folder-panel-refresh.integration.test.ts

@@ -0,0 +1,74 @@
+/* @vitest-environment node */
+
+import path from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { IntegrationHarness } from "./harness";
+
+describe.sequential("working folder panel refresh regression", () => {
+    const harness = new IntegrationHarness();
+    const prefix = `regression-working-panel-${Date.now()}`;
+
+    const workingFolder = path.resolve(process.cwd());
+
+    beforeAll(async () => {
+        await harness.start();
+        await harness.deleteTasksByPrefix(prefix);
+    }, 240_000);
+
+    afterAll(async () => {
+        await harness.deleteTasksByPrefix(prefix).catch(() => undefined);
+        await harness.stop();
+    }, 90_000);
+
+    it("refreshes working-folder file list immediately after first bind", async () => {
+        const title = `${prefix}-task`;
+        await harness.createTask(title, null);
+
+        const task = await harness.waitForTaskByTitle(title, 30_000);
+        await harness.setTask(task.id);
+        await harness.waitForTaskSettled(task.id, 90_000);
+
+        const beforeBind = await harness.waitForSnapshot((snapshot) => {
+            if (snapshot.task.currentTaskId !== task.id) {
+                return null;
+            }
+
+            if (!snapshot.runtime.rpcConnected || snapshot.runtime.taskSwitching) {
+                return null;
+            }
+
+            if (snapshot.task.currentWorkingFolder !== null) {
+                return null;
+            }
+
+            return snapshot;
+        }, 60_000);
+
+        expect(beforeBind.panels.workingFolderFileRowCount).toBe(0);
+
+        await harness.setFolder(workingFolder);
+
+        const afterBind = await harness.waitForSnapshot((snapshot) => {
+            if (snapshot.task.currentTaskId !== task.id) {
+                return null;
+            }
+
+            if (!snapshot.runtime.rpcConnected || snapshot.runtime.taskSwitching) {
+                return null;
+            }
+
+            if (snapshot.task.currentWorkingFolder !== workingFolder) {
+                return null;
+            }
+
+            if (snapshot.panels.workingFolderFileRowCount < 1) {
+                return null;
+            }
+
+            return snapshot;
+        }, 120_000);
+
+        expect(afterBind.panels.workingFolderFileRowCount).toBeGreaterThan(0);
+        expect(afterBind.panels.workingFolderEmptyVisible).toBe(false);
+    }, 240_000);
+});