Add Playwright Screenshots to Chromatic

.github/workflows/chromatic.yml → .github/workflows/storybook-playwright-snapshot.yml

@@ -1,4 +1,4 @@
-name: Chromatic Visual Testing
+name: Storybook Playwright Snapshot
 
 on:
     workflow_dispatch:
@@ -8,20 +8,24 @@ on:
         types: [opened, reopened, ready_for_review, synchronize]
         branches: [main]
 
-# Add concurrency to cancel in-progress jobs when a new workflow is triggered
+# Cancel in-progress jobs when new workflow is triggered
 concurrency:
     group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
     cancel-in-progress: true
 
 env:
+    DOCKER_BUILDKIT: 1
+    COMPOSE_DOCKER_CLI_BUILD: 1
     NODE_VERSION: 20.19.2
     PNPM_VERSION: 10.8.1
 
 jobs:
-    chromatic:
+    storybook-playwright-snapshot:
         runs-on: ubuntu-latest
+        timeout-minutes: 45
+
         steps:
-            - name: Checkout code
+            - name: Checkout repository
               uses: actions/checkout@v4
               with:
                   # Chromatic needs full git history for baseline comparison
@@ -32,14 +36,40 @@ jobs:
               with:
                   version: ${{ env.PNPM_VERSION }}
 
-            - name: Setup Node.js
+            - name: Set up Node.js
               uses: actions/setup-node@v4
               with:
                   node-version: ${{ env.NODE_VERSION }}
                   cache: "pnpm"
 
             - name: Install dependencies
-              run: pnpm install
+              run: pnpm install --frozen-lockfile
+
+            - name: Type check Playwright E2E
+              run: |
+                  cd apps/playwright-e2e
+                  pnpm check-types
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+
+            - name: Cache Docker layers
+              uses: actions/cache@v4
+              with:
+                  path: /tmp/.buildx-cache
+                  key: ${{ runner.os }}-buildx-${{ hashFiles('apps/playwright-e2e/Dockerfile.playwright-ci') }}
+                  restore-keys: ${{ runner.os }}-buildx-
+
+            - name: Run Playwright E2E tests
+              run: |
+                  cd apps/playwright-e2e
+                  node run-docker-playwright.js
+              env:
+                  OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+                  WORKSPACE_ROOT: ${{ github.workspace }}
+
+            - name: Generate Stories from Playwright Screenshots
+              run: pnpm --filter @roo-code/storybook generate-screenshot-stories
 
             - name: Build Storybook
               run: pnpm --filter @roo-code/storybook build-storybook
@@ -55,8 +85,6 @@ jobs:
                   exitOnceUploaded: ${{ github.event_name == 'pull_request' }}
                   # Skip dependabot PRs and renovate PRs using glob patterns
                   skip: "@(dependabot/**|renovate/**)"
-                  # Enable GitHub PR comments integration
-                  token: ${{ secrets.GITHUB_TOKEN }}
                   # Auto-accept changes on main branch to keep it clean
                   autoAcceptChanges: ${{ github.ref == 'refs/heads/main' && 'main' || '' }}
 

apps/playwright-e2e/Dockerfile.playwright-ci

@@ -5,7 +5,10 @@
 FROM mcr.microsoft.com/playwright:v1.53.1-noble
 
 # Install system dependencies (rarely changes - good for caching)
-RUN apt-get update && apt-get install -y \
+# Use BuildKit cache mounts for faster APT operations
+RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt \
+    --mount=type=cache,id=apt-lists,target=/var/lib/apt/lists \
+    apt-get update && apt-get install -y \
     # VSCode dependencies
     libasound2t64 \
     libatk-bridge2.0-0 \
@@ -29,9 +32,7 @@ RUN apt-get update && apt-get install -y \
     # VS Code secrets API support in Docker
     gnome-keyring \
     libsecret-1-0 \
-    libsecret-1-dev \
-    # Clean up
-    && rm -rf /var/lib/apt/lists/*
+    libsecret-1-dev
 
 # Install pnpm globally (rarely changes - good for caching)
 RUN npm install -g [email protected]
@@ -47,9 +48,8 @@ ENV NODE_ENV=production \
 # Create workspace directory
 WORKDIR /workspace
 
-# Copy entrypoint script (changes occasionally - separate layer for better caching)
-COPY apps/playwright-e2e/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
-RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+# Copy entrypoint script with execute permissions
+COPY --chmod=755 apps/playwright-e2e/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 # Set entrypoint
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]

apps/playwright-e2e/package.json

@@ -13,9 +13,9 @@
 	},
 	"devDependencies": {
 		"@playwright/test": "^1.53.1",
-		"@roo-code/types": "workspace:^",
 		"@roo-code/config-eslint": "workspace:^",
 		"@roo-code/config-typescript": "workspace:^",
+		"@roo-code/types": "workspace:^",
 		"@types/node": "^22.15.29",
 		"@vscode/test-electron": "^2.4.0",
 		"dotenv": "^16.4.5",
@@ -24,6 +24,7 @@
 	},
 	"dependencies": {
 		"chalk": "^5.4.1",
+		"change-case": "^5.4.4",
 		"fs-extra": "^11.2.0",
 		"signale": "^1.4.0"
 	}

apps/playwright-e2e/playwright.config.ts

@@ -16,21 +16,17 @@ export default defineConfig<void, TestOptions>({
 	expect: { timeout: 30_000 },
 	reporter: process.env.CI ? "html" : "list",
 	workers: process.env.CI ? 2 : 1,
-	retries: process.env.CI ? 3 : 1, // Retry in CI, 1 time locally
+	retries: process.env.CI ? 2 : 0, // Retry in CI, never locally
 	globalSetup: "./playwright.globalSetup",
 	testDir: "./tests",
 	testIgnore: "**/helpers/__tests__/**",
 	outputDir: "./test-results",
 	projects: [
 		// { name: "VSCode insiders", use: { vscodeVersion: "insiders" } },
-		{
-			name: "VSCode stable",
-			use: { vscodeVersion: "stable" },
-		},
+		{ name: "VSCode stable", use: { vscodeVersion: "stable" } },
 	],
 	use: {
 		trace: "on-first-retry",
-		screenshot: "on", // Capture screenshots for all tests
-		video: "retry-with-video", // Record videos for retries
+		video: "retry-with-video", // videos only on retries
 	},
 })

apps/playwright-e2e/run-docker-playwright.js

@@ -89,8 +89,7 @@ function killAllChildProcesses() {
 						child.kill("SIGKILL")
 					}
 				}, 5000)
-			} catch (_error) {
-			}
+			} catch (_error) {}
 		}
 
 		activeProcesses.clear()
@@ -171,7 +170,7 @@ async function buildDockerImage() {
 			"--cache-from",
 			"type=local,src=/tmp/.buildx-cache",
 			"--cache-to",
-			"type=local,dest=/tmp/.buildx-cache-new,mode=max",
+			"type=local,dest=/tmp/.buildx-cache,mode=max",
 		)
 	}
 

apps/playwright-e2e/tests/chat-with-response.test.ts

@@ -7,21 +7,26 @@ import {
 } from "../helpers/webview-helpers"
 
 test.describe("Full E2E Test", () => {
-	test("should configure credentials and send a message", async ({ workbox: page }: TestFixtures) => {
+	test("should configure credentials and send a message", async ({ workbox: page, takeScreenshot }: TestFixtures) => {
 		await verifyExtensionInstalled(page)
 
 		await waitForWebviewText(page, "Welcome to Kilo Code!")
+		await takeScreenshot("welcome")
 
 		await configureApiKeyThroughUI(page)
-
 		await waitForWebviewText(page, "Generate, refactor, and debug code with AI assistance")
+		await takeScreenshot("ready-to-chat")
 
 		const webviewFrame = await findWebview(page)
 		const chatInput = webviewFrame.locator('textarea, input[type="text"]').first()
-		await chatInput.waitFor({ timeout: 5000 })
+		await chatInput.waitFor()
+
+		await chatInput.fill("Fill in the blanks for this phrase: 'hello w_r_d'")
+		await takeScreenshot("chat-prompt-entered")
 
-		await chatInput.fill("Output only the result of '1+1'")
+		// Don't take any more screenshots after the reponse starts-
+		// llm responses aren't deterministic any capturing the reponse would cause screenshot flakes
 		await chatInput.press("Enter")
-		await waitForWebviewText(page, "2", 30_000)
+		await waitForWebviewText(page, "hello world", 30_000)
 	})
 })

apps/playwright-e2e/tests/playwright-base-test.ts

@@ -4,6 +4,7 @@ import * as path from "path"
 import * as os from "os"
 import * as fs from "fs"
 import { fileURLToPath } from "url"
+import { camelCase } from "change-case"
 import { setupConsoleLogging, cleanLogMessage } from "../helpers/console-logging"
 
 // ES module equivalent of __dirname
@@ -18,6 +19,7 @@ export type TestFixtures = TestOptions & {
 	workbox: Page
 	createProject: () => Promise<string>
 	createTempDir: () => Promise<string>
+	takeScreenshot: (name?: string) => Promise<void>
 }
 
 export const test = base.extend<TestFixtures>({
@@ -135,9 +137,29 @@ export const test = base.extend<TestFixtures>({
 	// eslint-disable-next-line no-empty-pattern
 	createTempDir: async ({}, use) => {
 		const tempDirs: string[] = []
+		let counter = 0
+
 		await use(async () => {
-			const tempDirPath = await fs.promises.mkdtemp(path.join(os.tmpdir(), "e2e-test-"))
+			const testInfo = test.info()
+			const fileName = testInfo.file.split("/").pop()?.replace(".test.ts", "") || "unknown"
+			const sanitizedTestName = camelCase(testInfo.title)
+
+			const dirName = `e2e-${fileName}-${sanitizedTestName}-${counter++}`
+			const tempDirPath = path.join(os.tmpdir(), dirName)
+
+			// Clean up any existing directory first
+			try {
+				await fs.promises.rm(tempDirPath, { recursive: true })
+			} catch (_error) {
+				// Directory might not exist, which is fine
+			}
+
+			// Create the directory
+			await fs.promises.mkdir(tempDirPath, { recursive: true })
+
+			// Get the real path after directory exists
 			const tempDir = await fs.promises.realpath(tempDirPath)
+
 			tempDirs.push(tempDir)
 			return tempDir
 		})
@@ -150,4 +172,25 @@ export const test = base.extend<TestFixtures>({
 			}
 		}
 	},
+
+	takeScreenshot: async ({ workbox }, use) => {
+		await use(async (name?: string) => {
+			const testInfo = test.info()
+			// Extract test suite from the test file name or use a default
+			const fileName = testInfo.file.split("/").pop()?.replace(".test.ts", "") || "unknown"
+			const testSuite = camelCase(fileName)
+			const testName = testInfo.title || "Unknown Test"
+
+			// Create a hierarchical name: TestSuite__TestName__ScreenshotName
+			const screenshotName = name || `screenshot-${Date.now()}`
+			const hierarchicalName = `${testSuite}__${testName}__${screenshotName}`
+				.replace(/[^a-zA-Z0-9_-]/g, "-") // Replace special chars with dashes, keep underscores
+				.replace(/-+/g, "-") // Replace multiple dashes with single dash
+				.replace(/^-|-$/g, "") // Remove leading/trailing dashes
+
+			const screenshotPath = test.info().outputPath(`${hierarchicalName}.png`)
+			await workbox.screenshot({ path: screenshotPath, fullPage: true })
+			console.log(`📸 Screenshot captured: ${hierarchicalName}`)
+		})
+	},
 })

apps/playwright-e2e/tests/sanity.test.ts

@@ -2,7 +2,7 @@ import { test, expect, type TestFixtures } from "./playwright-base-test"
 import { verifyExtensionInstalled } from "../helpers/webview-helpers"
 
 test.describe("Sanity Tests", () => {
-	test("should launch VS Code with extension installed", async ({ workbox: page }: TestFixtures) => {
+	test("should launch VS Code with extension installed", async ({ workbox: page, takeScreenshot }: TestFixtures) => {
 		await expect(page.locator(".monaco-workbench")).toBeVisible()
 		console.log("✅ VS Code launched successfully")
 
@@ -20,5 +20,7 @@ test.describe("Sanity Tests", () => {
 		console.log("✅ Command palette working")
 
 		await verifyExtensionInstalled(page)
+
+		await takeScreenshot("sanity-extension-loaded")
 	})
 })