Basic playwright electron support (#12207)

This change allows to write tests against an Electron app by starting the
Electron app first and then attaching Playwright to it. Also, it solves
the workspace problem by creating it upfront and pointing to it on app
start.

Because playwright cannot access native UI elements, we introduce an
optional environment variable THEIA_ELECTRON_DISABLE_NATIVE_ELEMENTS
that, when set to 1, enforces the rendering of HTML-based menus and file
chooser dialogs, which can be accessed by playwright.

Playwright also requires us to disable early rendering of the window,
which we also do via an environment variable
(THEIA_ELECTRON_NO_EARLY_WINDOW).

A unified TheiaAppLoader is introduced that can be used in both browser
and electron tests. All existing tests are adjusted to use the new
AppLoader and can be run in Electron if the environment variable
USE_ELECTRON is set to true. For convenience, a yarn target
ui-tests-electron has been added to the playwright package.json.
Furthermore, the target ui-tests-ci has been added (runs all playwright
tests in the browser and in Electron) and the Github workflow has been
adjusted.

Contributed on behalf of STMicroelectronics

Signed-off-by: Olaf Lessenich <[email protected]>
Co-authored-by: Philip Langer <[email protected]>

Author: xai

.github/workflows/playwright.yml

@@ -51,4 +51,4 @@ jobs:
       - name: Test (playwright)
         uses: GabrielBB/xvfb-action@v1
         with:
-          run: yarn test:playwright
+          run: yarn --cwd examples/playwright ui-tests-ci

examples/playwright/configs/playwright.ci.config.ts

@@ -0,0 +1,27 @@
+// *****************************************************************************
+// Copyright (C) 2022 EclipseSource and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { PlaywrightTestConfig } from '@playwright/test';
+import baseConfig from './playwright.config';
+
+const ciConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    retries: 2,
+    reporter: [['list'], ['allure-playwright'], ['github']]
+};
+
+export default ciConfig;

examples/playwright/playwright.config.ts …/playwright/configs/playwright.config.tsexamples/playwright/playwright.config.ts renamed to examples/playwright/configs/playwright.config.ts examples/playwright/playwright.config.ts renamed to examples/playwright/configs/playwright.config.ts

@@ -1,5 +1,5 @@
 // *****************************************************************************
-// Copyright (C) 2021-2023 logi.cals GmbH, EclipseSource and others.
+// Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
 //
 // This program and the accompanying materials are made available under the
 // terms of the Eclipse Public License v. 2.0 which is available at
@@ -11,41 +11,35 @@
 // with the GNU Classpath Exception which is available at
 // https://www.gnu.org/software/classpath/license.html.
 //
-// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import { defineConfig } from '@playwright/test';
+import { PlaywrightTestConfig } from '@playwright/test';
 
-// Note: process.env.CI is always set to true for GitHub Actions
-
-export default defineConfig({
-    testDir: './lib/tests',
-    testMatch: ['**/*.test.js'],
-    workers: process.env.CI ? 1 : 2,
-    retries: process.env.CI ? 1 : 0,
-    // The number of times to repeat each test, useful for debugging flaky tests
-    repeatEach: 1,
+const config: PlaywrightTestConfig = {
+    testDir: '../lib/tests',
+    testMatch: ['**/*.js'],
+    workers: 1,
+    fullyParallel: false,
     // Timeout for each test in milliseconds.
-    timeout: 30 * 1000,
+    timeout: 60 * 1000,
     use: {
         baseURL: 'http://localhost:3000',
         browserName: 'chromium',
-        screenshot: 'only-on-failure',
         permissions: ['clipboard-read'],
-        viewport: { width: 1920, height: 1080 },
-    },
-    snapshotDir: './src/tests/snapshots',
-    expect: {
-        toMatchSnapshot: { threshold: 0.01 }
+        screenshot: 'only-on-failure'
     },
     preserveOutput: 'failures-only',
-    reporter: process.env.CI
-        ? [['list'], ['allure-playwright'], ['github']]
-        : [['list'], ['allure-playwright']],
+    reporter: [
+        ['list'],
+        ['allure-playwright']
+    ],
     // Reuse Theia backend on port 3000 or start instance before executing the tests
     webServer: {
         command: 'yarn theia:start',
         port: 3000,
         reuseExistingServer: true
     }
-});
+};
+
+export default config;

…ight/src/tests/fixtures/theia-fixture.ts …right/configs/playwright.debug.config.tsexamples/playwright/src/tests/fixtures/theia-fixture.ts renamed to examples/playwright/configs/playwright.debug.config.ts examples/playwright/src/tests/fixtures/theia-fixture.ts renamed to examples/playwright/configs/playwright.debug.config.ts

@@ -11,15 +11,17 @@
 // with the GNU Classpath Exception which is available at
 // https://www.gnu.org/software/classpath/license.html.
 //
-// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import test, { Page } from '@playwright/test';
+import { PlaywrightTestConfig } from '@playwright/test';
 
-export let page: Page;
+import baseConfig from './playwright.config';
 
-test.beforeAll(async ({ browser }) => {
-    page = await browser.newPage();
-});
+const debugConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    timeout: 15000000
+};
 
-export default test;
+export default debugConfig;

examples/playwright/configs/playwright.headful.config.ts

@@ -0,0 +1,30 @@
+// *****************************************************************************
+// Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { PlaywrightTestConfig } from '@playwright/test';
+
+import baseConfig from './playwright.config';
+
+const headfulConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    use: {
+        ...baseConfig.use,
+        headless: false
+    }
+};
+
+export default headfulConfig;

examples/playwright/configs/ui-tests.eslintrc.json

@@ -0,0 +1,7 @@
+{
+  // override existing rules for ui-tests package
+  "rules": {
+    "no-undef": "off", // disabled due to 'browser', '$', '$$'
+    "no-unused-expressions": "off"
+  }
+}

examples/playwright/configs/ui-tests.playwright.eslintrc.json

@@ -0,0 +1,6 @@
+{
+  // override existing rules for ui-tests playwright package
+  "rules": {
+    "no-null/no-null": "off"
+  }
+}

examples/playwright/package.json

@@ -19,8 +19,10 @@
     "lint": "eslint -c ./.eslintrc.js --ext .ts ./src",
     "lint:fix": "eslint -c ./.eslintrc.js --ext .ts ./src --fix",
     "playwright:install": "playwright install chromium",
-    "ui-tests": "yarn build && playwright test",
-    "ui-tests-headful": "yarn build && playwright test --headed",
+    "ui-tests": "yarn build && playwright test --config=./configs/playwright.config.ts",
+    "ui-tests-electron": "yarn build && USE_ELECTRON=true playwright test --config=./configs/playwright.config.ts",
+    "ui-tests-ci": "yarn build && playwright test --config=./configs/playwright.ci.config.ts",
+    "ui-tests-headful": "yarn build && playwright test --config=./configs/playwright.headful.config.ts",
     "ui-tests-report-generate": "allure generate ./allure-results --clean -o allure-results/allure-report",
     "ui-tests-report": "yarn ui-tests-report-generate && allure open allure-results/allure-report"
   },

examples/playwright/src/index.ts

@@ -16,6 +16,7 @@
 
 export * from './theia-about-dialog';
 export * from './theia-app';
+export * from './theia-app-loader';
 export * from './theia-context-menu';
 export * from './theia-dialog';
 export * from './theia-editor';

examples/playwright/src/tests/theia-app.test.ts

@@ -14,20 +14,19 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
 
+import { expect, test } from '@playwright/test';
+import { TheiaAppLoader } from '../theia-app-loader';
 import { TheiaApp } from '../theia-app';
 
-import { expect } from '@playwright/test';
-import test, { page } from './fixtures/theia-fixture';
-
-let app: TheiaApp;
-
 test.describe('Theia Application', () => {
+    let app: TheiaApp;
 
-    test('should load', async () => {
-        app = await TheiaApp.load(page);
+    test.afterAll(async () => {
+        await app.page.close();
     });
 
-    test('should show main content panel', async () => {
+    test('should load and should show main content panel', async ({ playwright, browser }) => {
+        app = await TheiaAppLoader.load({ playwright, browser });
         expect(await app.isMainContentPanelVisible()).toBe(true);
     });