Run E2E tests on windows and macOS instead of linux [WPB-26690]

.github/workflows/e2e-test.yml

@@ -15,9 +15,18 @@ concurrency:
 
 jobs:
   run_tests:
-    name: Run E2E Tests
-    runs-on: ubuntu-24.04
+    name: Run E2E Tests (${{ matrix.project }})
     if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-e2e') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        project: [windows, macOS]
+        include:
+          - project: windows
+            os: windows-2025
+          - project: macOS
+            os: macos-26
+    runs-on: ${{ matrix.os }}
 
     steps:
       - name: Checkout
@@ -30,31 +39,92 @@ jobs:
           cache: 'yarn'
 
       - name: Install JS dependencies
+        shell: bash
         run: yarn --immutable
 
       - name: Install Playwright browsers
+        shell: bash
         run: yarn playwright install --only-shell --with-deps chromium
 
       - uses: 1password/install-cli-action@8d006a0d0a4fd505af7f7ce589e7f768385ff5e4
       - name: Generate env file
+        shell: bash
         run: op inject -i e2e-tests/.env.tpl -o e2e-tests/.env
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
 
       - name: Build application
+        shell: bash
         run: yarn prestart
 
       - name: Run E2E tests
+        shell: bash
         continue-on-error: true
-        # We use xvfb to simulate a virtual display in order for electron to work in CI
-        run: xvfb-run yarn test:e2e
+        run: yarn test:e2e --project=${{ matrix.project }} --reporter=blob
+        env:
+          PLAYWRIGHT_BLOB_OUTPUT_DIR: playwright-blobs
+
+      - name: Upload blob report
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
+        with:
+          name: playwright-blobs-${{ matrix.project }}
+          path: playwright-blobs
+
+  merge_reports:
+    name: Merge and Publish Reports
+    needs: run_tests
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+
+      - name: Setup Node.js
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f
+        with:
+          node-version-file: '.node-version'
+          cache: 'yarn'
+
+      - name: Install JS dependencies
+        run: yarn --immutable
+
+      - name: Download playwright blobs
+        uses: actions/download-artifact@484a0b528fb4d7bd804637ccb632e47a0e638317
+        with:
+          path: playwright-blobs
+          pattern: playwright-blobs-*
+          merge-multiple: true
+
+      - name: Merge Playwright reports
+        run: yarn playwright merge-reports --config=playwright.config.ts playwright-blobs
 
       - name: Upload test report
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
         with:
           name: playwright-report
           path: playwright-report/html
 
+      - name: Upload report to Testiny
+        continue-on-error: true
+        env:
+          TESTINY_API_KEY: ${{ secrets.TESTINY_API_KEY }}
+          DESCRIPTION: 'Build URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'
+        run: |
+          # Upload test run for windows
+          node ./e2e-tests/scripts/uploadTestReport.ts \
+          --runName="Regression $(date +'%m/%d/%Y - %H:%M')" \
+          --project="windows"
+          --testPlanId="118" \
+          --description="$DESCRIPTION" \
+          --reportPath="./playwright-report/report.json"
+
+          # Upload test run for macOS
+          node ./e2e-tests/scripts/uploadTestReport.ts \
+          --runName="Regression $(date +'%m/%d/%Y - %H:%M')" \
+          --project="macOS"
+          --testPlanId="119" \
+          --description="$DESCRIPTION" \
+          --reportPath="./playwright-report/report.json"
+
       - name: Generate report description
         id: generate-description
         run: |
@@ -82,6 +152,7 @@ jobs:
             ($tests | map(select(.status == "flaky")) | map("  ⚠️ " + .title) | unique | join("\n")) as $flaky_list |
 
             # Construct the final formatted message
+            "**This test run is part of a CI refactoring, please ignore its results**\n\n" +
             "📋 **Desktop E2E Test Run Summary**\n" +
             "Build URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\n" +
             "======================================\n" +
@@ -98,23 +169,6 @@ jobs:
           echo "failed_tests=$FAILED" >> $GITHUB_OUTPUT
           echo "report_message=$MESSAGE" >> $GITHUB_OUTPUT
 
-      - name: Upload report to Testiny
-        if: ${{ github.event_name != 'pull_request' }}
-        continue-on-error: true
-        env:
-          TESTINY_API_KEY: ${{ secrets.TESTINY_API_KEY }}
-          TESTINY_TEST_PLAN_ID: 117 # 117 is the id of the "Regression" test plan within Testiny
-          DESCRIPTION: ${{ steps.generate-description.outputs.report_message }}
-        run: |
-          # Extract description from json format so it can be quoted correctly
-          RAW_DESCRIPTION=$(echo "$DESCRIPTION" | jq -r '.')
-
-          node ./e2e-tests/scripts/uploadTestReport.ts \
-          --runName="Regression $(date +'%m/%d/%Y - %H:%M')" \
-          --testPlanId="$TESTINY_TEST_PLAN_ID" \
-          --description="$RAW_DESCRIPTION" \
-          --reportPath="./playwright-report/report.json"
-
       - name: Notify on Wire about failed tests
         if: ${{ github.event_name != 'pull_request' && !cancelled() && steps.generate-description.outputs.failed_tests > 0 }}
         uses: 8398a7/action-slack@293f8dc0f9731ac35321056641cdef895f4f65f8

e2e-tests/actions/createApp.ts

@@ -52,19 +52,6 @@ export const createApp = async (options: {env?: string; lang?: string; dataDir:
     console.log(...args);
   });
 
-  // Mock safeStorage and badge APIs of electron as they don't work for the headless linux used in CI
-  app.evaluate(({safeStorage, app}) => {
-    safeStorage.encryptString = (plainText: string) => Buffer.from(plainText, 'utf-8');
-    safeStorage.decryptString = (encrypted: Buffer) => Buffer.from(encrypted).toString('utf-8');
-
-    let badgeCount = 0;
-    app.setBadgeCount = (count: number) => {
-      badgeCount = count;
-      return true;
-    };
-    app.getBadgeCount = () => badgeCount;
-  });
-
   /**
    * The webview element isn't treated as a regular webcomponent / iframe by electron but as individual window.
    * So in order to access the contents of the application we need to use the second window.

e2e-tests/fixtures.ts

@@ -30,7 +30,10 @@ import {BrigApiClient} from './backend/BrigApiClient';
 import {GalleyApiClient} from './backend/GalleyApiClient';
 import {PublicApiClient, RegisteredUser, TeamOwner} from './backend/PublicApiClient';
 
-type FixtureOptions = {appOptions: {env?: string; lang?: string}};
+export type TestOptions = {
+  os: 'windows' | 'macOS';
+  appOptions: {env?: string; lang?: string};
+};
 
 type Fixtures = {
   app: App;
@@ -43,7 +46,9 @@ type Fixtures = {
   createTeam: (...args: Parameters<typeof createTeam> extends [any, ...infer Args] ? Args : never) => Promise<Team>;
 };
 
-export const test = baseTest.extend<FixtureOptions & Fixtures>({
+export const test = baseTest.extend<TestOptions & Fixtures>({
+  // The os option is set by the project within playwright.config.ts
+  os: ['macOS', {option: true}],
   appOptions: {env: process.env.WEBAPP_URL, lang: 'en'},
 
   publicApi: async ({}, use) => {

e2e-tests/scripts/uploadTestReport.ts

@@ -47,6 +47,10 @@ const {values: args} = parseArgs({
       type: 'string',
       description: 'ID of the test plan this run should be associated with',
     },
+    project: {
+      type: 'string',
+      description: 'The playwright project to create the run for',
+    },
     description: {
       type: 'string',
       description: 'Description to add to the run, supports markdown',
@@ -62,9 +66,9 @@ const getTests = (suite: JSONReportSuite): (JSONReportTest & Pick<JSONReportSpec
 };
 
 type TestRun = {id: number};
-async function createTestRun(options?: {testPlanId?: number; description?: string}): Promise<TestRun> {
+async function createTestRun(options: {testPlanId?: number; description?: string}): Promise<TestRun> {
   const body = {
-    title: args.runName,
+    title: `${args.runName} (${args.project})`,
     project_id: TESTINY_PROJECT_ID,
     testplan_id: options?.testPlanId,
     description: options?.description,
@@ -117,6 +121,7 @@ async function deleteTestRun(runId: number) {
 }
 
 type TestinyTestCaseMapping = {
+  project: string;
   ids: {testcase_id: number; testrun_id: number};
   mapped: {assigned_to: 'ANY'; result_status: 'NOTRUN' | 'PASSED' | 'FAILED' | 'BLOCKED' | 'SKIPPED'};
 };
@@ -140,6 +145,7 @@ function transformReportToTestinyMappings(report: JSONReport, runId: number) {
 
       for (const testId of testIds) {
         acc.push({
+          project: test.projectName,
           ids: {testrun_id: runId, testcase_id: testId},
           mapped: {assigned_to: 'ANY', result_status: resultMap[test.status]},
         });
@@ -174,6 +180,9 @@ async function main() {
   if (args.runName === undefined) {
     throw new Error('Missing required arg runName');
   }
+  if (args.project === undefined) {
+    throw new Error('Missing required arg project');
+  }
 
   const reportAbsPath = path.resolve(args.reportPath);
   if (!fs.existsSync(reportAbsPath)) {
@@ -188,7 +197,9 @@ async function main() {
   console.log(`Created test run with id: ${testRun.id}`);
 
   try {
-    const testResults = transformReportToTestinyMappings(report, testRun.id);
+    const testResults = transformReportToTestinyMappings(report, testRun.id).filter(
+      mapping => mapping.project === args.project,
+    );
 
     await addTestResultsToRun(testResults);
     console.log(`Added ${testResults.length} test results to test run`);

playwright.config.ts

@@ -22,12 +22,14 @@ import dotenv from 'dotenv';
 
 import path from 'node:path';
 
+import {TestOptions} from './e2e-tests/fixtures';
+
 dotenv.config({path: path.resolve(__dirname, './e2e-tests/.env')});
 
 /**
  * See https://playwright.dev/docs/test-configuration.
  */
-export default defineConfig({
+export default defineConfig<TestOptions>({
   testDir: './e2e-tests',
   /* Run tests in files in parallel */
   fullyParallel: true,
@@ -56,11 +58,26 @@ export default defineConfig({
     permissions: ['camera', 'microphone'],
   },
 
-  /* Configure projects for major browsers */
+  /* Configure projects for multiple operating systems */
   projects: [
     {
-      name: 'chromium',
+      name: 'windows',
+      use: {
+        os: 'windows',
+        ...devices['Desktop Chrome'],
+        launchOptions: {
+          args: [
+            '--use-fake-device-for-media-stream', // Provide fake devices for audio & video device input
+            '--use-fake-ui-for-media-stream', // Bypasses the popup to grant permission and select video / audio input device by automatically selecting the default one
+            '--mute-audio', // Mute all audio output from the test browser because e.g. the ringtone of a call can be annoying during testing
+          ],
+        },
+      },
+    },
+    {
+      name: 'macOS',
       use: {
+        os: 'macOS',
         ...devices['Desktop Chrome'],
         launchOptions: {
           args: [