diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 238a79cd121..a6e1aedf0cf 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -15,9 +15,18 @@ concurrency: jobs: run_tests: - name: Run E2E Tests - runs-on: ubuntu-24.04 + name: Run E2E Tests (${{ matrix.project }}) if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-e2e') }} + strategy: + fail-fast: false + matrix: + project: [windows, macOS] + include: + - project: windows + os: windows-2025 + - project: macOS + os: macos-26 + runs-on: ${{ matrix.os }} steps: - name: Checkout @@ -30,24 +39,63 @@ jobs: cache: 'yarn' - name: Install JS dependencies + shell: bash run: yarn --immutable - name: Install Playwright browsers + shell: bash run: yarn playwright install --only-shell --with-deps chromium - uses: 1password/install-cli-action@8d006a0d0a4fd505af7f7ce589e7f768385ff5e4 - name: Generate env file + shell: bash run: op inject -i e2e-tests/.env.tpl -o e2e-tests/.env env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - name: Build application + shell: bash run: yarn prestart - name: Run E2E tests + shell: bash continue-on-error: true - # We use xvfb to simulate a virtual display in order for electron to work in CI - run: xvfb-run yarn test:e2e + run: yarn test:e2e --project=${{ matrix.project }} --reporter=blob + env: + PLAYWRIGHT_BLOB_OUTPUT_DIR: playwright-blobs + + - name: Upload blob report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: playwright-blobs-${{ matrix.project }} + path: playwright-blobs + + merge_reports: + name: Merge and Publish Reports + needs: run_tests + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version-file: '.node-version' + cache: 'yarn' + + - name: Install JS dependencies + run: yarn --immutable + + - name: Download playwright blobs + uses: actions/download-artifact@484a0b528fb4d7bd804637ccb632e47a0e638317 + with: + path: playwright-blobs + pattern: playwright-blobs-* + merge-multiple: true + + - name: Merge Playwright reports + run: yarn playwright merge-reports --config=playwright.config.ts playwright-blobs - name: Upload test report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a @@ -55,6 +103,28 @@ jobs: name: playwright-report path: playwright-report/html + - name: Upload report to Testiny + continue-on-error: true + env: + TESTINY_API_KEY: ${{ secrets.TESTINY_API_KEY }} + DESCRIPTION: 'Build URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}' + run: | + # Upload test run for windows + node ./e2e-tests/scripts/uploadTestReport.ts \ + --runName="Regression $(date +'%m/%d/%Y - %H:%M')" \ + --project="windows" + --testPlanId="118" \ + --description="$DESCRIPTION" \ + --reportPath="./playwright-report/report.json" + + # Upload test run for macOS + node ./e2e-tests/scripts/uploadTestReport.ts \ + --runName="Regression $(date +'%m/%d/%Y - %H:%M')" \ + --project="macOS" + --testPlanId="119" \ + --description="$DESCRIPTION" \ + --reportPath="./playwright-report/report.json" + - name: Generate report description id: generate-description run: | @@ -82,6 +152,7 @@ jobs: ($tests | map(select(.status == "flaky")) | map(" ⚠️ " + .title) | unique | join("\n")) as $flaky_list | # Construct the final formatted message + "**This test run is part of a CI refactoring, please ignore its results**\n\n" + "📋 **Desktop E2E Test Run Summary**\n" + "Build URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\n" + "======================================\n" + @@ -98,23 +169,6 @@ jobs: echo "failed_tests=$FAILED" >> $GITHUB_OUTPUT echo "report_message=$MESSAGE" >> $GITHUB_OUTPUT - - name: Upload report to Testiny - if: ${{ github.event_name != 'pull_request' }} - continue-on-error: true - env: - TESTINY_API_KEY: ${{ secrets.TESTINY_API_KEY }} - TESTINY_TEST_PLAN_ID: 117 # 117 is the id of the "Regression" test plan within Testiny - DESCRIPTION: ${{ steps.generate-description.outputs.report_message }} - run: | - # Extract description from json format so it can be quoted correctly - RAW_DESCRIPTION=$(echo "$DESCRIPTION" | jq -r '.') - - node ./e2e-tests/scripts/uploadTestReport.ts \ - --runName="Regression $(date +'%m/%d/%Y - %H:%M')" \ - --testPlanId="$TESTINY_TEST_PLAN_ID" \ - --description="$RAW_DESCRIPTION" \ - --reportPath="./playwright-report/report.json" - - name: Notify on Wire about failed tests if: ${{ github.event_name != 'pull_request' && !cancelled() && steps.generate-description.outputs.failed_tests > 0 }} uses: 8398a7/action-slack@293f8dc0f9731ac35321056641cdef895f4f65f8 diff --git a/e2e-tests/actions/createApp.ts b/e2e-tests/actions/createApp.ts index 9d900859514..0aaa4570aab 100644 --- a/e2e-tests/actions/createApp.ts +++ b/e2e-tests/actions/createApp.ts @@ -52,19 +52,6 @@ export const createApp = async (options: {env?: string; lang?: string; dataDir: console.log(...args); }); - // Mock safeStorage and badge APIs of electron as they don't work for the headless linux used in CI - app.evaluate(({safeStorage, app}) => { - safeStorage.encryptString = (plainText: string) => Buffer.from(plainText, 'utf-8'); - safeStorage.decryptString = (encrypted: Buffer) => Buffer.from(encrypted).toString('utf-8'); - - let badgeCount = 0; - app.setBadgeCount = (count: number) => { - badgeCount = count; - return true; - }; - app.getBadgeCount = () => badgeCount; - }); - /** * The webview element isn't treated as a regular webcomponent / iframe by electron but as individual window. * So in order to access the contents of the application we need to use the second window. diff --git a/e2e-tests/fixtures.ts b/e2e-tests/fixtures.ts index f0a105c8bdc..27fffc34e5f 100644 --- a/e2e-tests/fixtures.ts +++ b/e2e-tests/fixtures.ts @@ -30,7 +30,10 @@ import {BrigApiClient} from './backend/BrigApiClient'; import {GalleyApiClient} from './backend/GalleyApiClient'; import {PublicApiClient, RegisteredUser, TeamOwner} from './backend/PublicApiClient'; -type FixtureOptions = {appOptions: {env?: string; lang?: string}}; +export type TestOptions = { + os: 'windows' | 'macOS'; + appOptions: {env?: string; lang?: string}; +}; type Fixtures = { app: App; @@ -43,7 +46,9 @@ type Fixtures = { createTeam: (...args: Parameters extends [any, ...infer Args] ? Args : never) => Promise; }; -export const test = baseTest.extend({ +export const test = baseTest.extend({ + // The os option is set by the project within playwright.config.ts + os: ['macOS', {option: true}], appOptions: {env: process.env.WEBAPP_URL, lang: 'en'}, publicApi: async ({}, use) => { diff --git a/e2e-tests/scripts/uploadTestReport.ts b/e2e-tests/scripts/uploadTestReport.ts index 51c7ac8e5f3..12bcd5b5e88 100644 --- a/e2e-tests/scripts/uploadTestReport.ts +++ b/e2e-tests/scripts/uploadTestReport.ts @@ -47,6 +47,10 @@ const {values: args} = parseArgs({ type: 'string', description: 'ID of the test plan this run should be associated with', }, + project: { + type: 'string', + description: 'The playwright project to create the run for', + }, description: { type: 'string', description: 'Description to add to the run, supports markdown', @@ -62,9 +66,9 @@ const getTests = (suite: JSONReportSuite): (JSONReportTest & Pick { +async function createTestRun(options: {testPlanId?: number; description?: string}): Promise { const body = { - title: args.runName, + title: `${args.runName} (${args.project})`, project_id: TESTINY_PROJECT_ID, testplan_id: options?.testPlanId, description: options?.description, @@ -117,6 +121,7 @@ async function deleteTestRun(runId: number) { } type TestinyTestCaseMapping = { + project: string; ids: {testcase_id: number; testrun_id: number}; mapped: {assigned_to: 'ANY'; result_status: 'NOTRUN' | 'PASSED' | 'FAILED' | 'BLOCKED' | 'SKIPPED'}; }; @@ -140,6 +145,7 @@ function transformReportToTestinyMappings(report: JSONReport, runId: number) { for (const testId of testIds) { acc.push({ + project: test.projectName, ids: {testrun_id: runId, testcase_id: testId}, mapped: {assigned_to: 'ANY', result_status: resultMap[test.status]}, }); @@ -174,6 +180,9 @@ async function main() { if (args.runName === undefined) { throw new Error('Missing required arg runName'); } + if (args.project === undefined) { + throw new Error('Missing required arg project'); + } const reportAbsPath = path.resolve(args.reportPath); if (!fs.existsSync(reportAbsPath)) { @@ -188,7 +197,9 @@ async function main() { console.log(`Created test run with id: ${testRun.id}`); try { - const testResults = transformReportToTestinyMappings(report, testRun.id); + const testResults = transformReportToTestinyMappings(report, testRun.id).filter( + mapping => mapping.project === args.project, + ); await addTestResultsToRun(testResults); console.log(`Added ${testResults.length} test results to test run`); diff --git a/playwright.config.ts b/playwright.config.ts index 8cc70c7fbd8..320fe0a196a 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -22,12 +22,14 @@ import dotenv from 'dotenv'; import path from 'node:path'; +import {TestOptions} from './e2e-tests/fixtures'; + dotenv.config({path: path.resolve(__dirname, './e2e-tests/.env')}); /** * See https://playwright.dev/docs/test-configuration. */ -export default defineConfig({ +export default defineConfig({ testDir: './e2e-tests', /* Run tests in files in parallel */ fullyParallel: true, @@ -56,11 +58,26 @@ export default defineConfig({ permissions: ['camera', 'microphone'], }, - /* Configure projects for major browsers */ + /* Configure projects for multiple operating systems */ projects: [ { - name: 'chromium', + name: 'windows', + use: { + os: 'windows', + ...devices['Desktop Chrome'], + launchOptions: { + args: [ + '--use-fake-device-for-media-stream', // Provide fake devices for audio & video device input + '--use-fake-ui-for-media-stream', // Bypasses the popup to grant permission and select video / audio input device by automatically selecting the default one + '--mute-audio', // Mute all audio output from the test browser because e.g. the ringtone of a call can be annoying during testing + ], + }, + }, + }, + { + name: 'macOS', use: { + os: 'macOS', ...devices['Desktop Chrome'], launchOptions: { args: [