perf(ci): Speed up Next.js E2E tests

.github/workflows/build.yml

@@ -139,6 +139,7 @@ jobs:
 
       - name: Determine which test applications should be run
         id: matrix
+        # Workflow changes run the full matrix, which also provides comparable CI timing measurements.
         run:
           pnpm --silent run ci:build-matrix --shards=2 --base=${{ (github.event_name == 'pull_request' &&
           needs.job_get_metadata.outputs.changed_ci != 'true' && github.event.pull_request.base.sha) || '' }} >>

dev-packages/e2e-tests/README.md

@@ -292,6 +292,10 @@ For example, if something is changed in the browser package, only E2E test apps
 You can add additional information about the test (e.g. canary versions, optional in CI) by adding `sentryTest` in the `package.json`
 of a test application.
 
+`sentryTest.baseVariant` overrides the default matrix entry using the same fields as an entry in `variants`. For example,
+set its `assert-command` to run production tests and add a development variant to run both environments in parallel.
+The local `test:assert` script can still run both; select the base variant locally with `--variant <label>`.
+
 The required E2E jobs run on 2 vCPUs (`namespace-profile-sentry-javascript-small`). If an app's job takes longer than
 about 90 seconds, set `"sentryTest": { "runner": "namespace-profile-sentry-javascript" }` to run it and its variants on
 4 vCPUs.

dev-packages/e2e-tests/lib/getTestMatrix.mjs

@@ -118,7 +118,7 @@ function addIncludesForTestApp(testApp, includes, { variant }) {
   // The test application itself is either required or optional, never canary, and only runs in the matrix of its mode.
   const appVariant = isOptional ? 'optional' : 'required';
   if (appVariant === variant) {
-    includes.push(baseInclude);
+    includes.push({ ...baseInclude, ...packageJson.sentryTest?.baseVariant });
   }
 
   variants.forEach(variant => {

dev-packages/e2e-tests/run.mjs

@@ -19,7 +19,7 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
  * @typedef {{ 'build-command': string; 'assert-command'?: string; label?: string; skip?: boolean; runtime?: string }} SentryTestVariant
  */
 
-/** @typedef {{ sentryTest?: { variants?: SentryTestVariant[]; optionalVariants?: SentryTestVariant[]; canaryVariants?: SentryTestVariant[] } }} PackageJson */
+/** @typedef {{ sentryTest?: { baseVariant?: SentryTestVariant; variants?: SentryTestVariant[]; optionalVariants?: SentryTestVariant[]; canaryVariants?: SentryTestVariant[] } }} PackageJson */
 
 const DEFAULT_DSN = 'https://username@domain/123';
 const DEFAULT_SENTRY_ORG_SLUG = 'sentry-javascript-sdks';
@@ -98,6 +98,7 @@ async function getVariantBuildCommand(packageJsonPath, variantLabel, testAppPath
     const packageJson = JSON.parse(packageJsonContent);
 
     const allVariants = [
+      ...(packageJson.sentryTest?.baseVariant ? [packageJson.sentryTest.baseVariant] : []),
       ...(packageJson.sentryTest?.variants || []),
       ...(packageJson.sentryTest?.optionalVariants || []),
       ...(packageJson.sentryTest?.canaryVariants || []),

dev-packages/e2e-tests/test-applications/nextjs-15/package.json

@@ -42,6 +42,18 @@
         "assert-command": "pnpm test:prod-turbo && pnpm test:dev-turbo",
         "label": "nextjs-15 (turbo)"
       }
+    ],
+    "baseVariant": {
+      "build-command": "pnpm test:build",
+      "label": "nextjs-15 (production)",
+      "assert-command": "pnpm test:prod"
+    },
+    "variants": [
+      {
+        "build-command": "pnpm test:build",
+        "label": "nextjs-15 (development)",
+        "assert-command": "pnpm test:dev"
+      }
     ]
   }
 }

dev-packages/e2e-tests/test-applications/nextjs-16-static/package.json

@@ -41,6 +41,18 @@
     "typescript": "^5"
   },
   "sentryTest": {
-    "runner": "namespace-profile-sentry-javascript"
+    "runner": "namespace-profile-sentry-javascript",
+    "baseVariant": {
+      "build-command": "pnpm test:build",
+      "label": "nextjs-16-static (production)",
+      "assert-command": "pnpm test:prod"
+    },
+    "variants": [
+      {
+        "build-command": "pnpm test:build",
+        "label": "nextjs-16-static (development)",
+        "assert-command": "pnpm test:dev"
+      }
+    ]
   }
 }

dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts

@@ -6,6 +6,7 @@ Sentry.init({
   dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN,
   tunnel: `http://localhost:3031/`, // proxy server
   tracesSampleRate: 1.0,
+  _flushInterval: 100,
   integrations: [
     Sentry.thirdPartyErrorFilterIntegration({
       filterKeys: ['nextjs-16-e2e'],

dev-packages/e2e-tests/test-applications/nextjs-16/package.json

@@ -67,24 +67,45 @@
   "sentryTest": {
     "runner": "namespace-profile-sentry-javascript",
     "variants": [
+      {
+        "build-command": "pnpm test:build",
+        "label": "nextjs-16 (turbopack, development)",
+        "assert-command": "pnpm test:dev"
+      },
       {
         "build-command": "NEXT_PUBLIC_EXCLUDE_APPLICATION_KEY_DEPENDENCIES=1 pnpm test:build",
         "label": "nextjs-16 (exclude application key dependencies)",
         "assert-command": "NEXT_PUBLIC_EXCLUDE_APPLICATION_KEY_DEPENDENCIES=1 pnpm test:assert"
       },
       {
         "build-command": "pnpm test:build-webpack",
-        "label": "nextjs-16 (webpack)",
-        "assert-command": "pnpm test:assert-webpack"
+        "label": "nextjs-16 (webpack, production)",
+        "assert-command": "pnpm test:prod"
+      },
+      {
+        "build-command": "pnpm test:build-webpack",
+        "label": "nextjs-16 (webpack, development)",
+        "assert-command": "pnpm test:dev-webpack"
       },
       {
         "build-command": "pnpm test:build-latest-webpack",
-        "label": "nextjs-16 (latest, webpack)",
-        "assert-command": "pnpm test:assert-webpack"
+        "label": "nextjs-16 (latest, webpack, production)",
+        "assert-command": "pnpm test:prod"
+      },
+      {
+        "build-command": "pnpm test:build-latest-webpack",
+        "label": "nextjs-16 (latest, webpack, development)",
+        "assert-command": "pnpm test:dev-webpack"
       },
       {
         "build-command": "pnpm test:build-latest",
-        "label": "nextjs-16 (latest, turbopack)"
+        "label": "nextjs-16 (latest, turbopack, production)",
+        "assert-command": "pnpm test:prod"
+      },
+      {
+        "build-command": "pnpm test:build-latest",
+        "label": "nextjs-16 (latest, turbopack, development)",
+        "assert-command": "pnpm test:dev"
       },
       {
         "runtime": "bun",
@@ -134,6 +155,11 @@
         "label": "nextjs-16 (canary, webpack)",
         "assert-command": "pnpm test:assert-webpack"
       }
-    ]
+    ],
+    "baseVariant": {
+      "build-command": "pnpm test:build",
+      "label": "nextjs-16 (turbopack, production)",
+      "assert-command": "pnpm test:prod"
+    }
   }
 }

dev-packages/e2e-tests/test-applications/nextjs-16/sentry.server.config.ts

@@ -8,6 +8,7 @@ Sentry.init({
   dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN,
   tunnel: `http://localhost:3031/`, // proxy server
   tracesSampleRate: 1.0,
+  _flushInterval: 100,
   tracePropagationTargets: ['http://localhost:3030/propagation/test-outgoing-fetch/check'],
   // debug: true,
   integrations: [

dev-packages/e2e-tests/test-applications/nextjs-app-dir/package.json

@@ -9,7 +9,8 @@
     "test:dev": "TEST_ENV=development playwright test",
     "test:build": "pnpm install && pnpm build",
     "test:test-build": "pnpm ts-node --script-mode assert-build.ts",
-    "test:build-15": "pnpm install && pnpm add next@15 && pnpm add react@latest && pnpm add react-dom@latest && pnpm build",
+    "test:install-15": "pnpm install && pnpm add next@15 && pnpm add react@latest && pnpm add react-dom@latest",
+    "test:build-15": "pnpm test:install-15 && pnpm build",
     "test:assert": "pnpm test:test-build && pnpm test:prod && pnpm test:dev"
   },
   "dependencies": {
@@ -31,10 +32,26 @@
   "sentryTest": {
     "runner": "namespace-profile-sentry-javascript",
     "variants": [
+      {
+        "build-command": "pnpm install",
+        "label": "nextjs-app-dir (development)",
+        "assert-command": "pnpm test:dev"
+      },
       {
         "build-command": "pnpm test:build-15",
-        "label": "nextjs-app-dir (next@15)"
+        "label": "nextjs-app-dir (next@15, production)",
+        "assert-command": "pnpm test:test-build && pnpm test:prod"
+      },
+      {
+        "build-command": "pnpm test:install-15",
+        "label": "nextjs-app-dir (next@15, development)",
+        "assert-command": "pnpm test:dev"
       }
-    ]
+    ],
+    "baseVariant": {
+      "build-command": "pnpm test:build",
+      "label": "nextjs-app-dir (production)",
+      "assert-command": "pnpm test:test-build && pnpm test:prod"
+    }
   }
 }