ci: measure and shard Playwright E2E suites (#434)

cmd/server/fork_guard_workflow_test.go

@@ -187,7 +187,7 @@ func TestImageCheckLeastPrivilegeAndTimeout(t *testing.T) {
 
 func TestForkGuardLeavesTestJobsRunning(t *testing.T) {
 	deploy := readWorkflow(t, deployWorkflowRel)
-	for _, name := range []string{"go-test", "e2e-test", "image-check"} {
+	for _, name := range []string{"go-test", "e2e-test-1", "e2e-test-2", "e2e-test-3", "image-check"} {
 		job := workflowJob(t, deploy, name)
 		if cond := jobIf(job); cond != "" {
 			t.Errorf("%s must not gain a job-level condition; got %q", name, cond)
@@ -196,6 +196,13 @@ func TestForkGuardLeavesTestJobsRunning(t *testing.T) {
 			t.Errorf("%s must not be repository-guarded: tests and CI artifact uploads run on forks too", name)
 		}
 	}
+	report := workflowJob(t, deploy, "e2e-report")
+	if cond := strings.TrimSpace(jobIf(report)); cond != "${{ !cancelled() }}" {
+		t.Errorf("e2e-report must run after failed or skipped shards; got if %q", cond)
+	}
+	if strings.Contains(report, "github.repository") {
+		t.Error("e2e-report must not be repository-guarded")
+	}
 }
 
 // The required check is configured by name, so renaming the gate job would
@@ -231,3 +238,72 @@ func TestForkGuardCount(t *testing.T) {
 		}
 	}
 }
+
+// A single shared steps anchor prevents fixtures or setup from silently
+// diverging between shards. The alias jobs may vary only their local port
+// and shard identity, never their preparation or suite runner.
+func TestE2EShardsShareFixtureSteps(t *testing.T) {
+	deploy := readWorkflow(t, deployWorkflowRel)
+	first := workflowJob(t, deploy, "e2e-test-1")
+	if !strings.Contains(first, "    steps: &e2e-shard-steps\n") {
+		t.Fatal("shard 1 must declare the shared E2E steps anchor")
+	}
+	for _, c := range []struct{ job, shard, port string }{
+		{"e2e-test-1", "1", "13581"},
+		{"e2e-test-2", "2", "13582"},
+		{"e2e-test-3", "3", "13583"},
+	} {
+		job := workflowJob(t, deploy, c.job)
+		for _, want := range []string{
+			"      E2E_SHARD: '" + c.shard + "'\n",
+			"      E2E_PORT: '" + c.port + "'\n",
+			"      BASE_URL: http://localhost:" + c.port + "\n",
+			"    timeout-minutes: 150\n",
+		} {
+			if !strings.Contains(job, want) {
+				t.Errorf("%s must contain %q", c.job, want)
+			}
+		}
+		if c.shard != "1" && !strings.HasSuffix(strings.TrimSpace(job), "steps: *e2e-shard-steps") {
+			t.Errorf("%s must use the identical shared steps, without overrides", c.job)
+		}
+	}
+	previous := -1
+	for _, marker := range []string{
+		"bash tools/freshen-fixture.sh test-fixtures/e2e-fixture.db",
+		"sqlite3 test-fixtures/e2e-fixture.db <<'SQL'",
+		"./corescope-migrate -db test-fixtures/e2e-fixture.db",
+		"sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-2073-route-adverts.sql",
+		"sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-199-inactive-observer.sql",
+		"sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-245-advert-intervals.sql",
+		"./corescope-server -port \"$E2E_PORT\" -db test-fixtures/e2e-fixture.db -public public-instrumented &",
+		"node scripts/run-e2e-shard.js\n",
+	} {
+		index := strings.Index(first, marker)
+		if index <= previous {
+			t.Fatalf("fixture step %q is missing or out of order", marker)
+		}
+		previous = index
+	}
+}
+
+func TestE2ECoverageRunsOnCoreShardOnly(t *testing.T) {
+	deploy := readWorkflow(t, deployWorkflowRel)
+	first := workflowJob(t, deploy, "e2e-test-1")
+	if !strings.Contains(first, "node scripts/run-e2e-shard.js --core-shard") {
+		t.Error("coverage owner must be identified from the checked-in manifest")
+	}
+	const condition = "if: success() && github.event_name == 'push' && steps.shard-meta.outputs.core_shard == env.E2E_SHARD"
+	if !strings.Contains(first, condition) {
+		t.Error("coverage collection must stay push-only and run once on the core suite's shard")
+	}
+	if strings.Contains(first, "Generate frontend coverage badges") {
+		t.Error("per-shard badges must not replace the aggregate badge summary")
+	}
+	report := workflowJob(t, deploy, "e2e-report")
+	for _, need := range []string{"node scripts/report-e2e-shards.js", "Generate frontend coverage badges", "name: e2e-badges"} {
+		if !strings.Contains(report, need) {
+			t.Errorf("e2e-report must preserve %q", need)
+		}
+	}
+}

scripts/ci/e2e-manifest.js

@@ -0,0 +1,113 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+
+const SHARDS = [1, 2, 3];
+const CORE_SUITE = 'test-e2e-playwright.js';
+const REPEAT_SUITE = 'test-slideover-1056-e2e.js';
+const NODE_LIVENESS = 'test-node-liveness-e2e.js';
+const LEGACY_EXCLUSIONS = new Set([
+  'test-rx-coverage-mobile-nav-e2e.js',
+]);
+const ENV_KEYS = new Set([
+  'BASE_URL', 'CHROMIUM_REQUIRE', 'AXE_SCREENSHOT_DIR',
+  'CORESCOPE_SERVER_BIN', 'CORESCOPE_INGESTOR_BIN', 'FIXTURE_DB',
+  'CORESCOPE_MIGRATE_BIN', 'CORESCOPE_LEAFLET_DIR',
+]);
+
+function suiteFile(file) {
+  return typeof file === 'string' && /^test-[A-Za-z0-9_-]+\.js$/.test(file);
+}
+
+function discoverSuites(root) {
+  const units = new Set([...fs.readFileSync(path.join(root, 'test-all.sh'), 'utf8')
+    .matchAll(/^run (test-[A-Za-z0-9_-]+\.js)\s*$/gm)].map(match => match[1]));
+  return fs.readdirSync(root).filter(file => /^test-.*\.js$/.test(file)).filter(file => {
+    const full = path.join(root, file);
+    assert.ok(fs.lstatSync(full).isFile(), `suite must be a regular file: ${file}`);
+    // #1697 has an E2E filename but is a VM unit test in the unit registry.
+    if (file === 'test-issue-1697-mqtt-mobile-e2e.js' && units.has(file)) return false;
+    if (/e2e|playwright/.test(file)) return true;
+    return /(?:\b(?:require|import)\s*\(\s*|\bfrom\s*)['"](?:playwright(?:\/[^'"]+)?|@playwright\/test(?:\/[^'"]+)?)['"]/.test(fs.readFileSync(full, 'utf8'));
+  }).sort();
+}
+
+function validateManifest(manifest, root) {
+  assert.ok(manifest && typeof manifest === 'object' && !Array.isArray(manifest), 'manifest must be an object');
+  assert.equal(manifest.version, 1, 'unsupported manifest version');
+  assert.ok(Array.isArray(manifest.suites), 'manifest suites must be an array');
+  assert.ok(Array.isArray(manifest.exclusions), 'manifest exclusions must be an array');
+  const assigned = new Map();
+  for (const suite of manifest.suites) {
+    assert.ok(suite && typeof suite === 'object', 'invalid suite entry');
+    assert.ok(suiteFile(suite.file), `invalid suite filename: ${suite.file}`);
+    assert.notEqual(suite.file, 'test-issue-1697-mqtt-mobile-e2e.js', '#1697 must remain in the unit registry');
+    assert.ok(SHARDS.includes(suite.shard), `invalid shard for ${suite.file}`);
+    assert.ok(!assigned.has(suite.file), `duplicate suite assignment: ${suite.file}`);
+    assert.ok(fs.lstatSync(path.join(root, suite.file)).isFile(), `missing suite file: ${suite.file}`);
+    assert.ok(suite.env && typeof suite.env === 'object' && !Array.isArray(suite.env), `env must be an object: ${suite.file}`);
+    for (const [key, value] of Object.entries(suite.env)) {
+      assert.ok(ENV_KEYS.has(key), `unsupported environment key: ${key}`);
+      assert.equal(typeof value, 'string', `non-string environment value: ${key}`);
+      assert.ok(!/[\x00\r\n]/.test(value), `invalid environment value: ${key}`);
+      assert.ok(!value.includes('$') || value === '$BASE_URL' || value === '$LEAFLET_DIR', `unsupported environment expansion: ${key}`);
+      if (key === 'BASE_URL') assert.equal(value, '$BASE_URL', 'BASE_URL must use the pinned local shard target');
+    }
+    assert.ok(['e2e', 'node-liveness'].includes(suite.output), `invalid output for ${suite.file}`);
+    assert.equal(suite.output === 'node-liveness', suite.file === NODE_LIVENESS, 'only node liveness uses its separate output');
+    if (suite.setup !== undefined) {
+      assert.equal(suite.file, 'test-issue-315-estimated-positions-e2e.js', 'setup is only allowed for #315');
+      assert.equal(suite.setup, 'leaflet-315', 'unknown suite setup');
+    }
+    if (suite.repeatGate !== undefined) {
+      assert.equal(suite.file, REPEAT_SUITE, 'only slide-over has the repeat gate');
+      assert.equal(suite.repeatGate, 3, 'the permanent slide-over gate must run three times');
+    }
+    assigned.set(suite.file, suite);
+  }
+  for (const shard of SHARDS) {
+    assert.ok(manifest.suites.some(suite => suite.shard === shard), `empty shard ${shard}`);
+  }
+  assert.ok(assigned.has(CORE_SUITE), 'missing core Playwright suite');
+  assert.ok(assigned.has(NODE_LIVENESS), 'missing node-liveness suite');
+  assert.equal(assigned.get(REPEAT_SUITE)?.repeatGate, 3, 'missing permanent slide-over repeat gate');
+  const excluded = new Set();
+  for (const entry of manifest.exclusions) {
+    assert.ok(entry && LEGACY_EXCLUSIONS.has(entry.file), `unapproved suite exclusion: ${entry?.file}`);
+    assert.ok(typeof entry.reason === 'string' && entry.reason.trim().length > 20, `missing exclusion rationale: ${entry.file}`);
+    assert.ok(!excluded.has(entry.file), `duplicate exclusion: ${entry.file}`);
+    assert.ok(!assigned.has(entry.file), `assigned suite also excluded: ${entry.file}`);
+    assert.ok(fs.lstatSync(path.join(root, entry.file)).isFile(), `missing excluded suite: ${entry.file}`);
+    excluded.add(entry.file);
+  }
+  for (const file of discoverSuites(root)) {
+    assert.ok(assigned.has(file) || excluded.has(file), `unassigned E2E suite: ${file}`);
+  }
+  return manifest;
+}
+
+function readManifest(file, root) {
+  const raw = fs.readFileSync(file, 'utf8');
+  return {
+    manifest: validateManifest(JSON.parse(raw), root),
+    manifestSha256: createHash('sha256').update(raw).digest('hex'),
+  };
+}
+
+function expectedRuns(manifest, shard) {
+  const suites = manifest.suites.filter(suite => suite.shard === shard);
+  const ordinary = suites.map(suite => ({ file: suite.file, kind: 'suite', iteration: 1 }));
+  const repeated = suites.flatMap(suite => Array.from({ length: suite.repeatGate || 0 }, (_, index) => ({
+    file: suite.file, kind: 'repeat-gate', iteration: index + 1,
+  })));
+  return [...ordinary, ...repeated];
+}
+
+function runContext(env) {
+  return { runId: env.GITHUB_RUN_ID || null, attempt: env.GITHUB_RUN_ATTEMPT || null, headSha: env.GITHUB_SHA || null };
+}
+
+module.exports = { SHARDS, CORE_SUITE, REPEAT_SUITE, NODE_LIVENESS, discoverSuites, validateManifest, readManifest, expectedRuns, runContext };

scripts/ci/prepare-e2e-leaflet.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+: "${RUNNER_TEMP:?RUNNER_TEMP is required}"
+leaflet_dir="$RUNNER_TEMP/corescope-315-leaflet"
+mkdir -p "$leaflet_dir/images"
+curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/leaflet.js -o "$leaflet_dir/leaflet.js"
+curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/leaflet.css -o "$leaflet_dir/leaflet.css"
+curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon.png -o "$leaflet_dir/images/marker-icon.png"
+curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon-2x.png -o "$leaflet_dir/images/marker-icon-2x.png"
+curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-shadow.png -o "$leaflet_dir/images/marker-shadow.png"
+echo "db49d009c841f5ca34a888c96511ae936fd9f5533e90d8b2c4d57596f4e5641a  $leaflet_dir/leaflet.js" | sha256sum --check
+echo "a7837102824184820dfa198d1ebcd109ff6d0ff9a2672a074b9a1b4d147d04c6  $leaflet_dir/leaflet.css" | sha256sum --check
+echo "574c3a5cca85f4114085b6841596d62f00d7c892c7b03f28cbfa301deb1dc437  $leaflet_dir/images/marker-icon.png" | sha256sum --check
+echo "00179c4c1ee830d3a108412ae0d294f55776cfeb085c60129a39aa6fc4ae2528  $leaflet_dir/images/marker-icon-2x.png" | sha256sum --check
+echo "264f5c640339f042dd729062cfc04c17f8ea0f29882b538e3848ed8f10edb4da  $leaflet_dir/images/marker-shadow.png" | sha256sum --check