diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 83550d6b7..b8e9d3a40 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -21,8 +21,8 @@ env: STAGING_SERVICE: staging-go STAGING_CONTAINER: corescope-staging-go -# Pipeline: independent Go tests, E2E and image validation run in parallel. -# The stable build-and-publish check fails unless all three pass; only then may +# Pipeline: Go tests, three E2E shards and image validation run in parallel. +# The stable build-and-publish check requires every shard and its report; only then may # an upstream push publish to GHCR and deploy. Fork runs never publish/deploy. # # Fork guard: everything with an external side effect (GHCR login/push, release @@ -231,14 +231,18 @@ jobs: # ─────────────────────────────────────────────────────────────── # 2. Playwright E2E Tests (against Go server with fixture DB) # ─────────────────────────────────────────────────────────────── - e2e-test: - name: "🎭 Playwright E2E Tests" + e2e-test-1: + name: "🎭 Playwright E2E Tests (1/3)" runs-on: ubuntu-latest timeout-minutes: 150 defaults: run: shell: bash - steps: + env: + E2E_SHARD: '1' + E2E_PORT: '13581' + BASE_URL: http://localhost:13581 + steps: &e2e-shard-steps - name: Checkout code uses: actions/checkout@v5 with: @@ -249,6 +253,13 @@ jobs: with: node-version: '22' + - name: Validate E2E shard assignments + run: node scripts/run-e2e-shard.js --validate + + - name: Identify the core coverage shard + id: shard-meta + run: echo "core_shard=$(node scripts/run-e2e-shard.js --core-shard)" >> "$GITHUB_OUTPUT" + - name: Clean Go module cache run: rm -rf ~/go/pkg/mod 2>/dev/null || true @@ -385,12 +396,12 @@ jobs: - name: Start Go server with fixture DB run: | - fuser -k 13581/tcp 2>/dev/null || true + fuser -k "$E2E_PORT"/tcp 2>/dev/null || true sleep 1 - ./corescope-server -port 13581 -db test-fixtures/e2e-fixture.db -public public-instrumented & + ./corescope-server -port "$E2E_PORT" -db test-fixtures/e2e-fixture.db -public public-instrumented & echo $! > .server.pid for i in $(seq 1 30); do - if curl -sf http://localhost:13581/api/healthz > /dev/null 2>&1; then + if curl -sf "$BASE_URL/api/healthz" > /dev/null 2>&1; then echo "Server ready after ${i}s" break fi @@ -401,225 +412,108 @@ jobs: sleep 1 done - - name: Run Playwright E2E tests (fail-fast) - run: | - node test-node-liveness-e2e.js 2>&1 | tee node-liveness-e2e-output.txt - BASE_URL=http://localhost:13581 node test-e2e-playwright.js 2>&1 | tee e2e-output.txt - # M5+M6 of #1668 — axe-core CI gate. - # M5: color-contrast on desktop dark+light. - # M6: expanded ruleset (image-alt, label, aria-required-attr, - # aria-valid-attr, aria-valid-attr-value, landmark-one-main, - # region, button-name, link-name, document-title, html-has-lang, - # duplicate-id) AND adds 375x812 mobile viewport (with - # color-contrast on mobile too). - # Allowlist: tests/a11y-allowlist.yaml (0 entries — hard pass policy). - # Per-viewport summary printed at the end; any net>0 fails the build. - BASE_URL=http://localhost:13581 AXE_SCREENSHOT_DIR=/tmp/axe-1668 \ - node test-a11y-axe-1668.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-issue-1087-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-issue-1111-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channels-client-state-152-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-154-155-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-map-modal-fluid-e2e.js 2>&1 | tee -a e2e-output.txt - # #332 — no map may request a tile from basemaps.cartocdn.com. - # No config.json here, so this exercises the built-in non-CARTO - # fallback (tile.openstreetmap.org) a default install resolves to. - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-332-no-carto-tiles-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-map-nodes-pagination-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-123-map-lifecycle-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-observer-iata-1188-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-xss-333-pages-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-xss-392-perf-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-xss-368-home-links-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-xss-429-home-footer-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1639-observers-sort-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-405-observers-listener-leak-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1758-ng-filter-rerenders-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-fluid-1055-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1102-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1311-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1391-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1413-nav-overlap-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1400-nav-vertical-clip.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-more-floor-1139-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-bottom-nav-1061-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gestures-1062-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gestures-1185-scroll-discriminator-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gesture-hints-1065-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-touch-gestures-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-fluid-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-table-fluid-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-charts-fluid-1058-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-slideover-1056-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1692-packets-init-parallel-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-147-packets-url-obs-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-165-grouped-hop-warn-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-packet-detail-sender-hash-size-obs-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-353-path-hash-warn-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-96-hide-control-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-242-type-exclusions-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-issue-406-packets-live-channel-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-180-packets-url-modal-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-slideover-1168-munger-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-logo-pulse-1173-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1122-packets-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1122-details-row-clamp-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-258-column-widths-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1128-packets-layout-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-189-group-caret-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-254-affinity-toggle-mobile-aria-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-259-mobile-expanded-group-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1128-multi-viewport-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1136-live-region-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-live-multibyte-only-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-125-live-toggles-early-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-220-live-filter-readiness-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1150-404-state-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-199-inactive-observer-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1146-path-link-contrast-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-issue-1705-subpath-contrast-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1147-section-order-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-2073-recent-adverts-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-245-advert-intervals-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1151-orphan-separators-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1486-collapse-reopens-detail-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-rebrand-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-theme-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-default-sage-teal-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1109-hamburger-dropdown-visible-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-layout-1178-1179-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1205-live-controls-anchor-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-mql-leak-1180-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1204-live-panel-structure-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1234-live-chrome-pass2-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-vcr-overlap-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1833-legend-toggle-clickable-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1244-live-vcr-row-hints-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1510-live-nav-pin-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-fullscreen-1572-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1599-replay-freeze-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-pr-1490-live-map-gpu-animations-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m1-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m2-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m3-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m4-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1657-analytics-channels-group-sprites-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1224-channels-mobile-ux-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1367-channels-chat-app-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-2052-touch-target-e2e.js 2>&1 | tee -a e2e-output.txt - # Standalone harness (no server): loads public/style.css into Chromium. - CHROMIUM_REQUIRE=1 node test-touch-targets.js 2>&1 | tee -a e2e-output.txt - # Starts its own ingestor + server on a temp copy of the fixture. - CHROMIUM_REQUIRE=1 CORESCOPE_SERVER_BIN=./corescope-server CORESCOPE_INGESTOR_BIN=./corescope-ingestor FIXTURE_DB=test-fixtures/e2e-fixture.db node test-channel-proposals-e2e.js 2>&1 | tee -a e2e-output.txt - # Starts its own ingestor + server on a temp copy of the fixture; seeds client-decryptable packets itself. - CHROMIUM_REQUIRE=1 CORESCOPE_SERVER_BIN=./corescope-server CORESCOPE_INGESTOR_BIN=./corescope-ingestor FIXTURE_DB=test-fixtures/e2e-fixture.db node test-channels-client-state-152-decrypt-e2e.js 2>&1 | tee -a e2e-output.txt - # #315 uses synthetic data and its own default/on/off Go servers. - # Pin the existing Leaflet runtime; browser network stays local-only. - leaflet_dir="$RUNNER_TEMP/corescope-315-leaflet" - mkdir -p "$leaflet_dir/images" - curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/leaflet.js -o "$leaflet_dir/leaflet.js" - curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/leaflet.css -o "$leaflet_dir/leaflet.css" - curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon.png -o "$leaflet_dir/images/marker-icon.png" - curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon-2x.png -o "$leaflet_dir/images/marker-icon-2x.png" - curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-shadow.png -o "$leaflet_dir/images/marker-shadow.png" - echo "db49d009c841f5ca34a888c96511ae936fd9f5533e90d8b2c4d57596f4e5641a $leaflet_dir/leaflet.js" | sha256sum --check - echo "a7837102824184820dfa198d1ebcd109ff6d0ff9a2672a074b9a1b4d147d04c6 $leaflet_dir/leaflet.css" | sha256sum --check - echo "574c3a5cca85f4114085b6841596d62f00d7c892c7b03f28cbfa301deb1dc437 $leaflet_dir/images/marker-icon.png" | sha256sum --check - echo "00179c4c1ee830d3a108412ae0d294f55776cfeb085c60129a39aa6fc4ae2528 $leaflet_dir/images/marker-icon-2x.png" | sha256sum --check - echo "264f5c640339f042dd729062cfc04c17f8ea0f29882b538e3848ed8f10edb4da $leaflet_dir/images/marker-shadow.png" | sha256sum --check - CHROMIUM_REQUIRE=1 CORESCOPE_SERVER_BIN=./corescope-server CORESCOPE_MIGRATE_BIN=./corescope-migrate CORESCOPE_LEAFLET_DIR="$leaflet_dir" node test-issue-315-estimated-positions-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1236-map-mobile-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1329-map-controls-accordion-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1273-qr-overlay-height-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1281-location-row-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1279-legend-p2-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1799-label-vocab-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-home-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-issue-304-my-mesh-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-issue-2027-my-mesh-node-page-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-path-inspector-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-resize-observer-leak-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-drawer-1064-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-111-drawer-version-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-audio-live-1297-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-audio-lab-1297-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-decrypt-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-qr-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-color-picker-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-theme-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-branding-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-display-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-export-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-drag-manager-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1567-corner-clears-drag-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1306-collisions-terminology-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-205-analytics-subtab-deeplinks-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-226-hash-stats-sort-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-236-analytics-sorting-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1374-route-map-a11y-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-list-render-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-selection-flow-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-add-modal-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-share-color-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-batch-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-race-1498-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-observed-path-hash-size-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1487-byop-modal-layout-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1630-reach-mobile-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-node-reach-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-reach-rank-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1640-compare-discovery-e2e.js 2>&1 | tee -a e2e-output.txt - # These eight existing root E2Es previously had no CI runner. - # test-rx-coverage-mobile-nav-e2e.js remains excluded because it - # skips when clientRxCoverage is disabled, the default configuration. - BASE_URL=http://localhost:13581 node test-1110-live-filter.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-analytics-fluid-charts.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-e2e-1267-mobile-vcr.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1274-legend-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m5-icons-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-live-dedup.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-nodes-export-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-show-neighbors.js 2>&1 | tee -a e2e-output.txt - # Root E2Es without a runner until #174. They do not use the fixture - # server: the two contrast gates render static markup, and the other two - # serve their own pages. - CHROMIUM_REQUIRE=1 node test-a11y-1705-subpath-hop-prefix-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-a11y-1719-contrast-root-causes-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-nodes-favorite-rerender-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-xss-333-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-packet-trace-alignment-e2e.js 2>&1 | tee -a e2e-output.txt - # Root E2Es repaired or ported to plain playwright in #189. - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-pulse-ring-weight-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channel-modal-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-node-reach-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-path-inspector-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1522-trace-url-sync-e2e.js 2>&1 | tee -a e2e-output.txt - - # #1616: slide-over focus-restore flake-gate. Runs the slide-over - # E2E 20 consecutive times against the SAME backend instance so - # the Chromium-headless focus race documented in #1172/#1616 has - # a 20× shot at firing. Any single non-zero exit aborts. This is - # the architectural-fix gate — if it ever turns red post-merge, - # the focused-but-hidden state has crept back in. - # - # PERMANENT step. Adds ~3-4 min to the e2e-test job in exchange - # for closing out a flake family that was blocking ~8 unrelated - # PRs at a time. If profiling pressures the budget later, drop - # repeat count first; do not delete. - - name: Slide-over E2E flake-gate (#1616, --repeat-each=3) - run: | - set -e - for i in $(seq 1 3); do - echo "--- slide-over E2E run $i/20 ---" - BASE_URL=http://localhost:13581 node test-slideover-1056-e2e.js 2>&1 | tee -a slideover-repeat-output.txt - done - echo "3 passed" + - name: Run assigned Playwright suites and repeat gate (fail-fast) + run: node scripts/run-e2e-shard.js - name: Collect frontend coverage (parallel) - if: success() && github.event_name == 'push' + if: success() && github.event_name == 'push' && steps.shard-meta.outputs.core_shard == env.E2E_SHARD + run: | + node scripts/collect-frontend-coverage.js 2>&1 | tee fe-coverage-output.txt || true + + - name: Bundle core frontend coverage + if: success() run: | - BASE_URL=http://localhost:13581 node scripts/collect-frontend-coverage.js 2>&1 | tee fe-coverage-output.txt || true + if [ -d .nyc_output ]; then + cp -R .nyc_output ".ci-e2e/shard-$E2E_SHARD/coverage" + fi + + - name: Stop test server + if: always() + run: | + if [ -f .server.pid ]; then + kill $(cat .server.pid) 2>/dev/null || true + rm -f .server.pid + fi + + - name: Upload E2E shard receipt and logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: e2e-shard-${{ env.E2E_SHARD }} + overwrite: true + path: .ci-e2e/shard-${{ env.E2E_SHARD }}/ + retention-days: 1 + if-no-files-found: error + include-hidden-files: true + + e2e-test-2: + name: "🎭 Playwright E2E Tests (2/3)" + runs-on: ubuntu-latest + timeout-minutes: 150 + defaults: + run: + shell: bash + env: + E2E_SHARD: '2' + E2E_PORT: '13582' + BASE_URL: http://localhost:13582 + steps: *e2e-shard-steps + + e2e-test-3: + name: "🎭 Playwright E2E Tests (3/3)" + runs-on: ubuntu-latest + timeout-minutes: 150 + defaults: + run: + shell: bash + env: + E2E_SHARD: '3' + E2E_PORT: '13583' + BASE_URL: http://localhost:13583 + steps: *e2e-shard-steps + + e2e-report: + name: "📊 Verify E2E Shards & Coverage" + needs: [e2e-test-1, e2e-test-2, e2e-test-3] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash + env: + E2E_1_RESULT: ${{ needs.e2e-test-1.result }} + E2E_2_RESULT: ${{ needs.e2e-test-2.result }} + E2E_3_RESULT: ${{ needs.e2e-test-3.result }} + steps: + - name: Require every E2E shard to succeed + run: | + for result in "$E2E_1_RESULT" "$E2E_2_RESULT" "$E2E_3_RESULT"; do + if [ "$result" != success ]; then + echo "E2E shard did not succeed: '$result'" + exit 1 + fi + done + + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Node.js 22 + uses: actions/setup-node@v5 + with: + node-version: '22' + + - name: Install npm dependencies + run: npm ci --production=false + + - name: Download E2E shard receipts and logs + uses: actions/download-artifact@v6 + with: + pattern: e2e-shard-* + path: .ci-e2e/downloaded + merge-multiple: false + + - name: Verify every assigned suite executed successfully + run: node scripts/report-e2e-shards.js - name: Generate frontend coverage badges if: success() @@ -652,19 +546,12 @@ jobs: fi echo "{\"schemaVersion\":1,\"label\":\"e2e tests\",\"message\":\"${E2E_MSG}\",\"color\":\"${E2E_COLOR}\"}" > .badges/e2e-tests.json - - name: Stop test server - if: always() - run: | - if [ -f .server.pid ]; then - kill $(cat .server.pid) 2>/dev/null || true - rm -f .server.pid - fi - - name: Upload E2E badges if: success() uses: actions/upload-artifact@v6 with: name: e2e-badges + overwrite: true path: .badges/ retention-days: 1 if-no-files-found: ignore @@ -701,7 +588,7 @@ jobs: # ─────────────────────────────────────────────────────────────── build-and-publish: name: "🏗️ Build & Publish Docker Image" - needs: [go-test, e2e-test, image-check] + needs: [go-test, e2e-test-1, e2e-test-2, e2e-test-3, e2e-report, image-check] # A failed dependency normally skips this job. A skipped required check # can count as passing in branch protection, so run and fail explicitly. # A cancelled run reports this job as cancelled (not skipped), which does @@ -713,10 +600,13 @@ jobs: - name: Require tests and image validation env: GO_RESULT: ${{ needs.go-test.result }} - E2E_RESULT: ${{ needs.e2e-test.result }} + E2E_1_RESULT: ${{ needs.e2e-test-1.result }} + E2E_2_RESULT: ${{ needs.e2e-test-2.result }} + E2E_3_RESULT: ${{ needs.e2e-test-3.result }} + E2E_REPORT_RESULT: ${{ needs.e2e-report.result }} IMAGE_RESULT: ${{ needs.image-check.result }} run: | - for result in "$GO_RESULT" "$E2E_RESULT" "$IMAGE_RESULT"; do + for result in "$GO_RESULT" "$E2E_1_RESULT" "$E2E_2_RESULT" "$E2E_3_RESULT" "$E2E_REPORT_RESULT" "$IMAGE_RESULT"; do if [ "$result" != success ]; then echo "Required CI job did not pass: $result" exit 1 diff --git a/cmd/server/fork_guard_workflow_test.go b/cmd/server/fork_guard_workflow_test.go index f754829f4..2a8ec73e5 100644 --- a/cmd/server/fork_guard_workflow_test.go +++ b/cmd/server/fork_guard_workflow_test.go @@ -187,7 +187,7 @@ func TestImageCheckLeastPrivilegeAndTimeout(t *testing.T) { func TestForkGuardLeavesTestJobsRunning(t *testing.T) { deploy := readWorkflow(t, deployWorkflowRel) - for _, name := range []string{"go-test", "e2e-test", "image-check"} { + for _, name := range []string{"go-test", "e2e-test-1", "e2e-test-2", "e2e-test-3", "image-check"} { job := workflowJob(t, deploy, name) if cond := jobIf(job); cond != "" { t.Errorf("%s must not gain a job-level condition; got %q", name, cond) @@ -196,6 +196,13 @@ func TestForkGuardLeavesTestJobsRunning(t *testing.T) { t.Errorf("%s must not be repository-guarded: tests and CI artifact uploads run on forks too", name) } } + report := workflowJob(t, deploy, "e2e-report") + if cond := strings.TrimSpace(jobIf(report)); cond != "${{ !cancelled() }}" { + t.Errorf("e2e-report must run after failed or skipped shards; got if %q", cond) + } + if strings.Contains(report, "github.repository") { + t.Error("e2e-report must not be repository-guarded") + } } // The required check is configured by name, so renaming the gate job would @@ -231,3 +238,72 @@ func TestForkGuardCount(t *testing.T) { } } } + +// A single shared steps anchor prevents fixtures or setup from silently +// diverging between shards. The alias jobs may vary only their local port +// and shard identity, never their preparation or suite runner. +func TestE2EShardsShareFixtureSteps(t *testing.T) { + deploy := readWorkflow(t, deployWorkflowRel) + first := workflowJob(t, deploy, "e2e-test-1") + if !strings.Contains(first, " steps: &e2e-shard-steps\n") { + t.Fatal("shard 1 must declare the shared E2E steps anchor") + } + for _, c := range []struct{ job, shard, port string }{ + {"e2e-test-1", "1", "13581"}, + {"e2e-test-2", "2", "13582"}, + {"e2e-test-3", "3", "13583"}, + } { + job := workflowJob(t, deploy, c.job) + for _, want := range []string{ + " E2E_SHARD: '" + c.shard + "'\n", + " E2E_PORT: '" + c.port + "'\n", + " BASE_URL: http://localhost:" + c.port + "\n", + " timeout-minutes: 150\n", + } { + if !strings.Contains(job, want) { + t.Errorf("%s must contain %q", c.job, want) + } + } + if c.shard != "1" && !strings.HasSuffix(strings.TrimSpace(job), "steps: *e2e-shard-steps") { + t.Errorf("%s must use the identical shared steps, without overrides", c.job) + } + } + previous := -1 + for _, marker := range []string{ + "bash tools/freshen-fixture.sh test-fixtures/e2e-fixture.db", + "sqlite3 test-fixtures/e2e-fixture.db <<'SQL'", + "./corescope-migrate -db test-fixtures/e2e-fixture.db", + "sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-2073-route-adverts.sql", + "sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-199-inactive-observer.sql", + "sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-245-advert-intervals.sql", + "./corescope-server -port \"$E2E_PORT\" -db test-fixtures/e2e-fixture.db -public public-instrumented &", + "node scripts/run-e2e-shard.js\n", + } { + index := strings.Index(first, marker) + if index <= previous { + t.Fatalf("fixture step %q is missing or out of order", marker) + } + previous = index + } +} + +func TestE2ECoverageRunsOnCoreShardOnly(t *testing.T) { + deploy := readWorkflow(t, deployWorkflowRel) + first := workflowJob(t, deploy, "e2e-test-1") + if !strings.Contains(first, "node scripts/run-e2e-shard.js --core-shard") { + t.Error("coverage owner must be identified from the checked-in manifest") + } + const condition = "if: success() && github.event_name == 'push' && steps.shard-meta.outputs.core_shard == env.E2E_SHARD" + if !strings.Contains(first, condition) { + t.Error("coverage collection must stay push-only and run once on the core suite's shard") + } + if strings.Contains(first, "Generate frontend coverage badges") { + t.Error("per-shard badges must not replace the aggregate badge summary") + } + report := workflowJob(t, deploy, "e2e-report") + for _, need := range []string{"node scripts/report-e2e-shards.js", "Generate frontend coverage badges", "name: e2e-badges"} { + if !strings.Contains(report, need) { + t.Errorf("e2e-report must preserve %q", need) + } + } +} diff --git a/scripts/ci/e2e-manifest.js b/scripts/ci/e2e-manifest.js new file mode 100644 index 000000000..a4bbeba5e --- /dev/null +++ b/scripts/ci/e2e-manifest.js @@ -0,0 +1,113 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { createHash } = require('node:crypto'); + +const SHARDS = [1, 2, 3]; +const CORE_SUITE = 'test-e2e-playwright.js'; +const REPEAT_SUITE = 'test-slideover-1056-e2e.js'; +const NODE_LIVENESS = 'test-node-liveness-e2e.js'; +const LEGACY_EXCLUSIONS = new Set([ + 'test-rx-coverage-mobile-nav-e2e.js', +]); +const ENV_KEYS = new Set([ + 'BASE_URL', 'CHROMIUM_REQUIRE', 'AXE_SCREENSHOT_DIR', + 'CORESCOPE_SERVER_BIN', 'CORESCOPE_INGESTOR_BIN', 'FIXTURE_DB', + 'CORESCOPE_MIGRATE_BIN', 'CORESCOPE_LEAFLET_DIR', +]); + +function suiteFile(file) { + return typeof file === 'string' && /^test-[A-Za-z0-9_-]+\.js$/.test(file); +} + +function discoverSuites(root) { + const units = new Set([...fs.readFileSync(path.join(root, 'test-all.sh'), 'utf8') + .matchAll(/^run (test-[A-Za-z0-9_-]+\.js)\s*$/gm)].map(match => match[1])); + return fs.readdirSync(root).filter(file => /^test-.*\.js$/.test(file)).filter(file => { + const full = path.join(root, file); + assert.ok(fs.lstatSync(full).isFile(), `suite must be a regular file: ${file}`); + // #1697 has an E2E filename but is a VM unit test in the unit registry. + if (file === 'test-issue-1697-mqtt-mobile-e2e.js' && units.has(file)) return false; + if (/e2e|playwright/.test(file)) return true; + return /(?:\b(?:require|import)\s*\(\s*|\bfrom\s*)['"](?:playwright(?:\/[^'"]+)?|@playwright\/test(?:\/[^'"]+)?)['"]/.test(fs.readFileSync(full, 'utf8')); + }).sort(); +} + +function validateManifest(manifest, root) { + assert.ok(manifest && typeof manifest === 'object' && !Array.isArray(manifest), 'manifest must be an object'); + assert.equal(manifest.version, 1, 'unsupported manifest version'); + assert.ok(Array.isArray(manifest.suites), 'manifest suites must be an array'); + assert.ok(Array.isArray(manifest.exclusions), 'manifest exclusions must be an array'); + const assigned = new Map(); + for (const suite of manifest.suites) { + assert.ok(suite && typeof suite === 'object', 'invalid suite entry'); + assert.ok(suiteFile(suite.file), `invalid suite filename: ${suite.file}`); + assert.notEqual(suite.file, 'test-issue-1697-mqtt-mobile-e2e.js', '#1697 must remain in the unit registry'); + assert.ok(SHARDS.includes(suite.shard), `invalid shard for ${suite.file}`); + assert.ok(!assigned.has(suite.file), `duplicate suite assignment: ${suite.file}`); + assert.ok(fs.lstatSync(path.join(root, suite.file)).isFile(), `missing suite file: ${suite.file}`); + assert.ok(suite.env && typeof suite.env === 'object' && !Array.isArray(suite.env), `env must be an object: ${suite.file}`); + for (const [key, value] of Object.entries(suite.env)) { + assert.ok(ENV_KEYS.has(key), `unsupported environment key: ${key}`); + assert.equal(typeof value, 'string', `non-string environment value: ${key}`); + assert.ok(!/[\x00\r\n]/.test(value), `invalid environment value: ${key}`); + assert.ok(!value.includes('$') || value === '$BASE_URL' || value === '$LEAFLET_DIR', `unsupported environment expansion: ${key}`); + if (key === 'BASE_URL') assert.equal(value, '$BASE_URL', 'BASE_URL must use the pinned local shard target'); + } + assert.ok(['e2e', 'node-liveness'].includes(suite.output), `invalid output for ${suite.file}`); + assert.equal(suite.output === 'node-liveness', suite.file === NODE_LIVENESS, 'only node liveness uses its separate output'); + if (suite.setup !== undefined) { + assert.equal(suite.file, 'test-issue-315-estimated-positions-e2e.js', 'setup is only allowed for #315'); + assert.equal(suite.setup, 'leaflet-315', 'unknown suite setup'); + } + if (suite.repeatGate !== undefined) { + assert.equal(suite.file, REPEAT_SUITE, 'only slide-over has the repeat gate'); + assert.equal(suite.repeatGate, 3, 'the permanent slide-over gate must run three times'); + } + assigned.set(suite.file, suite); + } + for (const shard of SHARDS) { + assert.ok(manifest.suites.some(suite => suite.shard === shard), `empty shard ${shard}`); + } + assert.ok(assigned.has(CORE_SUITE), 'missing core Playwright suite'); + assert.ok(assigned.has(NODE_LIVENESS), 'missing node-liveness suite'); + assert.equal(assigned.get(REPEAT_SUITE)?.repeatGate, 3, 'missing permanent slide-over repeat gate'); + const excluded = new Set(); + for (const entry of manifest.exclusions) { + assert.ok(entry && LEGACY_EXCLUSIONS.has(entry.file), `unapproved suite exclusion: ${entry?.file}`); + assert.ok(typeof entry.reason === 'string' && entry.reason.trim().length > 20, `missing exclusion rationale: ${entry.file}`); + assert.ok(!excluded.has(entry.file), `duplicate exclusion: ${entry.file}`); + assert.ok(!assigned.has(entry.file), `assigned suite also excluded: ${entry.file}`); + assert.ok(fs.lstatSync(path.join(root, entry.file)).isFile(), `missing excluded suite: ${entry.file}`); + excluded.add(entry.file); + } + for (const file of discoverSuites(root)) { + assert.ok(assigned.has(file) || excluded.has(file), `unassigned E2E suite: ${file}`); + } + return manifest; +} + +function readManifest(file, root) { + const raw = fs.readFileSync(file, 'utf8'); + return { + manifest: validateManifest(JSON.parse(raw), root), + manifestSha256: createHash('sha256').update(raw).digest('hex'), + }; +} + +function expectedRuns(manifest, shard) { + const suites = manifest.suites.filter(suite => suite.shard === shard); + const ordinary = suites.map(suite => ({ file: suite.file, kind: 'suite', iteration: 1 })); + const repeated = suites.flatMap(suite => Array.from({ length: suite.repeatGate || 0 }, (_, index) => ({ + file: suite.file, kind: 'repeat-gate', iteration: index + 1, + }))); + return [...ordinary, ...repeated]; +} + +function runContext(env) { + return { runId: env.GITHUB_RUN_ID || null, attempt: env.GITHUB_RUN_ATTEMPT || null, headSha: env.GITHUB_SHA || null }; +} + +module.exports = { SHARDS, CORE_SUITE, REPEAT_SUITE, NODE_LIVENESS, discoverSuites, validateManifest, readManifest, expectedRuns, runContext }; diff --git a/scripts/ci/prepare-e2e-leaflet.sh b/scripts/ci/prepare-e2e-leaflet.sh new file mode 100644 index 000000000..2679614c0 --- /dev/null +++ b/scripts/ci/prepare-e2e-leaflet.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${RUNNER_TEMP:?RUNNER_TEMP is required}" +leaflet_dir="$RUNNER_TEMP/corescope-315-leaflet" +mkdir -p "$leaflet_dir/images" +curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/leaflet.js -o "$leaflet_dir/leaflet.js" +curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/leaflet.css -o "$leaflet_dir/leaflet.css" +curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon.png -o "$leaflet_dir/images/marker-icon.png" +curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon-2x.png -o "$leaflet_dir/images/marker-icon-2x.png" +curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-shadow.png -o "$leaflet_dir/images/marker-shadow.png" +echo "db49d009c841f5ca34a888c96511ae936fd9f5533e90d8b2c4d57596f4e5641a $leaflet_dir/leaflet.js" | sha256sum --check +echo "a7837102824184820dfa198d1ebcd109ff6d0ff9a2672a074b9a1b4d147d04c6 $leaflet_dir/leaflet.css" | sha256sum --check +echo "574c3a5cca85f4114085b6841596d62f00d7c892c7b03f28cbfa301deb1dc437 $leaflet_dir/images/marker-icon.png" | sha256sum --check +echo "00179c4c1ee830d3a108412ae0d294f55776cfeb085c60129a39aa6fc4ae2528 $leaflet_dir/images/marker-icon-2x.png" | sha256sum --check +echo "264f5c640339f042dd729062cfc04c17f8ea0f29882b538e3848ed8f10edb4da $leaflet_dir/images/marker-shadow.png" | sha256sum --check diff --git a/scripts/e2e-shards.json b/scripts/e2e-shards.json new file mode 100644 index 000000000..c0e800d73 --- /dev/null +++ b/scripts/e2e-shards.json @@ -0,0 +1,2318 @@ +{ + "version": 1, + "suites": [ + { + "file": "test-node-liveness-e2e.js", + "shard": 2, + "env": {}, + "output": "node-liveness", + "meanSeconds": 2.9313333333333333, + "secondsByExecution": [ + 2.005, + 3.123, + 3.666 + ] + }, + { + "file": "test-e2e-playwright.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 113.346, + "secondsByExecution": [ + 102.226, + 118.828, + 118.984 + ] + }, + { + "file": "test-a11y-axe-1668.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL", + "AXE_SCREENSHOT_DIR": "/tmp/axe-1668" + }, + "output": "e2e", + "meanSeconds": 395.7253333333333, + "secondsByExecution": [ + 338.17, + 425.371, + 423.635 + ] + }, + { + "file": "test-filter-ux-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.3060000000000005, + "secondsByExecution": [ + 1.872, + 2.494, + 2.552 + ] + }, + { + "file": "test-channel-issue-1087-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.5233333333333334, + "secondsByExecution": [ + 2.084, + 2.738, + 2.748 + ] + }, + { + "file": "test-channel-issue-1111-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.8086666666666666, + "secondsByExecution": [ + 1.454, + 1.983, + 1.989 + ] + }, + { + "file": "test-channels-client-state-152-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 9.437333333333333, + "secondsByExecution": [ + 8.786, + 9.751, + 9.775 + ] + }, + { + "file": "test-channels-154-155-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 13.274000000000001, + "secondsByExecution": [ + 12.169, + 13.74, + 13.913 + ] + }, + { + "file": "test-map-modal-fluid-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.851666666666667, + "secondsByExecution": [ + 5.217, + 6.144, + 6.194 + ] + }, + { + "file": "test-issue-332-no-carto-tiles-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 86.52933333333334, + "secondsByExecution": [ + 80.243, + 89.187, + 90.158 + ] + }, + { + "file": "test-map-nodes-pagination-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.5026666666666666, + "secondsByExecution": [ + 1.165, + 1.668, + 1.675 + ] + }, + { + "file": "test-issue-123-map-lifecycle-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 29.82766666666667, + "secondsByExecution": [ + 27.094, + 30.984, + 31.405 + ] + }, + { + "file": "test-observer-iata-1188-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.287333333333334, + "secondsByExecution": [ + 3.418, + 4.688, + 4.756 + ] + }, + { + "file": "test-xss-333-pages-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 13.585666666666667, + "secondsByExecution": [ + 11.445, + 14.462, + 14.85 + ] + }, + { + "file": "test-xss-392-perf-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.9543333333333333, + "secondsByExecution": [ + 1.642, + 2.115, + 2.106 + ] + }, + { + "file": "test-xss-368-home-links-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.8883333333333334, + "secondsByExecution": [ + 1.548, + 2.059, + 2.058 + ] + }, + { + "file": "test-xss-429-home-footer-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 13.470333333333334, + "secondsByExecution": [ + 11.592, + 14.217, + 14.602 + ] + }, + { + "file": "test-issue-1639-observers-sort-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.427333333333333, + "secondsByExecution": [ + 4.609, + 5.81, + 5.863 + ] + }, + { + "file": "test-issue-405-observers-listener-leak-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 8.784666666666666, + "secondsByExecution": [ + 7.694, + 9.225, + 9.435 + ] + }, + { + "file": "test-issue-1758-ng-filter-rerenders-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.2823333333333333, + "secondsByExecution": [ + 1.95, + 2.452, + 2.445 + ] + }, + { + "file": "test-nav-fluid-1055-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.2216666666666662, + "secondsByExecution": [ + 2.917, + 3.29, + 3.458 + ] + }, + { + "file": "test-nav-priority-1102-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.7716666666666667, + "secondsByExecution": [ + 1.537, + 1.868, + 1.91 + ] + }, + { + "file": "test-nav-priority-1311-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.0623333333333336, + "secondsByExecution": [ + 2.691, + 3.238, + 3.258 + ] + }, + { + "file": "test-nav-priority-1391-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.211333333333333, + "secondsByExecution": [ + 3.765, + 4.436, + 4.433 + ] + }, + { + "file": "test-issue-1413-nav-overlap-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.9426666666666668, + "secondsByExecution": [ + 1.678, + 2.075, + 2.075 + ] + }, + { + "file": "test-issue-1400-nav-vertical-clip.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.619, + "secondsByExecution": [ + 1.359, + 1.764, + 1.734 + ] + }, + { + "file": "test-nav-more-floor-1139-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.504, + "secondsByExecution": [ + 2.204, + 2.647, + 2.661 + ] + }, + { + "file": "test-bottom-nav-1061-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.9260000000000002, + "secondsByExecution": [ + 1.657, + 2.035, + 2.086 + ] + }, + { + "file": "test-gestures-1062-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.934666666666668, + "secondsByExecution": [ + 4.518, + 5.102, + 5.184 + ] + }, + { + "file": "test-gestures-1185-scroll-discriminator-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.285666666666667, + "secondsByExecution": [ + 2.029, + 2.406, + 2.422 + ] + }, + { + "file": "test-gesture-hints-1065-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 25.150333333333332, + "secondsByExecution": [ + 23.726, + 25.735, + 25.99 + ] + }, + { + "file": "test-touch-gestures-coverage-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 15.239333333333333, + "secondsByExecution": [ + 14.664, + 15.463, + 15.591 + ] + }, + { + "file": "test-channel-fluid-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.991666666666667, + "secondsByExecution": [ + 5.003, + 6.415, + 6.557 + ] + }, + { + "file": "test-table-fluid-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 9.770666666666665, + "secondsByExecution": [ + 8.671, + 10.226, + 10.415 + ] + }, + { + "file": "test-charts-fluid-1058-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 11.130333333333333, + "secondsByExecution": [ + 9.469, + 11.822, + 12.1 + ] + }, + { + "file": "test-slideover-1056-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "repeatGate": 3, + "meanSeconds": 14.956666666666669, + "secondsByExecution": [ + 12.841, + 15.702, + 16.327 + ] + }, + { + "file": "test-issue-1692-packets-init-parallel-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.4906666666666666, + "secondsByExecution": [ + 1.172, + 1.636, + 1.664 + ] + }, + { + "file": "test-issue-147-packets-url-obs-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 13.307333333333332, + "secondsByExecution": [ + 11.652, + 13.967, + 14.303 + ] + }, + { + "file": "test-issue-165-grouped-hop-warn-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 6.162333333333334, + "secondsByExecution": [ + 5.179, + 6.572, + 6.736 + ] + }, + { + "file": "test-packet-detail-sender-hash-size-obs-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.9596666666666667, + "secondsByExecution": [ + 2.324, + 3.264, + 3.291 + ] + }, + { + "file": "test-issue-353-path-hash-warn-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 17.19, + "secondsByExecution": [ + 13.948, + 18.905, + 18.717 + ] + }, + { + "file": "test-issue-96-hide-control-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 9.615666666666668, + "secondsByExecution": [ + 7.105, + 11.092, + 10.65 + ] + }, + { + "file": "test-issue-242-type-exclusions-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.44, + "secondsByExecution": [ + 4.897, + 5.844, + 5.579 + ] + }, + { + "file": "test-issue-406-packets-live-channel-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 4.835333333333334, + "secondsByExecution": [ + 4.57, + 4.999, + 4.937 + ] + }, + { + "file": "test-issue-180-packets-url-modal-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 20.19, + "secondsByExecution": [ + 16.914, + 22.457, + 21.199 + ] + }, + { + "file": "test-slideover-1168-munger-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.8729999999999998, + "secondsByExecution": [ + 3.083, + 4.466, + 4.07 + ] + }, + { + "file": "test-logo-pulse-1173-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.418666666666667, + "secondsByExecution": [ + 3.687, + 4.942, + 4.627 + ] + }, + { + "file": "test-issue-1122-packets-filter-ux-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.8433333333333335, + "secondsByExecution": [ + 1.471, + 2.078, + 1.981 + ] + }, + { + "file": "test-issue-1122-details-row-clamp-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.007000000000001, + "secondsByExecution": [ + 3.968, + 5.832, + 5.221 + ] + }, + { + "file": "test-issue-258-column-widths-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 25.788, + "secondsByExecution": [ + 22.59, + 28.13, + 26.644 + ] + }, + { + "file": "test-issue-1128-packets-layout-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.2053333333333334, + "secondsByExecution": [ + 1.791, + 2.523, + 2.302 + ] + }, + { + "file": "test-issue-189-group-caret-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.8366666666666667, + "secondsByExecution": [ + 1.443, + 2.11, + 1.957 + ] + }, + { + "file": "test-issue-254-affinity-toggle-mobile-aria-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 7.23, + "secondsByExecution": [ + 6.069, + 7.984, + 7.637 + ] + }, + { + "file": "test-issue-259-mobile-expanded-group-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.244666666666667, + "secondsByExecution": [ + 4.286, + 5.831, + 5.617 + ] + }, + { + "file": "test-issue-1128-multi-viewport-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.819333333333333, + "secondsByExecution": [ + 4.619, + 6.548, + 6.291 + ] + }, + { + "file": "test-issue-1136-live-region-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.6823333333333335, + "secondsByExecution": [ + 1.197, + 1.979, + 1.871 + ] + }, + { + "file": "test-live-multibyte-only-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 8.583, + "secondsByExecution": [ + 6.273, + 9.878, + 9.598 + ] + }, + { + "file": "test-issue-125-live-toggles-early-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 19.155666666666665, + "secondsByExecution": [ + 14.006, + 22.092, + 21.369 + ] + }, + { + "file": "test-issue-220-live-filter-readiness-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 22.893666666666665, + "secondsByExecution": [ + 18.948, + 25.106, + 24.627 + ] + }, + { + "file": "test-issue-1150-404-state-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.352, + "secondsByExecution": [ + 0.945, + 1.585, + 1.526 + ] + }, + { + "file": "test-issue-199-inactive-observer-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.483666666666667, + "secondsByExecution": [ + 1.061, + 1.708, + 1.682 + ] + }, + { + "file": "test-issue-1146-path-link-contrast-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.446000000000001, + "secondsByExecution": [ + 3.242, + 5.082, + 5.014 + ] + }, + { + "file": "test-issue-1705-subpath-contrast-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 0.646, + "secondsByExecution": [ + 0.446, + 0.761, + 0.731 + ] + }, + { + "file": "test-issue-1147-section-order-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.8010000000000002, + "secondsByExecution": [ + 1.068, + 2.156, + 2.179 + ] + }, + { + "file": "test-issue-2073-recent-adverts-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 7.09, + "secondsByExecution": [ + 6.217, + 7.546, + 7.507 + ] + }, + { + "file": "test-issue-245-advert-intervals-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.9570000000000003, + "secondsByExecution": [ + 2.118, + 3.414, + 3.339 + ] + }, + { + "file": "test-issue-1151-orphan-separators-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.9963333333333333, + "secondsByExecution": [ + 1.444, + 2.34, + 2.205 + ] + }, + { + "file": "test-issue-1486-collapse-reopens-detail-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.472, + "secondsByExecution": [ + 2.029, + 2.714, + 2.673 + ] + }, + { + "file": "test-logo-rebrand-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.3160000000000003, + "secondsByExecution": [ + 1.75, + 2.633, + 2.565 + ] + }, + { + "file": "test-logo-theme-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.7260000000000002, + "secondsByExecution": [ + 1.245, + 1.998, + 1.935 + ] + }, + { + "file": "test-logo-default-sage-teal-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.3966666666666665, + "secondsByExecution": [ + 1.703, + 2.762, + 2.725 + ] + }, + { + "file": "test-issue-1109-hamburger-dropdown-visible-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.8033333333333335, + "secondsByExecution": [ + 1.259, + 2.117, + 2.034 + ] + }, + { + "file": "test-live-layout-1178-1179-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.392333333333333, + "secondsByExecution": [ + 2.611, + 3.793, + 3.773 + ] + }, + { + "file": "test-issue-1205-live-controls-anchor-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.241333333333333, + "secondsByExecution": [ + 4.099, + 5.858, + 5.767 + ] + }, + { + "file": "test-live-mql-leak-1180-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.613, + "secondsByExecution": [ + 4.04, + 4.835, + 4.964 + ] + }, + { + "file": "test-issue-1204-live-panel-structure-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.158666666666667, + "secondsByExecution": [ + 4.08, + 5.714, + 5.682 + ] + }, + { + "file": "test-issue-1234-live-chrome-pass2-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.488, + "secondsByExecution": [ + 2.733, + 3.861, + 3.87 + ] + }, + { + "file": "test-issue-1206-vcr-overlap-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.4006666666666665, + "secondsByExecution": [ + 2.591, + 3.815, + 3.796 + ] + }, + { + "file": "test-issue-1833-legend-toggle-clickable-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.6976666666666667, + "secondsByExecution": [ + 2.113, + 2.985, + 2.995 + ] + }, + { + "file": "test-issue-1244-live-vcr-row-hints-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 5.410666666666667, + "secondsByExecution": [ + 4.693, + 5.864, + 5.675 + ] + }, + { + "file": "test-issue-1510-live-nav-pin-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.0323333333333333, + "secondsByExecution": [ + 1.389, + 2.101, + 2.607 + ] + }, + { + "file": "test-live-fullscreen-1572-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.0613333333333332, + "secondsByExecution": [ + 1.549, + 2.342, + 2.293 + ] + }, + { + "file": "test-issue-1599-replay-freeze-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.010666666666666, + "secondsByExecution": [ + 3.49, + 4.264, + 4.278 + ] + }, + { + "file": "test-pr-1490-live-map-gpu-animations-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.1513333333333335, + "secondsByExecution": [ + 3.295, + 4.61, + 4.549 + ] + }, + { + "file": "test-issue-1648-m1-icons-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.6990000000000003, + "secondsByExecution": [ + 3.069, + 4.025, + 4.003 + ] + }, + { + "file": "test-issue-1648-m2-icons-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.095, + "secondsByExecution": [ + 1.412, + 2.492, + 2.381 + ] + }, + { + "file": "test-issue-1648-m3-icons-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.394666666666667, + "secondsByExecution": [ + 3.987, + 4.597, + 4.6 + ] + }, + { + "file": "test-issue-1648-m4-icons-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 13.578333333333333, + "secondsByExecution": [ + 12.997, + 13.895, + 13.843 + ] + }, + { + "file": "test-issue-1657-analytics-channels-group-sprites-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.135, + "secondsByExecution": [ + 1.695, + 2.363, + 2.347 + ] + }, + { + "file": "test-issue-1224-channels-mobile-ux-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 7.440333333333332, + "secondsByExecution": [ + 4.479, + 8.933, + 8.909 + ] + }, + { + "file": "test-issue-1367-channels-chat-app-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.667666666666667, + "secondsByExecution": [ + 2.061, + 3.019, + 2.923 + ] + }, + { + "file": "test-issue-2052-touch-target-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 19.30966666666667, + "secondsByExecution": [ + 16.273, + 21.09, + 20.566 + ] + }, + { + "file": "test-touch-targets.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 0.8516666666666666, + "secondsByExecution": [ + 0.583, + 0.996, + 0.976 + ] + }, + { + "file": "test-channel-proposals-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "CORESCOPE_SERVER_BIN": "./corescope-server", + "CORESCOPE_INGESTOR_BIN": "./corescope-ingestor", + "FIXTURE_DB": "test-fixtures/e2e-fixture.db" + }, + "output": "e2e", + "meanSeconds": 20.708333333333332, + "secondsByExecution": [ + 21.874, + 20.011, + 20.24 + ] + }, + { + "file": "test-channels-client-state-152-decrypt-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "CORESCOPE_SERVER_BIN": "./corescope-server", + "CORESCOPE_INGESTOR_BIN": "./corescope-ingestor", + "FIXTURE_DB": "test-fixtures/e2e-fixture.db" + }, + "output": "e2e", + "meanSeconds": 8.410333333333334, + "secondsByExecution": [ + 7.98, + 8.644, + 8.607 + ] + }, + { + "file": "test-issue-315-estimated-positions-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "CORESCOPE_SERVER_BIN": "./corescope-server", + "CORESCOPE_MIGRATE_BIN": "./corescope-migrate", + "CORESCOPE_LEAFLET_DIR": "$LEAFLET_DIR" + }, + "output": "e2e", + "setup": "leaflet-315", + "meanSeconds": 5.682666666666667, + "secondsByExecution": [ + 4.65, + 6.435, + 5.963 + ] + }, + { + "file": "test-issue-1236-map-mobile-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.169, + "secondsByExecution": [ + 2.425, + 3.668, + 3.414 + ] + }, + { + "file": "test-issue-1329-map-controls-accordion-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.4303333333333335, + "secondsByExecution": [ + 2.695, + 3.842, + 3.754 + ] + }, + { + "file": "test-issue-1273-qr-overlay-height-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.442, + "secondsByExecution": [ + 2.627, + 3.857, + 3.842 + ] + }, + { + "file": "test-issue-1281-location-row-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 20.979, + "secondsByExecution": [ + 23.193, + 26.846, + 12.898 + ] + }, + { + "file": "test-issue-1279-legend-p2-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.1256666666666666, + "secondsByExecution": [ + 1.765, + 2.399, + 2.213 + ] + }, + { + "file": "test-issue-1799-label-vocab-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.8633333333333333, + "secondsByExecution": [ + 2.201, + 3.148, + 3.241 + ] + }, + { + "file": "test-home-coverage-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.992, + "secondsByExecution": [ + 3.746, + 5.683, + 5.547 + ] + }, + { + "file": "test-issue-304-my-mesh-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 1.8373333333333335, + "secondsByExecution": [ + 1.405, + 2.13, + 1.977 + ] + }, + { + "file": "test-issue-2027-my-mesh-node-page-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 7.748666666666668, + "secondsByExecution": [ + 6.32, + 8.694, + 8.232 + ] + }, + { + "file": "test-path-inspector-coverage-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.6776666666666669, + "secondsByExecution": [ + 1.357, + 2.149, + 1.527 + ] + }, + { + "file": "test-issue-1206-resize-observer-leak-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 30.474666666666668, + "secondsByExecution": [ + 22.17, + 35.933, + 33.321 + ] + }, + { + "file": "test-nav-drawer-1064-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 12.766333333333334, + "secondsByExecution": [ + 10.993, + 14.289, + 13.017 + ] + }, + { + "file": "test-issue-111-drawer-version-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 11.359333333333334, + "secondsByExecution": [ + 9.816, + 12.367, + 11.895 + ] + }, + { + "file": "test-audio-live-1297-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.7009999999999998, + "secondsByExecution": [ + 1.232, + 1.994, + 1.877 + ] + }, + { + "file": "test-audio-lab-1297-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.5236666666666667, + "secondsByExecution": [ + 1.971, + 2.875, + 2.725 + ] + }, + { + "file": "test-channel-decrypt-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.413, + "secondsByExecution": [ + 1.028, + 1.654, + 1.557 + ] + }, + { + "file": "test-channel-qr-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.662, + "secondsByExecution": [ + 1.238, + 1.95, + 1.798 + ] + }, + { + "file": "test-channel-color-picker-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.7793333333333334, + "secondsByExecution": [ + 1.358, + 2.068, + 1.912 + ] + }, + { + "file": "test-customize-theme-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.401666666666667, + "secondsByExecution": [ + 3.355, + 3.468, + 3.382 + ] + }, + { + "file": "test-customize-branding-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.4750000000000005, + "secondsByExecution": [ + 3.882, + 4.876, + 4.667 + ] + }, + { + "file": "test-customize-display-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.909333333333333, + "secondsByExecution": [ + 3.364, + 4.261, + 4.103 + ] + }, + { + "file": "test-customize-export-e2e.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.7936666666666667, + "secondsByExecution": [ + 2.274, + 3.074, + 3.033 + ] + }, + { + "file": "test-drag-manager-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.559333333333334, + "secondsByExecution": [ + 3.678, + 4.977, + 5.023 + ] + }, + { + "file": "test-issue-1567-corner-clears-drag-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.5086666666666666, + "secondsByExecution": [ + 2.004, + 2.74, + 2.782 + ] + }, + { + "file": "test-issue-1306-collisions-terminology-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 18.437, + "secondsByExecution": [ + 15.702, + 19.864, + 19.745 + ] + }, + { + "file": "test-issue-205-analytics-subtab-deeplinks-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 10.950333333333333, + "secondsByExecution": [ + 9.219, + 11.902, + 11.73 + ] + }, + { + "file": "test-issue-226-hash-stats-sort-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 14.807, + "secondsByExecution": [ + 13.119, + 15.718, + 15.584 + ] + }, + { + "file": "test-issue-236-analytics-sorting-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.725333333333333, + "secondsByExecution": [ + 4.004, + 5.098, + 5.074 + ] + }, + { + "file": "test-issue-1374-route-map-a11y-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 10.14, + "secondsByExecution": [ + 9.718, + 10.599, + 10.103 + ] + }, + { + "file": "test-channels-list-render-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.0893333333333337, + "secondsByExecution": [ + 1.77, + 2.277, + 2.221 + ] + }, + { + "file": "test-channels-selection-flow-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.695, + "secondsByExecution": [ + 1.323, + 1.912, + 1.85 + ] + }, + { + "file": "test-channels-add-modal-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.2243333333333335, + "secondsByExecution": [ + 1.842, + 2.441, + 2.39 + ] + }, + { + "file": "test-channels-share-color-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.8726666666666667, + "secondsByExecution": [ + 1.5, + 2.086, + 2.032 + ] + }, + { + "file": "test-channels-ws-batch-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.5493333333333332, + "secondsByExecution": [ + 1.186, + 1.769, + 1.693 + ] + }, + { + "file": "test-channels-ws-race-1498-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.0766666666666667, + "secondsByExecution": [ + 2.349, + 3.467, + 3.414 + ] + }, + { + "file": "test-channels-observed-path-hash-size-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.4740000000000002, + "secondsByExecution": [ + 1.104, + 1.671, + 1.647 + ] + }, + { + "file": "test-issue-1487-byop-modal-layout-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.6130000000000002, + "secondsByExecution": [ + 1.282, + 1.79, + 1.767 + ] + }, + { + "file": "test-issue-1630-reach-mobile-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.7493333333333334, + "secondsByExecution": [ + 1.61, + 1.57, + 2.068 + ] + }, + { + "file": "test-node-reach-coverage-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 0.7589999999999999, + "secondsByExecution": [ + 0.627, + 0.834, + 0.816 + ] + }, + { + "file": "test-reach-rank-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.7043333333333335, + "secondsByExecution": [ + 3.74, + 5.244, + 5.129 + ] + }, + { + "file": "test-issue-1640-compare-discovery-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.011, + "secondsByExecution": [ + 3.252, + 4.476, + 4.305 + ] + }, + { + "file": "test-1110-live-filter.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 4.525666666666666, + "secondsByExecution": [ + 3.553, + 5.115, + 4.909 + ] + }, + { + "file": "test-analytics-fluid-charts.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.6236666666666664, + "secondsByExecution": [ + 2.293, + 2.807, + 2.771 + ] + }, + { + "file": "test-e2e-1267-mobile-vcr.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 6.154333333333334, + "secondsByExecution": [ + 5.614, + 6.44, + 6.409 + ] + }, + { + "file": "test-issue-1274-legend-coverage-e2e.js", + "shard": 1, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 3.3513333333333333, + "secondsByExecution": [ + 2.696, + 3.668, + 3.69 + ] + }, + { + "file": "test-issue-1648-m5-icons-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 9.148, + "secondsByExecution": [ + 8.467, + 9.498, + 9.479 + ] + }, + { + "file": "test-live-dedup.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 6.673333333333333, + "secondsByExecution": [ + 6.259, + 6.882, + 6.879 + ] + }, + { + "file": "test-nodes-export-e2e.js", + "shard": 3, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.015, + "secondsByExecution": [ + 1.519, + 2.292, + 2.234 + ] + }, + { + "file": "test-show-neighbors.js", + "shard": 2, + "env": { + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 9.449, + "secondsByExecution": [ + 9.062, + 9.668, + 9.617 + ] + }, + { + "file": "test-a11y-1705-subpath-hop-prefix-e2e.js", + "shard": 1, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 0.029333333333333333, + "secondsByExecution": [ + 0.023, + 0.034, + 0.031 + ] + }, + { + "file": "test-a11y-1719-contrast-root-causes-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 0.03566666666666667, + "secondsByExecution": [ + 0.029, + 0.041, + 0.037 + ] + }, + { + "file": "test-nodes-favorite-rerender-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 1.2483333333333333, + "secondsByExecution": [ + 0.976, + 1.432, + 1.337 + ] + }, + { + "file": "test-xss-333-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 0.5643333333333334, + "secondsByExecution": [ + 0.414, + 0.647, + 0.632 + ] + }, + { + "file": "test-packet-trace-alignment-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1" + }, + "output": "e2e", + "meanSeconds": 0.7123333333333334, + "secondsByExecution": [ + 0.55, + 0.804, + 0.783 + ] + }, + { + "file": "test-live-pulse-ring-weight-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.5719999999999996, + "secondsByExecution": [ + 2.082, + 2.878, + 2.756 + ] + }, + { + "file": "test-channel-modal-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.746333333333333, + "secondsByExecution": [ + 1.277, + 2.062, + 1.9 + ] + }, + { + "file": "test-node-reach-e2e.js", + "shard": 2, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.6333333333333335, + "secondsByExecution": [ + 1.19, + 1.906, + 1.804 + ] + }, + { + "file": "test-path-inspector-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 2.856, + "secondsByExecution": [ + 2.497, + 3.144, + 2.927 + ] + }, + { + "file": "test-issue-1522-trace-url-sync-e2e.js", + "shard": 3, + "env": { + "CHROMIUM_REQUIRE": "1", + "BASE_URL": "$BASE_URL" + }, + "output": "e2e", + "meanSeconds": 1.4789999999999999, + "secondsByExecution": [ + 1.098, + 1.713, + 1.626 + ] + } + ], + "exclusions": [ + { + "file": "test-rx-coverage-mobile-nav-e2e.js", + "reason": "skips when clientRxCoverage is disabled, the default configuration." + } + ], + "metadata": { + "repository": "dborup/CoreScope", + "sourceWorkflowSha256": "c7591283434035556a480d98206c2db35bde0a05e865e0fc9c8b31a76fd63fa0", + "baselineHeadSha": "529e008c941f86c1901e21a62da85173e739200d", + "measuredExecutions": [ + { + "runId": 38410864651, + "attempt": 1, + "headSha": "529e008c941f86c1901e21a62da85173e739200d" + }, + { + "runId": 38410864651, + "attempt": 2, + "headSha": "529e008c941f86c1901e21a62da85173e739200d" + }, + { + "runId": 38410864651, + "attempt": 3, + "headSha": "529e008c941f86c1901e21a62da85173e739200d" + } + ], + "algorithm": "Longest processing time first; tie by suite filename then shard id; execution order retained from baseline", + "permanentSlideOverGateMeanSeconds": 45.494, + "predictionLimitation": "Permanent slide-over repeat gate weight is bundled with its base suite; fixture/setup/coverage overhead is not included in suite prediction.", + "shardPredictions": [ + { + "id": 1, + "port": 13581, + "predictedSuiteSeconds": 505.9773333333333, + "slideOverRepeats": 0, + "collectFrontendCoverage": false, + "suiteCount": 39 + }, + { + "id": 2, + "port": 13582, + "predictedSuiteSeconds": 506.17066666666665, + "slideOverRepeats": 0, + "collectFrontendCoverage": true, + "suiteCount": 58 + }, + { + "id": 3, + "port": 13583, + "predictedSuiteSeconds": 505.98266666666655, + "slideOverRepeats": 3, + "collectFrontendCoverage": false, + "suiteCount": 58 + } + ] + } +} diff --git a/scripts/report-e2e-shards.js b/scripts/report-e2e-shards.js new file mode 100644 index 000000000..5c6f16983 --- /dev/null +++ b/scripts/report-e2e-shards.js @@ -0,0 +1,91 @@ +#!/usr/bin/env node +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { SHARDS, CORE_SUITE, readManifest, expectedRuns, runContext } = require('./ci/e2e-manifest'); + +function requireFile(file) { + assert.ok(fs.lstatSync(file).isFile(), `missing or non-regular shard artifact: ${file}`); + return fs.readFileSync(file, 'utf8'); +} + +function aggregateShards({ root, manifestFile, artifactsDir, outputDir = root, env = process.env }) { + const { manifest, manifestSha256 } = readManifest(manifestFile, root); + const current = runContext(env); + const receipts = []; + const logs = []; + for (const shard of SHARDS) { + assert.equal(env[`E2E_${shard}_RESULT`], 'success', `current shard ${shard} result must be success`); + const folder = path.join(artifactsDir, `e2e-shard-${shard}`); + const receipt = JSON.parse(requireFile(path.join(folder, 'result.json'))); + assert.equal(receipt.version, 1, `unsupported receipt version: shard ${shard}`); + assert.equal(receipt.shard, shard, 'artifact assigned to wrong shard'); + assert.equal(receipt.manifestSha256, manifestSha256, `stale manifest receipt: shard ${shard}`); + assert.deepEqual({ runId: receipt.runContext?.runId, headSha: receipt.runContext?.headSha }, + { runId: current.runId, headSha: current.headSha }, `receipt from a different CI run or head: shard ${shard}`); + if (current.attempt === null) { + assert.equal(receipt.runContext.attempt, null, 'local receipt unexpectedly has a CI attempt'); + } else { + const previous = receipt.runContext.attempt; + assert.ok(/^[1-9][0-9]*$/.test(current.attempt) && /^[1-9][0-9]*$/.test(previous), 'invalid receipt attempt'); + assert.ok(Number.isSafeInteger(Number(current.attempt)) && Number.isSafeInteger(Number(previous)), 'unsafe receipt attempt'); + // --failed reruns carry over already successful jobs. Their older + // receipts remain valid for the same run/head/manifest, but never for + // a newer attempt. The actual current needs.result was checked above. + assert.ok(Number(previous) <= Number(current.attempt), `receipt from a future attempt: shard ${shard}`); + } + assert.equal(receipt.status, 'success', `shard ${shard} did not succeed`); + assert.ok(Number.isFinite(Date.parse(receipt.startedAt)) && Number.isFinite(Date.parse(receipt.completedAt)), 'invalid receipt timestamps'); + assert.ok(Date.parse(receipt.completedAt) >= Date.parse(receipt.startedAt), 'receipt end precedes start'); + assert.ok(Array.isArray(receipt.runs), `missing run receipt: shard ${shard}`); + const expected = expectedRuns(manifest, shard); + assert.equal(receipt.runs.length, expected.length, `missing or extra suite execution: shard ${shard}`); + for (const [index, run] of receipt.runs.entries()) { + assert.deepEqual({ file: run.file, kind: run.kind, iteration: run.iteration }, expected[index], `incorrect suite execution: shard ${shard}, index ${index}`); + assert.equal(run.code, 0, `failed child receipt: ${run.file}`); + assert.equal(run.signal, null, `signalled child receipt: ${run.file}`); + assert.ok(Number.isFinite(run.durationMs) && run.durationMs >= 0, `invalid duration: ${run.file}`); + } + logs.push(requireFile(path.join(folder, 'e2e-output.txt'))); + requireFile(path.join(folder, 'node-liveness-e2e-output.txt')); + requireFile(path.join(folder, 'slideover-repeat-output.txt')); + receipts.push(receipt); + } + const coreShard = manifest.suites.find(suite => suite.file === CORE_SUITE).shard; + const coverageSource = path.join(artifactsDir, `e2e-shard-${coreShard}`, 'coverage'); + fs.mkdirSync(outputDir, { recursive: true }); + fs.writeFileSync(path.join(outputDir, 'e2e-output.txt'), logs.join('\n')); + if (fs.existsSync(coverageSource)) { + const coverageTarget = path.join(outputDir, '.nyc_output'); + fs.mkdirSync(coverageTarget, { recursive: true }); + for (const file of fs.readdirSync(coverageSource)) { + assert.ok(/^[a-zA-Z0-9_-]+\.json$/.test(file), `unexpected coverage artifact: ${file}`); + const raw = requireFile(path.join(coverageSource, file)); + JSON.parse(raw); + fs.writeFileSync(path.join(coverageTarget, file), raw); + } + } + const report = { + version: 1, manifestSha256, status: 'success', coreShard, + runContext: runContext(env), + suiteCount: manifest.suites.length, + shardDurationsMs: receipts.map(receipt => receipt.runs.reduce((total, run) => total + run.durationMs, 0)), + receipts, + }; + fs.writeFileSync(path.join(outputDir, 'e2e-shards-report.json'), JSON.stringify(report, null, 2) + '\n'); + return report; +} + +if (require.main === module) { + try { + const root = process.cwd(); + const report = aggregateShards({ + root, manifestFile: process.env.E2E_MANIFEST || path.join(root, 'scripts/e2e-shards.json'), + artifactsDir: process.env.E2E_ARTIFACTS || path.join(root, '.ci-e2e', 'downloaded'), + }); + console.log(`Verified ${report.suiteCount} suites and every repeat across three successful shards`); + } catch (error) { console.error(error.message); process.exitCode = 1; } +} +module.exports = { aggregateShards }; diff --git a/scripts/run-e2e-shard.js b/scripts/run-e2e-shard.js new file mode 100644 index 000000000..892085e4b --- /dev/null +++ b/scripts/run-e2e-shard.js @@ -0,0 +1,124 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const { performance } = require('node:perf_hooks'); +const { SHARDS, CORE_SUITE, readManifest, expectedRuns, runContext } = require('./ci/e2e-manifest'); + +function spawnLogged(command, args, options) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd: options.root, env: options.env, stdio: ['ignore', 'pipe', 'pipe'] }); + const write = chunk => { + fs.appendFileSync(options.log, chunk); + process.stdout.write(chunk); + }; + child.stdout.on('data', write); + child.stderr.on('data', write); + let interruption = null; + const forward = signal => { + interruption = signal; + child.kill(signal); + }; + const onInt = () => forward('SIGINT'); + const onTerm = () => forward('SIGTERM'); + process.once('SIGINT', onInt); + process.once('SIGTERM', onTerm); + child.once('error', reject); + child.once('close', (code, signal) => { + process.removeListener('SIGINT', onInt); + process.removeListener('SIGTERM', onTerm); + // A child may gracefully handle TERM/INT and exit zero. The parent + // nevertheless was cancelled and must never record successful work. + resolve({ code: interruption ? 1 : code, signal: interruption || signal }); + }); + }); +} + +function suiteEnv(suite, baseEnv, leafletDir) { + const env = { ...baseEnv }; + for (const [key, value] of Object.entries(suite.env)) { + env[key] = value === '$BASE_URL' ? baseEnv.BASE_URL : value === '$LEAFLET_DIR' ? leafletDir : value; + assert.ok(env[key] !== undefined, `missing environment expansion for ${key}`); + } + return env; +} + +async function runShard(options) { + const { root, manifestFile, shard } = options; + assert.ok(SHARDS.includes(shard), `invalid shard: ${shard}`); + const { manifest, manifestSha256 } = readManifest(manifestFile, root); + const baseEnv = { ...process.env, ...options.env }; + const expectedPort = String(13580 + shard); + assert.equal(baseEnv.E2E_PORT, expectedPort, 'shard server port must be distinct and pinned'); + assert.equal(baseEnv.BASE_URL, `http://localhost:${expectedPort}`, 'shard BASE_URL must match its server'); + const outputDir = options.outputDir || path.join(root, '.ci-e2e', `shard-${shard}`); + fs.mkdirSync(outputDir, { recursive: true }); + const output = path.join(outputDir, 'e2e-output.txt'); + const liveness = path.join(outputDir, 'node-liveness-e2e-output.txt'); + const repeat = path.join(outputDir, 'slideover-repeat-output.txt'); + for (const log of [output, liveness, repeat]) fs.writeFileSync(log, ''); + const receipt = { + version: 1, shard, manifestSha256, status: 'failure', + runContext: runContext(baseEnv), + startedAt: new Date().toISOString(), completedAt: null, runs: [], + }; + const runProcess = options.runProcess || spawnLogged; + const suites = manifest.suites.filter(suite => suite.shard === shard); + const byFile = new Map(suites.map(suite => [suite.file, suite])); + try { + for (const expected of expectedRuns(manifest, shard)) { + const suite = byFile.get(expected.file); + const label = expected.kind === 'repeat-gate' ? `${suite.file} (repeat-gate)` : suite.file; + const firstRepeat = expected.kind === 'repeat-gate' && expected.iteration === 1; + if (expected.kind !== 'repeat-gate' || firstRepeat) console.log(`=== E2E SUITE: ${label} ===`); + if (expected.kind === 'repeat-gate') console.log(`--- slide-over E2E run ${expected.iteration}/3 ---`); + const log = expected.kind === 'repeat-gate' ? repeat : suite.output === 'node-liveness' ? liveness : output; + const begin = performance.now(); + let leafletDir; + if (suite.setup && expected.kind === 'suite') { + assert.ok(baseEnv.RUNNER_TEMP, 'RUNNER_TEMP is required for #315 assets'); + leafletDir = path.join(baseEnv.RUNNER_TEMP, 'corescope-315-leaflet'); + const setup = await runProcess('bash', ['scripts/ci/prepare-e2e-leaflet.sh'], { root, env: baseEnv, log }); + assert.equal(setup.signal, null, '#315 setup terminated by a signal'); + assert.equal(setup.code, 0, '#315 asset preparation failed'); + } + const result = await runProcess(process.execPath, [suite.file], { root, env: suiteEnv(suite, baseEnv, leafletDir), log }); + const measured = { ...expected, code: result.code, signal: result.signal, durationMs: Math.round((performance.now() - begin) * 1000) / 1000 }; + receipt.runs.push(measured); + assert.equal(result.signal, null, `${suite.file} terminated by ${result.signal}`); + assert.equal(result.code, 0, `${suite.file} failed with exit ${result.code}`); + if (expected.kind !== 'repeat-gate' || expected.iteration === suite.repeatGate) console.log(`=== E2E SUITE END: ${label} ===`); + } + receipt.status = 'success'; + return receipt; + } catch (error) { + receipt.error = error.message; + throw error; + } finally { + receipt.completedAt = new Date().toISOString(); + fs.writeFileSync(path.join(outputDir, 'result.json'), JSON.stringify(receipt, null, 2) + '\n'); + } +} + +async function main() { + const root = process.cwd(); + const manifestFile = process.env.E2E_MANIFEST || path.join(root, 'scripts/e2e-shards.json'); + if (process.argv[2] === '--validate') { + const { manifest } = readManifest(manifestFile, root); + console.log(`E2E assignments: ${manifest.suites.length} suites across three non-empty shards`); + return; + } + if (process.argv[2] === '--core-shard') { + const { manifest } = readManifest(manifestFile, root); + console.log(manifest.suites.find(suite => suite.file === CORE_SUITE).shard); + return; + } + assert.equal(process.argv.length, 2, 'unexpected arguments'); + await runShard({ root, manifestFile, shard: Number(process.env.E2E_SHARD) }); +} + +if (require.main === module) main().catch(error => { console.error(error.message); process.exitCode = 1; }); +module.exports = { spawnLogged, suiteEnv, runShard }; diff --git a/test-all.sh b/test-all.sh index d39e9dcc3..1f510f1aa 100755 --- a/test-all.sh +++ b/test-all.sh @@ -284,6 +284,7 @@ run test-table-sort.js # test-all.sh self-test (#174) run test-test-all.js run test-ci-parallel-gates.js +run test-ci-shards.js echo "" echo "═══════════════════════════════════════" diff --git a/test-ci-parallel-gates.js b/test-ci-parallel-gates.js index 5b376891f..e870212ed 100644 --- a/test-ci-parallel-gates.js +++ b/test-ci-parallel-gates.js @@ -1,10 +1,7 @@ -// Regression checks for the required CI gate. A skipped required job can be -// treated as passing by GitHub branch protection, so failed dependencies must -// cause a red build-and-publish check rather than skipping it. +// Regression checks for independent shards and the stable fail-closed gate. const assert = require('node:assert/strict'); const fs = require('node:fs'); const { spawnSync } = require('node:child_process'); - const workflow = fs.readFileSync('.github/workflows/deploy.yml', 'utf8'); function job(name) { @@ -12,56 +9,59 @@ function job(name) { const start = workflow.indexOf(marker); assert.notEqual(start, -1, `missing ${name} job`); const remainder = workflow.slice(start + marker.length); - const next = remainder.search(/\n [a-z][a-z-]*:\n/); + const next = remainder.search(/\n [a-z][a-z0-9-]*:\n/); return next < 0 ? remainder : remainder.slice(0, next); } const go = job('go-test'); -const e2e = job('e2e-test'); +const shards = [1, 2, 3].map(number => job(`e2e-test-${number}`)); +const report = job('e2e-report'); const image = job('image-check'); const publish = job('build-and-publish'); - -assert.ok(!/^ needs:/m.test(go), 'Go tests must start independently'); -assert.ok(!/^ needs:/m.test(e2e), 'E2E must not wait for Go tests'); -assert.ok(!/^ needs:/m.test(image), 'image check must start independently'); -assert.match(image, /docker compose .* build/, 'image check must build the staging image'); -assert.match(publish, /^ needs: \[go-test, e2e-test, image-check\]$/m); -// A cancelled run reports the gate as cancelled (not skipped), which does -// not pass, so !cancelled() is enough. -assert.match(publish, /^ if: \$\{\{ !cancelled\(\) \}\}$/m, 'gate must run after failed or skipped prerequisites'); +for (const [name, body] of [['Go', go], ['image', image], ...shards.map((body, index) => [`E2E shard ${index + 1}`, body])]) { + assert.ok(!/^ needs:/m.test(body), `${name} must start independently`); +} +for (const [index, shard] of shards.entries()) { + assert.match(shard, new RegExp(`^ E2E_SHARD: '${index + 1}'$`, 'm')); + assert.match(shard, new RegExp(`^ E2E_PORT: '${13581 + index}'$`, 'm')); + assert.match(shard, new RegExp(`^ BASE_URL: http://localhost:${13581 + index}$`, 'm')); +} +assert.match(shards[0], /^ steps: &e2e-shard-steps$/m); +for (const shard of shards.slice(1)) assert.match(shard, /^ steps: \*e2e-shard-steps$/m); +assert.match(shards[0], /node scripts\/run-e2e-shard\.js --validate/); +assert.match(shards[0], /node scripts\/run-e2e-shard\.js\n/); +assert.match(shards[0], /name: e2e-shard-\$\{\{ env\.E2E_SHARD \}\}\n overwrite: true\n/); +assert.match(shards[0], /github\.event_name == 'push' && steps\.shard-meta\.outputs\.core_shard == env\.E2E_SHARD/); +assert.match(report, /^ needs: \[e2e-test-1, e2e-test-2, e2e-test-3\]$/m); +assert.match(report, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); +assert.match(report, /node scripts\/report-e2e-shards\.js/); +assert.match(report, /name: e2e-badges\n overwrite: true\n/); +assert.match(image, /docker compose .* build/); +assert.match(publish, /^ name: "🏗️ Build & Publish Docker Image"$/m); +assert.match(publish, /^ needs: \[go-test, e2e-test-1, e2e-test-2, e2e-test-3, e2e-report, image-check\]$/m); +assert.match(publish, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); assert.match(publish, /github\.repository == 'Kpa-clawbot\/CoreScope'/); +assert.doesNotMatch(publish, /^ if:.*\b(always|cancelled|failure)\(\)/m); -// The gate job runs after failures, so only the implicit success() of each -// later step keeps publishing behind the gate step. No step may override it. -assert.doesNotMatch(publish, /^ if:.*\b(always|cancelled|failure)\(\)/m, 'publish steps must stay behind the gate step'); - -const gate = publish.match(/ - name: Require tests and image validation\n([\s\S]*?)(?= - name: Checkout code)/); -assert.ok(gate, 'stable required check needs an explicit gate'); -const script = gate[1].match(/^ run: \|\n((?:^ .*\n)+)/m); +const required = publish.match(/ - name: Require tests and image validation\n([\s\S]*?)(?= - name: Checkout code)/); +assert.ok(required, 'stable gate needs its explicit prerequisite check'); +const script = required[1].match(/^ run: \|\n((?:^ .*\n)+)/m); assert.ok(script, 'missing gate script'); const bash = script[1].replace(/^ /gm, ''); - -for (const [goResult, e2eResult, imageResult, expected] of [ - ['success', 'success', 'success', 0], - ['failure', 'success', 'success', 1], - ['success', 'failure', 'success', 1], - ['success', 'success', 'failure', 1], - ['skipped', 'success', 'success', 1], - ['success', 'cancelled', 'success', 1], - ['success', 'success', 'skipped', 1], - ['cancelled', 'success', 'success', 1], - ['success', 'success', 'cancelled', 1], - ['', 'success', 'success', 1], -]) { - const result = spawnSync('bash', ['-e', '-c', bash], { - env: { ...process.env, GO_RESULT: goResult, E2E_RESULT: e2eResult, IMAGE_RESULT: imageResult }, - encoding: 'utf8', - }); - assert.equal(result.status, expected, `${goResult}/${e2eResult}/${imageResult}: ${result.stderr}`); +const keys = ['GO_RESULT', 'E2E_1_RESULT', 'E2E_2_RESULT', 'E2E_3_RESULT', 'E2E_REPORT_RESULT', 'IMAGE_RESULT']; +const expectedBindings = ['go-test', 'e2e-test-1', 'e2e-test-2', 'e2e-test-3', 'e2e-report', 'image-check']; +for (const [index, key] of keys.entries()) { + assert.ok(required[1].includes(`${key}: \${{ needs.${expectedBindings[index]}.result }}`), `missing needs binding for ${key}`); +} +const positive = Object.fromEntries(keys.map(key => [key, 'success'])); +const cases = [{ values: positive, expected: 0 }]; +for (const key of keys) for (const value of ['failure', 'skipped', 'cancelled', '']) { + cases.push({ values: { ...positive, [key]: value }, expected: 1 }); +} +for (const { values, expected } of cases) { + const result = spawnSync('bash', ['-e', '-c', bash], { env: { ...process.env, ...values }, encoding: 'utf8' }); + assert.equal(result.status, expected, `${JSON.stringify(values)}: ${result.stderr}`); } - -console.log('CI parallelism and fail-closed gates: PASS'); - // Follow-ups to #383 (#426). // 1. Registry login/push steps must be fork-guarded in EVERY job, not only // build-and-publish (job-level or step-level guard). @@ -86,3 +86,4 @@ assert.match(image, /^ permissions:\n contents: read\n/m, 'image-check m assert.doesNotMatch(image, /packages:\s*write/); const timeout = image.match(/^ timeout-minutes: (\d+)$/m); assert.ok(timeout && +timeout[1] >= 10 && +timeout[1] <= 30, 'image-check needs timeout-minutes of about 20'); +console.log('CI independent shards and all-six fail-closed prerequisites: PASS'); diff --git a/test-ci-shards.js b/test-ci-shards.js new file mode 100644 index 000000000..5c149564f --- /dev/null +++ b/test-ci-shards.js @@ -0,0 +1,239 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const { spawn, spawnSync } = require('node:child_process'); +const { test } = require('node:test'); +const { validateManifest, expectedRuns } = require('./scripts/ci/e2e-manifest'); +const { aggregateShards } = require('./scripts/report-e2e-shards'); + +function successfulResults(extra = {}) { + return { ...process.env, E2E_1_RESULT: 'success', E2E_2_RESULT: 'success', E2E_3_RESULT: 'success', ...extra }; +} + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ci434-shard-test-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.writeFileSync(path.join(root, 'test-all.sh'), '#!/bin/sh\n'); + const scripts = { + 'test-node-liveness-e2e.js': "console.log('999 passed, 0 failed');\n", + 'test-e2e-playwright.js': "console.log('3 passed, 0 failed');\n", + 'test-slideover-1056-e2e.js': "console.log('2 passed, 0 failed');\n", + 'test-probe-e2e.js': 'console.log(process.env.' + "BASE_URL + '|' + process.env.CHROMIUM_REQUIRE);console.log('1 passed, 0 failed');\n", + }; + for (const [file, source] of Object.entries(scripts)) fs.writeFileSync(path.join(root, file), source); + const manifest = { + version: 1, exclusions: [], suites: [ + { file: 'test-node-liveness-e2e.js', shard: 1, env: {}, output: 'node-liveness' }, + { file: 'test-e2e-playwright.js', shard: 2, env: { BASE_URL: '$BASE_URL' }, output: 'e2e' }, + { file: 'test-probe-e2e.js', shard: 2, env: { BASE_URL: '$BASE_URL', CHROMIUM_REQUIRE: '1' }, output: 'e2e' }, + { file: 'test-slideover-1056-e2e.js', shard: 3, env: { BASE_URL: '$BASE_URL' }, output: 'e2e', repeatGate: 3 }, + ], + }; + const manifestFile = path.join(root, 'e2e-shards.json'); + function save() { fs.writeFileSync(manifestFile, JSON.stringify(manifest, null, 2) + '\n'); } + save(); + function run(shard) { + return spawnSync(process.execPath, [path.join(__dirname, 'scripts/run-e2e-shard.js')], { + cwd: root, encoding: 'utf8', timeout: 15000, + env: { ...process.env, E2E_MANIFEST: manifestFile, E2E_SHARD: String(shard), E2E_PORT: String(13580 + shard), BASE_URL: `http://localhost:${13580 + shard}` }, + }); + } + function receipt(shard) { return JSON.parse(fs.readFileSync(path.join(root, '.ci-e2e', `shard-${shard}`, 'result.json'), 'utf8')); } + function artifacts() { + const folder = path.join(root, 'artifacts'); + fs.mkdirSync(folder, { recursive: true }); + for (const shard of [1, 2, 3]) fs.cpSync(path.join(root, '.ci-e2e', `shard-${shard}`), path.join(folder, `e2e-shard-${shard}`), { recursive: true }); + return folder; + } + return { root, manifest, manifestFile, run, receipt, save, artifacts }; +} + +test('guard validates current assignments and rejects newly added browser files', t => { + const f = fixture(t); + assert.equal(validateManifest(f.manifest, f.root), f.manifest); + // Mutant: a new conventionally named E2E has no shard assignment. + fs.writeFileSync(path.join(f.root, 'test-new_unassigned-e2e.js'), 'console.log("not run");\n'); + assert.throws(() => validateManifest(f.manifest, f.root), /unassigned E2E suite: test-new_unassigned-e2e\.js/); + fs.unlinkSync(path.join(f.root, 'test-new_unassigned-e2e.js')); + // Non-e2e names which import Playwright must not fall outside discovery. + fs.writeFileSync(path.join(f.root, 'test-new-browser.js'), 'const { chromium } = requ' + "ire('playwright');\n"); + assert.throws(() => validateManifest(f.manifest, f.root), /unassigned E2E suite: test-new-browser\.js/); + fs.unlinkSync(path.join(f.root, 'test-new-browser.js')); + fs.writeFileSync(path.join(f.root, 'test-new-import.js'), 'const browser = await imp' + "ort('playwright');\n"); + assert.throws(() => validateManifest(f.manifest, f.root), /unassigned E2E suite: test-new-import\.js/); +}); + +test('guard rejects duplicate assignments, missing assignments, invalid shards and traversal', t => { + const f = fixture(t); + for (const change of [ + manifest => manifest.suites.push({ ...manifest.suites[2], shard: 1 }), + manifest => { manifest.suites = manifest.suites.filter(suite => suite.file !== 'test-probe-e2e.js'); }, + manifest => { manifest.suites[2].shard = 4; }, + manifest => { manifest.suites[2].file = '../test-probe-e2e.js'; }, + manifest => { manifest.suites[2].file = '/tmp/test-probe-e2e.js'; }, + manifest => { manifest.suites[3].repeatGate = 2; }, + manifest => { manifest.suites[2].output = 'node-liveness'; }, + manifest => { manifest.suites[2].env.BASE_URL = '$(touch forbidden)'; }, + manifest => manifest.exclusions.push({ file: 'test-probe-e2e.js', reason: 'a new file must not receive an ad-hoc exclusion' }), + ]) { + const mutated = structuredClone(f.manifest); + change(mutated); + assert.throws(() => validateManifest(mutated, f.root)); + } +}); + +test('unit registry cannot hide newly added browser imports or E2E filenames', t => { + const f = fixture(t); + fs.writeFileSync(path.join(f.root, 'test-new-browser.js'), 'const { chromium } = requ' + "ire('playwright');\n"); + fs.writeFileSync(path.join(f.root, 'test-all.sh'), '#!/bin/sh\nrun test-new-browser.js\n'); + assert.throws(() => validateManifest(f.manifest, f.root), /unassigned E2E suite: test-new-browser.js/); + fs.unlinkSync(path.join(f.root, 'test-new-browser.js')); + fs.writeFileSync(path.join(f.root, 'test-added-e2e.js'), 'console.log("browser suite");\n'); + fs.appendFileSync(path.join(f.root, 'test-all.sh'), 'run test-added-e2e.js\n'); + assert.throws(() => validateManifest(f.manifest, f.root), /unassigned E2E suite: test-added-e2e.js/); +}); + +test('manifest cannot override the pinned local BASE_URL', t => { + const f = fixture(t); + for (const target of ['https://staging.invalid', 'https://production.invalid', 'http://localhost:13581']) { + const manifest = structuredClone(f.manifest); + manifest.suites[2].env.BASE_URL = target; + assert.throws(() => validateManifest(manifest, f.root), /BASE_URL/); + } +}); + +test('parent interruption remains failure when the child handles the signal and exits zero', async t => { + for (const signal of ['SIGTERM', 'SIGINT']) { + const f = fixture(t); + fs.writeFileSync(path.join(f.root, 'test-node-liveness-e2e.js'), + `process.on('${signal}', () => process.exit(0));console.log('CHILD_READY');setTimeout(() => process.exit(2), 5000);\n`); + const child = spawn(process.execPath, [path.join(__dirname, 'scripts/run-e2e-shard.js')], { + cwd: f.root, stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, E2E_MANIFEST: f.manifestFile, E2E_SHARD: '1', E2E_PORT: '13581', BASE_URL: 'http://localhost:13581' }, + }); + t.after(() => { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); }); + const exit = new Promise((resolve, reject) => { child.once('error', reject); child.once('close', (code, signal) => resolve({ code, signal })); }); + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('child readiness timed out')), 4000); + let output = ''; + child.stdout.on('data', chunk => { + output += chunk; + if (output.includes('CHILD_READY')) { clearTimeout(timeout); resolve(); } + }); + child.once('error', reject); + }); + assert.ok(child.kill(signal), 'signal only the spawned runner after its child is ready'); + const result = await exit; + assert.notEqual(result.code, 0, 'cancelled runner must not report success after a graceful child exit'); + const receipt = f.receipt(1); + assert.equal(receipt.status, 'failure'); + assert.equal(receipt.runs[0].signal, signal); + } +}); + +test('runner executes real child processes, preserves env, repeats only on owning shard, and records every run', t => { + const f = fixture(t); + for (const shard of [1, 2, 3]) { + const result = f.run(shard); + assert.equal(result.status, 0, result.stderr); + const receipt = f.receipt(shard); + assert.equal(receipt.status, 'success'); + assert.deepEqual(receipt.runs.map(({ file, kind, iteration }) => ({ file, kind, iteration })), expectedRuns(f.manifest, shard)); + assert.ok(receipt.runs.every(run => run.code === 0 && run.signal === null && Number.isFinite(run.durationMs))); + } + assert.match(fs.readFileSync(path.join(f.root, '.ci-e2e/shard-2/e2e-output.txt'), 'utf8'), /http:\/\/localhost:13582\|1/); + assert.equal(f.receipt(3).runs.length, 4, 'one ordinary and three repeat executions'); + const report = aggregateShards({ root: f.root, manifestFile: f.manifestFile, artifactsDir: f.artifacts(), outputDir: path.join(f.root, 'report'), env: successfulResults() }); + assert.equal(report.status, 'success'); + const combined = fs.readFileSync(path.join(f.root, 'report/e2e-output.txt'), 'utf8'); + assert.doesNotMatch(combined, /999 passed/, 'node-liveness remains excluded from original badge summary'); + assert.equal((combined.match(/2 passed/g) || []).length, 1, 'repeat gate remains excluded from original badge summary'); +}); + +test('real child non-zero exit and signal fail the shard without running later suites', t => { + for (const source of ['process.exit(23);\n', "process.kill(process.pid, 'SIGTERM');\n"]) { + const f = fixture(t); + fs.writeFileSync(path.join(f.root, 'test-e2e-playwright.js'), source); + fs.writeFileSync(path.join(f.root, 'test-probe-e2e.js'), "require('node:fs').writeFileSync('incorrectly-ran', 'yes');\n"); + const result = f.run(2); + assert.notEqual(result.status, 0); + assert.ok(!fs.existsSync(path.join(f.root, 'incorrectly-ran')), 'fail-fast must prevent next suite'); + const receipt = f.receipt(2); + assert.equal(receipt.status, 'failure'); + assert.equal(receipt.runs.length, 1); + assert.ok(receipt.runs[0].code !== 0 || receipt.runs[0].signal !== null); + } +}); + +test('report is fail-closed for every shard status, missing artifacts, missing execution, stale manifest and forged success', t => { + const f = fixture(t); + for (const shard of [1, 2, 3]) assert.equal(f.run(shard).status, 0); + const artifactsDir = f.artifacts(); + const options = { root: f.root, manifestFile: f.manifestFile, artifactsDir, outputDir: path.join(f.root, 'report'), env: successfulResults() }; + assert.equal(aggregateShards(options).status, 'success'); + for (const shard of [1, 2, 3]) { + for (const result of ['failure', 'skipped', 'cancelled', '']) { + assert.throws(() => aggregateShards({ ...options, env: { ...options.env, [`E2E_${shard}_RESULT`]: result } }), /current shard.*result must be success/); + } + const file = path.join(artifactsDir, `e2e-shard-${shard}`, 'result.json'); + const original = fs.readFileSync(file, 'utf8'); + for (const status of ['failure', 'skipped', 'cancelled', '']) { + const receipt = JSON.parse(original); + receipt.status = status; + fs.writeFileSync(file, JSON.stringify(receipt)); + assert.throws(() => aggregateShards(options), /did not succeed/); + } + for (const mutate of [ + receipt => receipt.runs.pop(), + receipt => { receipt.runs[0].code = 1; }, + receipt => { receipt.runs[0].signal = 'SIGTERM'; }, + receipt => { receipt.manifestSha256 = '0'.repeat(64); }, + receipt => { receipt.runContext.runId = 'another-run'; }, + receipt => { receipt.shard = 4; }, + receipt => { receipt.runs[0].file = 'test-other-e2e.js'; }, + receipt => { receipt.runs[0].durationMs = -1; }, + ]) { + const receipt = JSON.parse(original); + mutate(receipt); + fs.writeFileSync(file, JSON.stringify(receipt)); + assert.throws(() => aggregateShards(options)); + } + fs.unlinkSync(file); + assert.throws(() => aggregateShards(options)); + fs.writeFileSync(file, original); + } + const missingLog = path.join(artifactsDir, 'e2e-shard-3/e2e-output.txt'); + fs.unlinkSync(missingLog); + assert.throws(() => aggregateShards(options)); +}); + +test('valid earlier-attempt successes carry over, but wrong run/head or future attempts cannot', t => { + const f = fixture(t); + for (const shard of [1, 2, 3]) assert.equal(f.run(shard).status, 0); + const artifactsDir = f.artifacts(); + const options = { root: f.root, manifestFile: f.manifestFile, artifactsDir, outputDir: path.join(f.root, 'report'), env: successfulResults({ GITHUB_RUN_ID: '123', GITHUB_RUN_ATTEMPT: '2', GITHUB_SHA: 'abc' }) }; + for (const shard of [1, 2, 3]) { + const file = path.join(artifactsDir, `e2e-shard-${shard}/result.json`); + const receipt = JSON.parse(fs.readFileSync(file, 'utf8')); + receipt.runContext = { runId: '123', attempt: shard === 2 ? '2' : '1', headSha: 'abc' }; + fs.writeFileSync(file, JSON.stringify(receipt)); + } + assert.equal(aggregateShards(options).status, 'success'); + const file = path.join(artifactsDir, 'e2e-shard-1/result.json'); + const original = fs.readFileSync(file, 'utf8'); + for (const mutation of [ + receipt => { receipt.runContext.attempt = '3'; }, + receipt => { receipt.runContext.attempt = '0'; }, + receipt => { receipt.runContext.attempt = 'not-a-number'; }, + receipt => { receipt.runContext.headSha = 'different'; }, + receipt => { receipt.runContext.runId = 'different'; }, + ]) { + const receipt = JSON.parse(original); + mutation(receipt); + fs.writeFileSync(file, JSON.stringify(receipt)); + assert.throws(() => aggregateShards(options)); + } +}); diff --git a/test-test-all.js b/test-test-all.js index 28bda4ae8..b09716468 100644 --- a/test-test-all.js +++ b/test-test-all.js @@ -135,13 +135,16 @@ const KNOWN_UNREGISTERED = { console.log('root test registry'); const deploy = fs.readFileSync(path.join(ROOT, '.github/workflows/deploy.yml'), 'utf8'); const inDeploy = new Set([...deploy.matchAll(/\bnode (test-[\w.-]+\.js)\b/g)].map((m) => m[1])); -const runSomewhere = (f) => files.includes(f) || inDeploy.has(f); +const { manifest: e2eManifest } = require('./scripts/ci/e2e-manifest').readManifest( + path.join(ROOT, 'scripts/e2e-shards.json'), ROOT); +const inE2EManifest = new Set(e2eManifest.suites.map((suite) => suite.file)); +const runSomewhere = (f) => files.includes(f) || inDeploy.has(f) || inE2EManifest.has(f); const rootTests = fs.readdirSync(ROOT).filter((f) => /^test-.*\.js$/.test(f)); -test('every root test-*.js runs in test-all.sh or deploy.yml', () => { +test('every root test-*.js runs in test-all.sh or the CI E2E manifest', () => { const orphans = rootTests.filter((f) => !runSomewhere(f) && !(f in KNOWN_UNREGISTERED)); assert.deepStrictEqual(orphans, [], orphans.join(', ') + ' run nowhere: add `run ` to ' + - 'test-all.sh (unit) or a line to the Playwright step in deploy.yml (E2E)'); + 'test-all.sh (unit) or an assignment in scripts/e2e-shards.json (E2E)'); }); test('KNOWN_UNREGISTERED has no stale entries', () => {