diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 312790e5..0841f731 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,9 +10,14 @@ on: # workflow on the branch explicitly instead, so the fixed commit is checked. workflow_dispatch: +# Pull requests run without production credentials. Forks receive only a +# read-only workflow token; neither collection nor test lanes may read secrets. jobs: - check: + prepare: runs-on: ubuntu-latest + outputs: + corrected: ${{ steps.autofix.outputs.changed }} + revision: ${{ steps.revision.outputs.sha }} permissions: contents: write actions: write @@ -40,9 +45,10 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - # Check out the branch itself (not the merge ref) so a fix can be - # committed back to it. - ref: ${{ github.head_ref || github.ref_name }} + # Test the event's exact revision; fixes push explicitly to its branch. + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - id: revision + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true @@ -68,6 +74,8 @@ jobs: JOBTRACKER_SERVICE_TOKEN: ci APP_ENCRYPTION_KEY: ci GH_TOKEN: ${{ github.token }} + HEAD_BRANCH: ${{ github.head_ref }} + WORKFLOW_NAME: ${{ github.workflow }} run: | ruff check --fix src tests || true ruff format src tests @@ -91,8 +99,8 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" git add src tests openapi.json git commit -m "chore: auto-fix lint, format and schema" - git push - gh workflow run "${{ github.workflow }}" --ref "${{ github.head_ref || github.ref_name }}" + git push origin "HEAD:$HEAD_BRANCH" + gh workflow run "$WORKFLOW_NAME" --ref "$HEAD_BRANCH" echo "changed=true" >> "$GITHUB_OUTPUT" echo "::notice::Pushed mechanical fixes as a new commit; its own run judges the branch" - name: Lint @@ -110,26 +118,22 @@ jobs: - name: Migrations apply cleanly if: steps.autofix.outputs.changed != 'true' run: python -m alembic upgrade head - # The WHOLE suite, with no marker filter. Three populations run here: - # the hermetic tests on a truncated database, the `corpus` tests against - # the generated catalog in tests/corpus.py - built from the committed - # measurement of production, so it needs no credential - and the - # `integration` tests, which skip because this database is not a copy of - # production and must never be one. - # - # NOTHING IN THIS FILE MAY READ A SECRET. That is the whole point: this - # workflow runs on pull_request, so anything it can reach is reachable - # from a branch an outside contributor can push. The workflow token is - # not a secret in that sense: on a fork's pull request it is read-only. - - name: Run tests + - name: Collect the complete test manifest if: steps.autofix.outputs.changed != 'true' env: TEST_DATABASE_URL: postgresql://ci:ci@localhost:5432/jobtracker_ci JOBTRACKER_SERVICE_TOKEN: ci run: | + export TEST_REVISION=$(git rev-parse HEAD) export APP_ENCRYPTION_KEY=$(python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())") node --test tests/extension/*.test.cjs - pytest -q tests + python -m pytest -p tests.ci_suite --collect-only -q tests --test-report=manifest.json + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: steps.autofix.outputs.changed != 'true' + with: + name: test-manifest + path: manifest.json + if-no-files-found: error - name: Models match migrations (autogen must be empty) if: steps.autofix.outputs.changed != 'true' run: | @@ -138,3 +142,53 @@ jobs: if grep -qE "op\.(add|drop|create|alter)" "$f"; then echo "::error::ORM models drifted from migrations - generate a real migration"; cat "$f"; exit 1 fi + + tests: + needs: prepare + if: needs.prepare.outputs.corrected != 'true' + strategy: + fail-fast: false + matrix: + lane: [shard-0, shard-1, shard-2, corpus] + uses: ./.github/workflows/test-run.yml + with: + lane: ${{ matrix.lane }} + shards: 3 + revision: ${{ needs.prepare.outputs.revision }} + + # Preserve the required check name. Every lane must pass and account for + # the complete collection before this gate can be green. + check: + if: always() + needs: [prepare, tests] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Require successful preparation and execution + env: + PREPARE_RESULT: ${{ needs.prepare.result }} + TEST_RESULT: ${{ needs.tests.result }} + CORRECTED: ${{ needs.prepare.outputs.corrected }} + run: | + test "$PREPARE_RESULT" = success + if [ "$CORRECTED" != true ]; then + test "$TEST_RESULT" = success + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: needs.prepare.outputs.corrected != 'true' + with: + ref: ${{ needs.prepare.outputs.revision }} + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + if: needs.prepare.outputs.corrected != 'true' + with: + name: test-manifest + path: collection + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + if: needs.prepare.outputs.corrected != 'true' + with: + pattern: timings-* + path: measurements + - name: Verify every collected test executed exactly once + if: needs.prepare.outputs.corrected != 'true' + run: python tools/test_performance.py measurements --shards=3 --manifest=collection/manifest.json | tee "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test-performance.yml b/.github/workflows/test-performance.yml new file mode 100644 index 00000000..21fa7c7e --- /dev/null +++ b/.github/workflows/test-performance.yml @@ -0,0 +1,39 @@ +name: Test performance + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + benchmark: + strategy: + fail-fast: false + matrix: + lane: [serial, profile, shard-0, shard-1, shard-2, corpus] + repetition: [1, 2] + exclude: + - lane: profile + repetition: 2 + uses: ./.github/workflows/test-run.yml + with: + lane: ${{ matrix.lane }} + repetition: ${{ matrix.repetition }} + shards: 3 + revision: ${{ github.event.pull_request.head.sha || github.sha }} + + compare: + if: always() + needs: benchmark + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: timings-* + path: measurements + - name: Verify coverage and compare timings + run: python tools/test_performance.py measurements --repetitions=2 --shards=3 | tee "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test-run.yml b/.github/workflows/test-run.yml new file mode 100644 index 00000000..9acccff1 --- /dev/null +++ b/.github/workflows/test-run.yml @@ -0,0 +1,79 @@ +name: Python test lane + +on: + workflow_call: + inputs: + lane: + type: string + required: true + revision: + type: string + required: true + shards: + type: number + required: true + repetition: + type: number + default: 1 + +permissions: + contents: read + +jobs: + run: + name: ${{ inputs.lane }} + runs-on: ubuntu-latest + timeout-minutes: 15 + services: + postgres: + image: pgvector/pgvector:pg18-trixie@sha256:78bf48b801e792f99e3ac62b5036fd3876e9be48afda16c1e331af1c75ceb2ff + env: + POSTGRES_USER: ci + POSTGRES_PASSWORD: ci + POSTGRES_DB: jobtracker_ci + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U ci" --health-interval 5s + --health-timeout 5s --health-retries 10 + env: + DATABASE_URL: postgresql://ci:ci@localhost:5432/jobtracker_ci + TEST_DATABASE_URL: postgresql://ci:ci@localhost:5432/jobtracker_ci + JOBTRACKER_SERVICE_TOKEN: ci + PYTHONPATH: src + TEST_SHARDS: ${{ inputs.shards }} + TEST_LANE: ${{ inputs.lane }} + TEST_REPETITION: ${{ inputs.repetition }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.revision }} + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - run: uv sync --frozen + - run: echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + - run: python -m alembic upgrade head + - name: Measure tests + run: | + mkdir -p results + export TEST_REVISION=$(git rev-parse HEAD) + export APP_ENCRYPTION_KEY=$(python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())") + args=(-p tests.ci_suite -q tests --durations=25 --test-report=results/timing.json --junitxml=results/junit.xml) + case "$TEST_LANE" in + serial|profile) ;; + corpus) args+=(-m corpus) ;; + shard-*) args+=(-m 'not corpus' --shard-count="$TEST_SHARDS" --shard-index="${TEST_LANE#shard-}") ;; + *) echo "Unknown test lane: $TEST_LANE"; exit 1 ;; + esac + if [ "$TEST_LANE" = profile ]; then + python -m cProfile -o results/profile.pstats -m pytest "${args[@]}" + else + python -m pytest "${args[@]}" + fi + - name: Upload measurements + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: timings-${{ inputs.lane }}-${{ inputs.repetition }} + path: results/ + if-no-files-found: error diff --git a/docs/testing-performance.md b/docs/testing-performance.md new file mode 100644 index 00000000..f685a119 --- /dev/null +++ b/docs/testing-performance.md @@ -0,0 +1,83 @@ +# Test performance + +Measured on GitHub-hosted Ubuntu runners on 2026-09-09, revision +`3271201a9fce2757a093d8da2e199e8a17284f97`. +[Experiment and artifacts](https://github.com/kensac/job-scripts/actions/runs/34328307413). + +## Measured comparison + +Each parallel trial used three deterministic shards of tests without the +`corpus` marker and one corpus job, with an independent PostgreSQL database +per job. Both trials matched the serial run's 1,471 test IDs and outcomes +exactly: 1,457 passed and 14 skipped. Subtests remain inside their parent case. + +| Trial | Serial pytest | Parallel pytest critical path | Serial job | Parallel job span | Parallel runner total | +|---|---:|---:|---:|---:|---:| +| 1 | 171.52s | 96.47s | 201s | 135s | 350s | +| 2 | 185.86s | 57.90s | 214s | 90s | 307s | + +Job times include provisioning, dependency installation and artifact upload. +Parallel span is the earliest parallel job start through the last job finish; +it is not the entire CI workflow, which also performs static checks and a +final coverage check. Queue delays before jobs start are excluded. Two trials +support adopting this split, but do not establish a stable speedup: one shard +was considerably slower in trial 1, and the cause was not determined. +Parallel execution consumes more total runner time. + +## Where time goes + +In ordinary, unprofiled serial execution, setup phases consumed 132.81s and +145.02s; test call phases consumed 31.18s and 33.17s. Teardown was about 0.32s. +Four corpus setup phases each took about six seconds. Grouping corpus tests +into their own job also avoids rebuilding it between ordinary tests. + +A separate cProfile run counted four corpus builds and 1,433 `_reseed` calls. +The current `_reseed` implementation inserts two group defaults and 24 +application defaults individually, so those calls account for 37,258 default +insert executions. Profiled cumulative times overlap and instrumentation +substantially increases runtime; they must not be used as ordinary wall times. + +The worker entrypoint test's call took about 8.02 seconds in both serial runs. +Its source waits a fixed eight seconds to detect process fallthrough. + +## Adopted design and next experiments + +Normal CI shares its lane implementation with the manual benchmark. A full +collection manifest and the lane reports must identify the same commit and +account for every case exactly once before the required `check` passes. +Each lane retains the existing database isolation and cleanup fixtures. +Lint, formatting, type checks, compilation, migration checks and extension +tests remain required. Mechanical lint and formatting corrections still run +before the tests. + +Prioritize subsequent work in this order: + +1. Batch default reseeding and measure setup time again. Preserve fresh state + per test, including defaults and sequences; transaction rollback is not a + replacement for tests that commit or use separate connections. +2. Introduce an explicit database-free test population after auditing actual + imports and fixture dependencies. A missing `db` fixture argument alone + cannot establish that a test does not access the database. +3. Replace the worker test's fixed grace period with observable worker-loop + progress and a bounded failure timeout. Process liveness alone does not + prove it reached the loop. +4. Revisit shard balancing only after repeated timing reports show a persistent + imbalance. Keep deterministic assignment and exact coverage verification. + +These follow-up changes are recommendations, not measured implementations. + +## Reproduce + +Dispatch the `Test performance` workflow on a selected revision. It runs two +serial and parallel trials plus one separate call profile. Download its +`timings-*` artifacts, preserving their directories, then run: + +```sh +python tools/test_performance.py measurements --repetitions=2 --shards=3 +``` + +The comparison rejects missing, duplicated or changed test outcomes and +reports from different revisions. Compare job timestamps as well as pytest +phase times before changing job counts. Normal CI also uploads these phase +reports, so later changes can be evaluated without adding a profiler to the +regular test path. diff --git a/tests/ci_suite.py b/tests/ci_suite.py new file mode 100644 index 00000000..fc6fc8eb --- /dev/null +++ b/tests/ci_suite.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import hashlib +import json +import os +import platform +import time +from pathlib import Path + +import pytest + +_STARTED = pytest.StashKey[float]() + + +def pytest_addoption(parser): + group = parser.getgroup("test execution") + group.addoption("--shard-count", type=int, default=1) + group.addoption("--shard-index", type=int, default=0) + group.addoption("--test-report", help="write selected cases and phase timings as JSON") + + +def pytest_load_initial_conftests(early_config): + early_config.stash[_STARTED] = time.perf_counter() + + +def pytest_configure(config): + count, index = config.getoption("shard_count"), config.getoption("shard_index") + if count < 1 or not 0 <= index < count: + raise pytest.UsageError("--shard-index must be between 0 and --shard-count minus one") + output = config.getoption("test_report") + if output: + config.pluginmanager.register(_Report(config, Path(output).resolve()), "test-phase-report") + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config, items): + count, index = config.getoption("shard_count"), config.getoption("shard_index") + if count == 1: + return + selected, deselected = [], [] + for item in items: + # Stable across processes and Python hash seeds. Partition after -m/-k + # selection; each job must still hold its own isolated database. + slot = int.from_bytes(hashlib.sha256(item.nodeid.encode()).digest()[:8]) % count + (selected if slot == index else deselected).append(item) + items[:] = selected + config.hook.pytest_deselected(items=deselected) + + +class _Report: + def __init__(self, config, output): + self.config = config + self.output = output + self.started = config.stash.get(_STARTED, time.perf_counter()) + self.selected = [] + self.reports = {} + + def pytest_collection_finish(self, session): + self.selected = [item.nodeid for item in session.items] + + def pytest_runtest_logreport(self, report): + self.reports.setdefault(report.nodeid, []).append( + { + "phase": report.when, + "seconds": report.duration, + "outcome": report.outcome, + "subtest": hasattr(report, "context"), + } + ) + + def pytest_sessionfinish(self, session, exitstatus): + self.output.parent.mkdir(parents=True, exist_ok=True) + self.output.write_text( + json.dumps( + { + "revision": os.environ.get("TEST_REVISION"), + "lane": os.environ.get("TEST_LANE", "serial"), + "repetition": int(os.environ.get("TEST_REPETITION", "1")), + "python": platform.python_version(), + "shard_count": self.config.getoption("shard_count"), + "shard_index": self.config.getoption("shard_index"), + "pytest_seconds": time.perf_counter() - self.started, + "exitstatus": int(exitstatus), + "selected": self.selected, + "reports": self.reports, + }, + indent=2, + ) + + "\n" + ) diff --git a/tests/test_ci_suite.py b/tests/test_ci_suite.py new file mode 100644 index 00000000..3ab17c69 --- /dev/null +++ b/tests/test_ci_suite.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import json + +import pytest + +pytest_plugins = ["pytester"] + + +def test_shards_partition_selected_tests_without_losing_cases(pytester): + pytester.makeini("[pytest]\nmarkers = corpus: generated database") + pytester.makepyfile( + test_sample=""" + import pytest + @pytest.mark.parametrize('value', range(12)) + def test_ordinary(value): + assert value >= 0 + @pytest.mark.corpus + def test_catalog(): + pass + """ + ) + selected = [] + for index in range(3): + output = pytester.path / f"shard-{index}.json" + result = pytester.runpytest( + "-p", + "tests.ci_suite", + "-m", + "not corpus", + "--shard-count=3", + f"--shard-index={index}", + f"--test-report={output}", + ) + assert result.ret == 0 + report = json.loads(output.read_text()) + selected.extend(report["selected"]) + expected = {f"test_sample.py::test_ordinary[{value}]" for value in range(12)} + assert len(selected) == len(set(selected)) == 12 + assert set(selected) == expected + + +def test_invalid_shard_refuses_to_run(pytester): + pytester.makepyfile("def test_ok(): pass") + result = pytester.runpytest("-p", "tests.ci_suite", "--shard-count=2", "--shard-index=2") + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*--shard-index must be between 0 and*", "*"]) + + +def test_report_preserves_failures_skips_and_phase_durations(pytester): + pytester.makepyfile( + test_sample=""" + import pytest + def test_fails(): + assert False + @pytest.mark.skip(reason='deliberate') + def test_skips(): + pass + """ + ) + output = pytester.path / "timing.json" + result = pytester.runpytest("-p", "tests.ci_suite", f"--test-report={output}") + result.assert_outcomes(failed=1, skipped=1) + report = json.loads(output.read_text()) + assert report["exitstatus"] == 1 + assert set(report["selected"]) == {"test_sample.py::test_fails", "test_sample.py::test_skips"} + failure = report["reports"]["test_sample.py::test_fails"] + assert any(r["phase"] == "call" and r["outcome"] == "failed" for r in failure) + assert {r["phase"] for r in failure} == {"setup", "call", "teardown"} + assert all(r["seconds"] >= 0 for rows in report["reports"].values() for r in rows) + assert report["pytest_seconds"] > 0 + + +@pytest.mark.parametrize( + "defect", ["missing", "duplicate", "outcome", "revision", "failed", "unreported"] +) +def test_comparison_rejects_incomplete_or_changed_coverage(defect): + from tools.test_performance import compare + + def report(lane, selected): + return { + "revision": "same-revision", + "repetition": 1, + "lane": lane, + "exitstatus": 0, + "pytest_seconds": 1, + "selected": selected, + "reports": { + nodeid: [{"phase": "call", "outcome": "passed", "subtest": False}] + for nodeid in selected + }, + } + + baseline = report("serial", ["a", "b"]) + shard = report("shard-0", ["a"]) + corpus = report("corpus", ["b"]) + if defect == "missing": + corpus = report("corpus", []) + elif defect == "duplicate": + corpus = report("corpus", ["a", "b"]) + elif defect == "outcome": + corpus["reports"]["b"][0]["outcome"] = "skipped" + elif defect == "revision": + corpus["revision"] = "different-revision" + elif defect == "failed": + corpus["exitstatus"] = 1 + elif defect == "unreported": + corpus["reports"] = {} + with pytest.raises(ValueError): + compare([baseline, shard, corpus], repetitions=1, shards=1) + + +def test_manifest_requires_complete_same_revision_execution(pytester, monkeypatch): + from tools.test_performance import verify_manifest + + monkeypatch.setenv("TEST_REVISION", "manifest-revision") + monkeypatch.setenv("TEST_LANE", "shard-0") + pytester.makepyfile(test_sample="def test_ok(): pass") + manifest_path = pytester.path / "manifest.json" + execution_path = pytester.path / "timing.json" + assert ( + pytester.runpytest( + "-p", "tests.ci_suite", "--collect-only", f"--test-report={manifest_path}" + ).ret + == 0 + ) + assert pytester.runpytest("-p", "tests.ci_suite", f"--test-report={execution_path}").ret == 0 + manifest = json.loads(manifest_path.read_text()) + execution = json.loads(execution_path.read_text()) + empty_corpus = dict(execution, lane="corpus", selected=[], reports={}) + reports = [execution, empty_corpus] + assert verify_manifest(manifest, reports, shards=1) == {"test_sample.py::test_ok": "passed"} + with pytest.raises(ValueError, match="lanes"): + verify_manifest(manifest, [execution], shards=1) + manifest["revision"] = "stale-revision" + with pytest.raises(ValueError, match="revisions"): + verify_manifest(manifest, reports, shards=1) + manifest["revision"] = execution["revision"] + manifest["selected"].append("missing.py::test_missing") + with pytest.raises(ValueError, match="partition"): + verify_manifest(manifest, reports, shards=1) diff --git a/tools/test_performance.py b/tools/test_performance.py new file mode 100644 index 00000000..e7a603a0 --- /dev/null +++ b/tools/test_performance.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import argparse +import collections +import json +import pstats +import statistics +from pathlib import Path + + +def outcomes(report): + result = {} + for nodeid in report["selected"]: + rows = report["reports"].get(nodeid, []) + main = [row for row in rows if not row["subtest"]] + if any(row["outcome"] == "failed" for row in rows): + result[nodeid] = "failed" + elif any(row["phase"] == "call" and row["outcome"] == "passed" for row in main): + result[nodeid] = "passed" + elif any(row["outcome"] == "skipped" for row in main): + result[nodeid] = "skipped" + else: + raise ValueError(f"No completed outcome for {nodeid}") + return result + + +def index_reports(reports): + revisions = {report["revision"] for report in reports} + if len(revisions) != 1 or None in revisions: + raise ValueError("All reports must identify the same tested revision") + indexed = {} + for report in reports: + key = (report["repetition"], report["lane"]) + if key in indexed: + raise ValueError(f"Duplicate report for {key}") + if report["exitstatus"] != 0: + raise ValueError(f"Test run failed: {key}") + if len(report["selected"]) != len(set(report["selected"])): + raise ValueError(f"Duplicate selected cases in {key}") + if set(report["selected"]) != set(report["reports"]): + raise ValueError(f"Selected and executed cases disagree in {key}") + indexed[key] = report + return indexed + + +def partition(selected, reports): + counts = collections.Counter(nodeid for report in reports for nodeid in report["selected"]) + if counts != collections.Counter(selected): + raise ValueError("Parallel cases must partition the baseline exactly once") + return {nodeid: outcome for report in reports for nodeid, outcome in outcomes(report).items()} + + +def verify_manifest(manifest, reports, *, shards): + indexed = index_reports(reports) + if manifest["exitstatus"] != 0 or not manifest["selected"]: + raise ValueError("A successful nonempty collection manifest is required") + if any(report["revision"] != manifest["revision"] for report in reports): + raise ValueError("Collection and execution revisions differ") + expected = {(1, f"shard-{index}") for index in range(shards)} | {(1, "corpus")} + if set(indexed) != expected: + raise ValueError("All configured lanes must report exactly once") + return partition(manifest["selected"], reports) + + +def compare(reports, *, repetitions, shards): + indexed = index_reports(reports) + comparisons = [] + for repetition in range(1, repetitions + 1): + baseline = indexed[(repetition, "serial")] + parallel = [indexed[(repetition, f"shard-{index}")] for index in range(shards)] + parallel.append(indexed[(repetition, "corpus")]) + parallel_outcomes = partition(baseline["selected"], parallel) + baseline_outcomes = outcomes(baseline) + if baseline_outcomes != parallel_outcomes: + raise ValueError("Serial and parallel outcomes differ") + comparisons.append( + { + "repetition": repetition, + "tests": len(parallel_outcomes), + "outcomes": dict(collections.Counter(baseline_outcomes.values())), + "serial_seconds": baseline["pytest_seconds"], + "parallel_seconds": max(report["pytest_seconds"] for report in parallel), + "parallel_total_seconds": sum(report["pytest_seconds"] for report in parallel), + } + ) + return comparisons + + +def main(): + parser = argparse.ArgumentParser(description="Compare complete, same-revision test runs") + parser.add_argument("directory", type=Path) + parser.add_argument("--repetitions", type=int) + parser.add_argument("--shards", type=int, required=True) + parser.add_argument( + "--manifest", type=Path, help="verify CI execution against complete collection" + ) + args = parser.parse_args() + reports = [json.loads(path.read_text()) for path in args.directory.rglob("timing.json")] + if args.manifest: + result = verify_manifest(json.loads(args.manifest.read_text()), reports, shards=args.shards) + print( + f"Verified {len(result)} cases executed exactly once: {dict(collections.Counter(result.values()))}" + ) + return + if not args.repetitions or args.repetitions < 1: + parser.error("--repetitions must be positive for a benchmark comparison") + comparisons = compare(reports, repetitions=args.repetitions, shards=args.shards) + print("# Test performance comparison\n") + print("All parallel test IDs and outcomes match the serial run exactly.\n") + print("| Repeat | Cases | Serial pytest | Parallel critical path | Parallel total |") + print("|---|---:|---:|---:|---:|") + for row in comparisons: + print( + f"| {row['repetition']} | {row['tests']} | {row['serial_seconds']:.2f}s | {row['parallel_seconds']:.2f}s | {row['parallel_total_seconds']:.2f}s |" + ) + print( + "\nThese times cover pytest startup and execution, excluding runner provisioning, dependency installation and artifact upload. Parallel critical path assumes jobs are available together; workflow/job timestamps must also be compared.\n" + ) + for report in reports: + if report["lane"] != "serial": + continue + phases = collections.defaultdict(float) + for rows in report["reports"].values(): + for row in rows: + if not row["subtest"]: + phases[row["phase"]] += row["seconds"] + print(f"## Serial repetition {report['repetition']}\n") + print( + "Phase totals: " + + ", ".join(f"{name} {seconds:.2f}s" for name, seconds in sorted(phases.items())) + + ".\n" + ) + for phase in ("setup", "call"): + ranked = sorted( + ( + ( + sum( + row["seconds"] + for row in rows + if row["phase"] == phase and not row["subtest"] + ), + nodeid, + ) + for nodeid, rows in report["reports"].items() + ), + reverse=True, + ) + print(f"Slowest {phase} phases:\n") + for seconds, nodeid in ranked[:10]: + print(f"- {seconds:.3f}s: `{nodeid}`") + print() + for path in args.directory.rglob("profile.pstats"): + stats = pstats.Stats(str(path)) + print("## Diagnostic call profile\n") + print( + "This separate instrumented run is excluded from the speed comparison. Cumulative function times overlap and must not be added together.\n" + ) + rows = [ + (values[3], values[1], file, name) + for (file, _line, name), values in stats.stats.items() + if file.endswith(("tests/conftest.py", "tests/corpus.py", "src/api/db.py")) + ] + for seconds, calls, file, name in sorted(rows, reverse=True)[:20]: + print(f"- {seconds:.3f}s cumulative, {calls} calls: `{Path(file).name}:{name}`") + serial = [row["serial_seconds"] for row in comparisons] + parallel = [row["parallel_seconds"] for row in comparisons] + print( + f"\nMedian across {len(comparisons)} repetitions: serial {statistics.median(serial):.2f}s; parallel critical path {statistics.median(parallel):.2f}s." + ) + + +if __name__ == "__main__": + main()