diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 00000000000..4ed70d500ab
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,243 @@
+version: 2.1
+
+# Runs this fork's validation workflows through pinned act because GitHub Actions is disabled.
+# Pull requests and branch pushes run the PR checks; merge queues add stage.yml. Event metadata is
+# generated once per CircleCI revision and shared by all shards.
+#
+# The read-only GitHub token is exposed to PR code, so its CircleCI context must remain
+# restricted to trusted contributors. Keep CircleCI auto-cancellation enabled.
+#
+# The act binary, checksum, and runner image are pinned. Compiler caches are no-ops because
+# act's local cache is not reusable across these machines; Pages plumbing is skipped after the
+# validation build. Lint and unit jobs are distributed; other workflows run up to four jobs.
+#
+# Excluded: pr-title.yml, label-pr.yml, and the privileged cyclops-audit.yml integration.
+
+jobs:
+ # Replaces lint.yml's `deny` job, which calls tempoxyz/ci's reusable deny.yml. That workflow starts
+ # with tempoxyz's secure-runner, which needs a GitHub OIDC token that act never receives, so the
+ # check itself runs natively here. Its policy comes from deny.toml in this repo.
+ #
+ # When re-applying this CI patch onto a new reth release, diff lint.yml's `deny:` job and the
+ # tempoxyz/ci deny.yml it calls against this job (toolchain, cargo-deny version, `check`
+ # arguments, deny flags), and update both pins here.
+ cargo-deny:
+ docker:
+ - image: rust:1.95-bookworm@sha256:6258907abe69656e41cd992e0b705cdcfabcbbe3db374f92ed2d47121282d4a1
+ resource_class: medium
+ steps:
+ - checkout
+ - run:
+ name: Install pinned cargo-deny
+ command: |
+ set -euo pipefail
+ readonly DENY_VERSION="0.20.2"
+ readonly DENY_SHA256="9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f"
+ readonly DENY_ARCHIVE="cargo-deny-${DENY_VERSION}-x86_64-unknown-linux-musl.tar.gz"
+ curl --fail --silent --show-error --location \
+ "https://github.com/EmbarkStudios/cargo-deny/releases/download/${DENY_VERSION}/${DENY_ARCHIVE}" \
+ --output "/tmp/${DENY_ARCHIVE}"
+ echo "${DENY_SHA256} /tmp/${DENY_ARCHIVE}" | sha256sum --check
+ tar -xzf "/tmp/${DENY_ARCHIVE}" -C /usr/local/bin --strip-components=1 \
+ "cargo-deny-${DENY_VERSION}-x86_64-unknown-linux-musl/cargo-deny"
+ test "$(cargo-deny --version)" = "cargo-deny ${DENY_VERSION}"
+ - run:
+ name: cargo deny check
+ command: cargo deny --log-level warn --all-features check all
+
+ prepare-act-event:
+ machine:
+ image: ubuntu-2404:2025.09.1
+ resource_class: medium
+ steps:
+ - checkout
+ - run:
+ name: Snapshot GitHub event
+ command: |
+ mkdir -p /tmp/act-event
+ if ! .circleci/prepare-act-event.sh; then
+ echo "GitHub event preparation failed; act shards will report the failure" |
+ tee /tmp/act-event/failed
+ fi
+ - persist_to_workspace:
+ root: /tmp/act-event
+ paths:
+ - .
+
+ act:
+ parameters:
+ wf:
+ type: string
+ job:
+ type: string
+ default: ""
+ concurrent_jobs:
+ type: integer
+ default: 4
+ machine:
+ image: ubuntu-2404:2025.09.1
+ resource_class: xlarge
+ steps:
+ - checkout
+ - attach_workspace:
+ at: /tmp/act-event
+ - run:
+ name: Check event snapshot
+ command: |
+ if [ -f /tmp/act-event/failed ]; then
+ cat /tmp/act-event/failed >&2
+ exit 1
+ fi
+ - run:
+ name: Install pinned act
+ command: |
+ set -euo pipefail
+ readonly ACT_VERSION="0.2.89-action-cache-fix.1"
+ readonly ACT_RELEASE="v0.2.89-action-cache-fix.4"
+ readonly ACT_BINARY="act_Linux_x86_64"
+ readonly ACT_SHA256="28bfcf1502f59a834b14f2fb23c39e5b253c2234d875592b5aa0a175f9674dce"
+ curl --fail --silent --show-error --location \
+ "https://github.com/ethereum-optimism/act/releases/download/${ACT_RELEASE}/${ACT_BINARY}" \
+ --output "/tmp/${ACT_BINARY}"
+ echo "${ACT_SHA256} /tmp/${ACT_BINARY}" | sha256sum --check
+ sudo install -m 0755 "/tmp/${ACT_BINARY}" /usr/local/bin/act
+ test "$(act --version)" = "act version ${ACT_VERSION}"
+ - run:
+ name: Stub unsupported non-check actions
+ command: |
+ mkdir -p /tmp/noop-action
+ printf '%s\n' \
+ 'name: noop' \
+ 'description: disabled service plumbing under act' \
+ 'inputs:' \
+ ' cache-bin: {required: false}' \
+ ' cache-on-failure: {required: false}' \
+ ' cache-targets: {required: false}' \
+ ' prefix-key: {required: false}' \
+ 'runs:' \
+ ' using: composite' \
+ ' steps:' \
+ ' - run: echo "Service plumbing disabled under act"' \
+ ' shell: bash' \
+ > /tmp/noop-action/action.yml
+ - run:
+ name: Run << parameters.wf >> via act
+ no_output_timeout: 45m
+ command: |
+ set -euo pipefail
+ event_path=/tmp/act-event/github-event.json
+ event_name="$(jq -er 'if .pull_request then "pull_request" elif .merge_group then "merge_group" elif .ref and .after then "push" else error("unsupported event") end' "$event_path")"
+ base_ref="$(jq -r '(.pull_request.base.ref // .merge_group.base_ref // "") | sub("^refs/heads/"; "")' "$event_path")"
+ target_job="<< parameters.job >>"
+ # Validate documentation without entering Pages deployment on branch pushes.
+ if [ "<< parameters.wf >>" = "book.yml" ]; then
+ target_job=build
+ fi
+ act_job=()
+ if [ -n "$target_job" ]; then
+ case "$target_job" in
+ crate-checks-*)
+ partition="${target_job#crate-checks-}"
+ if [[ ! "$partition" =~ ^[1-9][0-9]*$ ]]; then
+ echo "invalid crate-checks partition: $partition" >&2
+ exit 1
+ fi
+ act_job=(-j crate-checks --matrix "partition:${partition}")
+ ;;
+ *)
+ act_job=(-j "$target_job")
+ ;;
+ esac
+ fi
+
+ act_env=(--env "RUSTC_WRAPPER=" --env "GITHUB_BASE_REF=${base_ref}")
+ if [ "<< parameters.wf >>" = "unit.yml" ] && [ "$target_job" = "state" ]; then
+ # EF tests use Rayon internally, so unrestricted nextest processes multiply
+ # runnable threads and starve valid_blocks past its upstream timeout.
+ act_env+=(--env "NEXTEST_TEST_THREADS=2")
+ fi
+ if [ "$event_name" != "pull_request" ]; then
+ act_env+=(
+ --env "GITHUB_REF=refs/heads/${CIRCLE_BRANCH}"
+ --env "GITHUB_SHA=${CIRCLE_SHA1}"
+ )
+ fi
+
+ act "$event_name" -W ".github/workflows/<< parameters.wf >>" \
+ "${act_job[@]}" \
+ --eventpath "$event_path" \
+ -P "ubuntu-latest=catthehacker/ubuntu@sha256:b839c14c4410998529ec18f951262bdf87a2b23bc1467304d07b491b9455e074" \
+ --concurrent-jobs << parameters.concurrent_jobs >> \
+ "${act_env[@]}" \
+ -s GITHUB_TOKEN="${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \
+ --local-repository "mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba=/tmp/noop-action" \
+ --local-repository "Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6=/tmp/noop-action" \
+ --local-repository "actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d=/tmp/noop-action" \
+ --local-repository "actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9=/tmp/noop-action"
+
+workflows:
+ reth-act:
+ jobs:
+ - prepare-act-event:
+ context:
+ - circleci-repo-readonly-authenticated-github-token
+ - act:
+ context:
+ - circleci-repo-readonly-authenticated-github-token
+ name: "act-<< matrix.wf >>"
+ requires:
+ - prepare-act-event
+ matrix:
+ parameters:
+ wf:
+ - book.yml
+ - compact.yml
+ - e2e.yml
+ - grafana.yml
+ - integration.yml
+ - lint-actions.yml
+ - stage.yml
+ - act:
+ wf: lint.yml
+ concurrent_jobs: 1
+ context:
+ - circleci-repo-readonly-authenticated-github-token
+ name: "act-lint-<< matrix.job >>"
+ requires:
+ - prepare-act-event
+ matrix:
+ parameters:
+ job:
+ - clippy-binaries
+ - clippy
+ - wasm
+ - riscv
+ - crate-checks-1
+ - crate-checks-2
+ - crate-checks-3
+ - msrv
+ - docs
+ - fmt
+ - udeps
+ - book
+ - typos
+ - check-toml
+ - grafana
+ - no-test-deps
+ - feature-propagation
+ - cargo-deny:
+ name: lint-deny
+ - act:
+ wf: unit.yml
+ concurrent_jobs: 1
+ context:
+ - circleci-repo-readonly-authenticated-github-token
+ name: "act-unit-<< matrix.job >>"
+ requires:
+ - prepare-act-event
+ matrix:
+ parameters:
+ job:
+ - test
+ - state
+ - doc
diff --git a/.circleci/prepare-act-event.sh b/.circleci/prepare-act-event.sh
new file mode 100755
index 00000000000..99cccb27f3c
--- /dev/null
+++ b/.circleci/prepare-act-event.sh
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+readonly OUTPUT_DIR="${ACT_EVENT_OUTPUT_DIR:-/tmp/act-event}"
+readonly EVENT_PATH="${OUTPUT_DIR}/github-event.json"
+
+mkdir -p "$OUTPUT_DIR"
+rm -f "$EVENT_PATH"
+
+: "${CIRCLE_SHA1:?missing CircleCI revision}"
+: "${CIRCLE_PROJECT_USERNAME:?missing CircleCI project owner}"
+: "${CIRCLE_PROJECT_REPONAME:?missing CircleCI project repository}"
+
+repository="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
+
+if [[ "${CIRCLE_BRANCH:-}" == gh-readonly-queue/* ]]; then
+ event_name="merge_group"
+elif [[ -n "${CIRCLE_PULL_REQUEST:-}" ]]; then
+ event_name="pull_request"
+else
+ event_name="push"
+fi
+
+case "$event_name" in
+ pull_request)
+ : "${CIRCLE_PULL_REQUEST:?pull-request trigger must provide CIRCLE_PULL_REQUEST}"
+ pr_number="${CIRCLE_PULL_REQUEST##*/}"
+ if [[ ! "$pr_number" =~ ^[1-9][0-9]*$ ]]; then
+ echo "invalid pull-request URL: ${CIRCLE_PULL_REQUEST}" >&2
+ exit 1
+ fi
+
+ pull_request_path="${OUTPUT_DIR}/pull-request.json"
+ curl --fail --silent --show-error --location \
+ -H "Accept: application/vnd.github+json" \
+ -H "Authorization: Bearer ${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \
+ -H "X-GitHub-Api-Version: 2022-11-28" \
+ "https://api.github.com/repos/${repository}/pulls/${pr_number}" > "$pull_request_path"
+
+ event_head_sha="$(jq -er '.head.sha' "$pull_request_path")"
+ if [[ "$event_head_sha" != "$CIRCLE_SHA1" ]]; then
+ echo "PR head does not match CircleCI revision: ${CIRCLE_SHA1} != ${event_head_sha}" >&2
+ exit 1
+ fi
+
+ jq '{
+ action: "synchronize",
+ number: .number,
+ pull_request: .,
+ repository: .base.repo,
+ sender: .user
+ }' "$pull_request_path" > "$EVENT_PATH"
+ base_ref="$(jq -er '.base.ref | select(length > 0)' "$pull_request_path")"
+ head_ref="$(jq -er '.head.ref | select(length > 0)' "$pull_request_path")"
+ echo "act event: pull_request #${pr_number}, ${head_ref} -> ${base_ref}"
+ ;;
+ push)
+ : "${CIRCLE_BRANCH:?push trigger must provide CIRCLE_BRANCH}"
+ jq -n \
+ --arg ref "refs/heads/${CIRCLE_BRANCH}" \
+ --arg after "$CIRCLE_SHA1" \
+ '{ref: $ref, after: $after}' > "$EVENT_PATH"
+ echo "act event: push ${CIRCLE_BRANCH} at ${CIRCLE_SHA1}"
+ ;;
+ merge_group)
+ : "${CIRCLE_BRANCH:?merge-queue trigger must provide CIRCLE_BRANCH}"
+ if [[ ! "$CIRCLE_BRANCH" =~ ^gh-readonly-queue/(.+)/pr-([1-9][0-9]*)-[^/]+$ ]]; then
+ echo "merge_group requires a gh-readonly-queue//pr-- branch, got: ${CIRCLE_BRANCH}" >&2
+ exit 1
+ fi
+ base_ref="${BASH_REMATCH[1]}"
+
+ if ! base_sha="$(git rev-parse "${CIRCLE_SHA1}^1" 2>/dev/null)"; then
+ git fetch --no-tags --depth=2 origin "$CIRCLE_SHA1"
+ base_sha="$(git rev-parse "${CIRCLE_SHA1}^1")"
+ fi
+
+ jq -n \
+ --arg head_sha "$CIRCLE_SHA1" \
+ --arg head_ref "refs/heads/${CIRCLE_BRANCH}" \
+ --arg base_sha "$base_sha" \
+ --arg base_ref "refs/heads/${base_ref}" \
+ --arg repository "$repository" \
+ --arg repository_name "$CIRCLE_PROJECT_REPONAME" \
+ --arg repository_owner "$CIRCLE_PROJECT_USERNAME" \
+ '{
+ action: "checks_requested",
+ merge_group: {
+ head_sha: $head_sha,
+ head_ref: $head_ref,
+ base_sha: $base_sha,
+ base_ref: $base_ref
+ },
+ repository: {
+ full_name: $repository,
+ name: $repository_name,
+ owner: {login: $repository_owner}
+ }
+ }' > "$EVENT_PATH"
+ echo "act event: merge_group ${CIRCLE_BRANCH} -> ${base_ref} at ${base_sha}"
+ ;;
+esac
diff --git a/.config/nextest.toml b/.config/nextest.toml
index 01cb0158619..d07545e2400 100644
--- a/.config/nextest.toml
+++ b/.config/nextest.toml
@@ -6,6 +6,12 @@ slow-timeout = { period = "30s", terminate-after = 2 }
filter = "test(general_state_tests)"
slow-timeout = { period = "1m", terminate-after = 10 }
+# valid_blocks covers a full fixture directory, like the grouped state tests.
+# Keep its CircleCI runtime bounded by the same ten-minute budget.
+[[profile.default.overrides]]
+filter = "package(=ef-tests) and test(=valid_blocks)"
+slow-timeout = { period = "1m", terminate-after = 10 }
+
[[profile.default.overrides]]
filter = "test(eest_fixtures)"
slow-timeout = { period = "2m", terminate-after = 10 }
diff --git a/.config/zepter.yaml b/.config/zepter.yaml
index 7aa1bd725ef..d2d5dfb5255 100644
--- a/.config/zepter.yaml
+++ b/.config/zepter.yaml
@@ -12,7 +12,7 @@ workflows:
# Check that `A` activates the features of `B`.
"propagate-feature",
# These are the features to check:
- "--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global,trie-debug,secp256k1",
+ "--features=std,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global,trie-debug,secp256k1",
# Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually.
"--left-side-feature-missing=ignore",
# Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on.
diff --git a/.dockerignore b/.dockerignore
index f4baa18a38b..e9d3ac4167b 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -7,7 +7,6 @@
!/pkg
!/testing
!/.github/scripts/install_llvm_ubuntu.sh
-!book.toml
!Cargo.lock
!Cargo.toml
!Cross.toml
diff --git a/.github/scripts/bench-call-summary.py b/.github/scripts/bench-call-summary.py
new file mode 100755
index 00000000000..49eeba43069
--- /dev/null
+++ b/.github/scripts/bench-call-summary.py
@@ -0,0 +1,1097 @@
+#!/usr/bin/env python3
+"""Summarize an RPC replay benchmark and generate summary JSON + markdown.
+
+Usage:
+ bench-call-summary.py \
+ --output-summary \
+ --output-markdown \
+ --baseline-dir [ ...] \
+ --feature-dir [ ...]
+
+Each run directory holds the output of one `bench call` measured phase:
+
+ report.json run metadata, identity and per-method aggregates
+ requests.csv open-loop, one row per request
+ record_timings.csv closed-loop, one row per record and pass
+ responses.ndjson one response digest per corpus record
+ cpu.json CPU time the node's cgroup spent on the measured phase
+
+Latency, throughput and CPU per request are compared with whole-run cluster
+bootstrap confidence intervals, and every response digest is compared against
+the first baseline run. Any digest that differs between the arms fails the
+verdict regardless of the timings; digests that already differ between two
+baseline runs make the comparison inconclusive instead.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import importlib.util
+import json
+import os
+from pathlib import Path
+import random
+import sys
+
+# Latency floors below which a delta is treated as noise. These are starting
+# values for the first runs on the bench runner and are meant to be tightened
+# once the A/A spread of each class is known.
+PRACTICAL_FLOOR_PCT = {
+ "mean": 2.5,
+ "p50": 2.5,
+ "p90": 2.5,
+ "p99": 5.0,
+ "record_median": 2.5,
+ "closed_loop_rps": 2.5,
+ "cpu_per_request": 2.5,
+}
+MAX_DIVERGENT_REPORTED = 40
+OK_STATUS = "ok"
+
+
+def _load_engine_summary() -> object:
+ """Import the engine summary script for its statistics helpers."""
+ path = Path(__file__).resolve().with_name("bench-reth-summary.py")
+ spec = importlib.util.spec_from_file_location("bench_reth_summary", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+_engine = _load_engine_summary()
+BOOTSTRAP_ITERATIONS = _engine.BOOTSTRAP_ITERATIONS
+_bootstrap_ci = _engine._bootstrap_ci
+_ci_half_width = _engine._ci_half_width
+_mean = _engine._mean
+build_grafana_logs_url = _engine.build_grafana_logs_url
+build_grafana_traces_url = _engine.build_grafana_traces_url
+fmt_ms = _engine.fmt_ms
+generate_observability_section = _engine.generate_observability_section
+percentile = _engine.percentile
+resolve_observability_range = _engine.resolve_observability_range
+significance = _engine.significance
+
+
+def load_run(directory: Path) -> dict:
+ """Load one run directory into the per-run shape used by the comparison."""
+ report_path = directory / "report.json"
+ if not report_path.is_file():
+ raise SystemExit(f"Missing {report_path}")
+ with report_path.open() as f:
+ report = json.load(f)
+
+ responses_path = directory / "responses.ndjson"
+ if not responses_path.is_file():
+ raise SystemExit(f"Missing {responses_path}")
+
+ requests = parse_requests_csv(directory / "requests.csv")
+ records = parse_record_timings_csv(directory / "record_timings.csv")
+ if not requests and not records:
+ raise SystemExit(f"No open-loop or closed-loop timings in {directory}")
+
+ cpu = {}
+ cpu_path = directory / "cpu.json"
+ if cpu_path.is_file():
+ with cpu_path.open() as f:
+ cpu = json.load(f)
+
+ return {
+ "name": directory.name,
+ "report": report,
+ "requests": requests,
+ "records": records,
+ "responses": parse_responses(responses_path),
+ "cpu": cpu,
+ }
+
+
+def parse_requests_csv(path: Path) -> list[dict]:
+ """Parse the open-loop per-request CSV."""
+ if not path.is_file():
+ return []
+ rows = []
+ with path.open() as f:
+ for row in csv.DictReader(f):
+ rows.append(
+ {
+ "offset_ms": float(row["offset_ms"]),
+ "record_index": int(row["record_index"]),
+ "method": row["method"],
+ "latency_us": int(row["latency_us"]),
+ "status": row["status"],
+ }
+ )
+ return rows
+
+
+def parse_record_timings_csv(path: Path) -> list[dict]:
+ """Parse the closed-loop per-record CSV."""
+ if not path.is_file():
+ return []
+ rows = []
+ with path.open() as f:
+ for row in csv.DictReader(f):
+ rows.append(
+ {
+ "record_index": int(row["record_index"]),
+ "method": row["method"],
+ "pass": int(row["pass"]),
+ "latency_us": int(row["latency_us"]),
+ "status": row["status"],
+ }
+ )
+ return rows
+
+
+def parse_responses(path: Path) -> dict[int, dict]:
+ """Parse response digests, keyed by record index."""
+ responses = {}
+ with path.open() as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ entry = json.loads(line)
+ responses[int(entry["record_index"])] = {
+ "method": entry.get("method", "unknown"),
+ "kind": entry.get("kind", "unknown"),
+ "digest": entry.get("digest", ""),
+ "len": entry.get("len"),
+ }
+ return responses
+
+
+def call_section(report: dict) -> dict:
+ """The replay section of a report; `bench call` nests its fields under `call`."""
+ section = report.get("call")
+ return section if isinstance(section, dict) else report
+
+
+def report_field(report: dict, key: str):
+ """Look a field up in the report, tolerating the nested `call` and identity blocks."""
+ for scope in (report, call_section(report)):
+ if key in scope:
+ return scope[key]
+ for nested in ("identity", "metadata", "meta"):
+ block = scope.get(nested)
+ if isinstance(block, dict) and key in block:
+ return block[key]
+ return None
+
+
+def median(values: list[float]) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ mid = len(ordered) // 2
+ if len(ordered) % 2:
+ return ordered[mid]
+ return (ordered[mid - 1] + ordered[mid]) / 2.0
+
+
+def open_loop_stats(rows: list[dict]) -> dict:
+ """Latency distribution, achieved rate and error rate of one open-loop run."""
+ if not rows:
+ return {}
+ ok_rows = [row for row in rows if row["status"] == OK_STATUS]
+ latencies_ms = sorted(row["latency_us"] / 1_000 for row in ok_rows)
+ offsets = [row["offset_ms"] for row in rows]
+ span_s = (max(offsets) - min(offsets)) / 1_000 if len(rows) > 1 else 0.0
+ return {
+ "requests": len(rows),
+ "ok": len(ok_rows),
+ "errors": len(rows) - len(ok_rows),
+ "error_rate_pct": (len(rows) - len(ok_rows)) / len(rows) * 100.0,
+ "mean_ms": _mean(latencies_ms),
+ "p50_ms": percentile(latencies_ms, 50),
+ "p90_ms": percentile(latencies_ms, 90),
+ "p99_ms": percentile(latencies_ms, 99),
+ "open_loop_rps": len(rows) / span_s if span_s > 0 else 0.0,
+ }
+
+
+def record_medians(rows: list[dict]) -> dict[int, float]:
+ """Median closed-loop latency in ms per record, over the passes of one run."""
+ by_record: dict[int, list[float]] = {}
+ for row in rows:
+ if row["status"] != OK_STATUS:
+ continue
+ by_record.setdefault(row["record_index"], []).append(row["latency_us"] / 1_000)
+ return {index: median(values) for index, values in by_record.items()}
+
+
+def arm_record_medians(runs: list[dict], method: str | None = None) -> dict[int, float]:
+ """Median closed-loop latency per record, over all passes and runs of one arm."""
+ by_record: dict[int, list[float]] = {}
+ for run in runs:
+ for row in run["records"]:
+ if row["status"] != OK_STATUS:
+ continue
+ if method is not None and row["method"] != method:
+ continue
+ by_record.setdefault(row["record_index"], []).append(row["latency_us"] / 1_000)
+ return {index: median(values) for index, values in by_record.items()}
+
+
+def run_stats(run: dict, method: str | None = None) -> dict:
+ """Every displayed metric for one run, optionally restricted to one method."""
+ requests = run["requests"]
+ records = run["records"]
+ if method is not None:
+ requests = [row for row in requests if row["method"] == method]
+ records = [row for row in records if row["method"] == method]
+
+ stats = open_loop_stats(requests)
+ medians = record_medians(records)
+ if medians:
+ stats["record_median_ms"] = median(list(medians.values()))
+ stats["records"] = len(medians)
+
+ report = run["report"]
+ if method is None:
+ closed_loop_rps = report_field(report, "closed_loop_rps")
+ dropped = report_field(report, "dropped")
+ else:
+ methods = call_section(report).get("methods") or {}
+ entry = methods.get(method) or {}
+ closed_loop_rps = entry.get("closed_loop_rps")
+ if closed_loop_rps is None and isinstance(entry.get("closed_loop"), dict):
+ closed_loop_rps = entry["closed_loop"].get("rps")
+ dropped = entry.get("dropped")
+ if isinstance(closed_loop_rps, (int, float)):
+ stats["closed_loop_rps"] = float(closed_loop_rps)
+ if isinstance(dropped, int):
+ stats["dropped"] = dropped
+
+ if method is None:
+ cpu_ms = cpu_ms_per_request(run)
+ if cpu_ms is not None:
+ stats["cpu_ms_per_request"] = cpu_ms
+
+ lengths = [
+ entry["len"]
+ for entry in run["responses"].values()
+ if isinstance(entry.get("len"), int) and (method is None or entry["method"] == method)
+ ]
+ if lengths:
+ stats["response_bytes_median"] = median([float(v) for v in lengths])
+ return stats
+
+
+def cpu_ms_per_request(run: dict) -> float | None:
+ """CPU milliseconds the node spent per successful request of one run."""
+ cpu = run["cpu"]
+ usage_usec = cpu.get("usage_usec_delta")
+ if not isinstance(usage_usec, (int, float)) or usage_usec <= 0:
+ return None
+ requests_ok = cpu.get("requests_ok")
+ if not isinstance(requests_ok, int) or requests_ok <= 0:
+ requests_ok = sum(
+ 1 for row in run["requests"] + run["records"] if row["status"] == OK_STATUS
+ )
+ if requests_ok <= 0:
+ return None
+ return usage_usec / 1_000 / requests_ok
+
+
+def arm_stats(runs: list[dict], method: str | None = None) -> dict:
+ """Point estimates for one arm, averaged over its runs."""
+ per_run = [run_stats(run, method) for run in runs]
+ per_run = [stats for stats in per_run if stats]
+ if not per_run:
+ return {}
+
+ stats = {"runs": len(per_run)}
+ for key in (
+ "mean_ms",
+ "p50_ms",
+ "p90_ms",
+ "p99_ms",
+ "open_loop_rps",
+ "closed_loop_rps",
+ "record_median_ms",
+ "cpu_ms_per_request",
+ "error_rate_pct",
+ "response_bytes_median",
+ ):
+ values = [run[key] for run in per_run if key in run]
+ if values:
+ stats[key] = _mean(values)
+ for key in ("requests", "ok", "errors", "dropped"):
+ values = [run[key] for run in per_run if key in run]
+ if values:
+ stats[key] = sum(values)
+ records = [run["records"] for run in per_run if "records" in run]
+ if records:
+ stats["records"] = max(records)
+ return stats
+
+
+def per_run_values(runs: list[dict], method: str | None = None) -> dict[str, list[float]]:
+ """One value per run and metric, the input of the cluster bootstrap."""
+ values: dict[str, list[float]] = {}
+ for run in runs:
+ stats = run_stats(run, method)
+ for key, value in stats.items():
+ if isinstance(value, (int, float)):
+ values.setdefault(key, []).append(float(value))
+ return values
+
+
+def cluster_bootstrap_ci(
+ rng: random.Random,
+ baseline_values: dict[str, list[float]],
+ feature_values: dict[str, list[float]],
+ metrics: tuple[str, ...],
+ n_iter: int = BOOTSTRAP_ITERATIONS,
+) -> dict[str, float]:
+ """Resample whole runs to estimate run-to-run noise per metric.
+
+ Mirrors the engine summary's cluster bootstrap, which resamples block rows
+ and cannot be reused for request shaped runs.
+ """
+ empty = {metric: 0.0 for metric in metrics}
+ usable = [
+ metric
+ for metric in metrics
+ if len(baseline_values.get(metric, [])) >= 2 and len(feature_values.get(metric, [])) >= 2
+ ]
+ if not usable:
+ return empty
+
+ samples: dict[str, list[float]] = {metric: [] for metric in usable}
+ for metric in usable:
+ baseline_metric = baseline_values[metric]
+ feature_metric = feature_values[metric]
+ baseline_count = len(baseline_metric)
+ feature_count = len(feature_metric)
+ for _ in range(n_iter):
+ baseline_sample = [baseline_metric[rng.randrange(baseline_count)] for _ in range(baseline_count)]
+ feature_sample = [feature_metric[rng.randrange(feature_count)] for _ in range(feature_count)]
+ samples[metric].append(_mean(feature_sample) - _mean(baseline_sample))
+
+ result = dict(empty)
+ for metric in usable:
+ result[metric] = _ci_half_width(samples[metric])
+ return result
+
+
+def paired_record_delta(
+ rng: random.Random,
+ baseline_runs: list[dict],
+ feature_runs: list[dict],
+ method: str | None = None,
+) -> dict | None:
+ """Per-record closed-loop delta with a bootstrap CI over records."""
+ baseline_medians = arm_record_medians(baseline_runs, method)
+ feature_medians = arm_record_medians(feature_runs, method)
+ common = sorted(set(baseline_medians) & set(feature_medians))
+ if not common:
+ return None
+ diffs = [feature_medians[index] - baseline_medians[index] for index in common]
+ base = _mean([baseline_medians[index] for index in common])
+ return {
+ "records": len(common),
+ "baseline_ms": base,
+ "feature_ms": _mean([feature_medians[index] for index in common]),
+ "delta_ms": _mean(diffs),
+ "ci_ms": _bootstrap_ci(rng, diffs),
+ }
+
+
+def make_change(
+ baseline_value: float | None,
+ feature_value: float | None,
+ ci_value: float,
+ floor_pct: float,
+ lower_is_better: bool,
+ aa_pct: float | None = None,
+) -> dict | None:
+ """Build one `changes` entry in the shape the comment renderers consume."""
+ if baseline_value is None or feature_value is None or baseline_value <= 0:
+ return None
+ pct = (feature_value - baseline_value) / baseline_value * 100.0
+ ci_pct = ci_value / baseline_value * 100.0 if ci_value else 0.0
+ change = {
+ "pct": round(pct, 4),
+ "ci_pct": round(ci_pct, 4),
+ "floor_pct": round(floor_pct, 4),
+ "sig": significance(pct, ci_pct, floor_pct, lower_is_better),
+ "baseline": baseline_value,
+ "feature": feature_value,
+ }
+ if aa_pct is not None:
+ change["aa_pct"] = round(aa_pct, 4)
+ return change
+
+
+def aa_spread(runs: list[dict], method: str | None = None) -> dict[str, float]:
+ """Percent difference between the first two baseline runs, the noise floor."""
+ if len(runs) < 2:
+ return {}
+ first = run_stats(runs[0], method)
+ second = run_stats(runs[1], method)
+ spread = {}
+ for key, value in first.items():
+ if not isinstance(value, (int, float)) or value <= 0:
+ continue
+ other = second.get(key)
+ if isinstance(other, (int, float)):
+ spread[key] = (other - value) / value * 100.0
+ return spread
+
+
+def compute_changes(
+ rng: random.Random,
+ baseline_runs: list[dict],
+ feature_runs: list[dict],
+ baseline_stats: dict,
+ feature_stats: dict,
+ method: str | None = None,
+) -> dict:
+ """Compare one corpus or one method between the two arms."""
+ metrics = (
+ ("mean", "mean_ms", True),
+ ("p50", "p50_ms", True),
+ ("p90", "p90_ms", True),
+ ("p99", "p99_ms", True),
+ ("closed_loop_rps", "closed_loop_rps", False),
+ ("cpu_per_request", "cpu_ms_per_request", True),
+ )
+ ci = cluster_bootstrap_ci(
+ rng,
+ per_run_values(baseline_runs, method),
+ per_run_values(feature_runs, method),
+ tuple(stat_key for _, stat_key, _ in metrics) + ("record_median_ms",),
+ )
+ spread = aa_spread(baseline_runs, method)
+
+ changes = {}
+ for name, stat_key, lower_is_better in metrics:
+ change = make_change(
+ baseline_stats.get(stat_key),
+ feature_stats.get(stat_key),
+ ci.get(stat_key, 0.0),
+ PRACTICAL_FLOOR_PCT.get(name, 0.0),
+ lower_is_better,
+ spread.get(stat_key),
+ )
+ if change:
+ changes[name] = change
+
+ # The point estimate is the per-run record median averaged over runs, so its
+ # interval comes from the same whole-run bootstrap as the other metrics; the
+ # per-record pairing is kept as supplementary detail only.
+ change = make_change(
+ baseline_stats.get("record_median_ms"),
+ feature_stats.get("record_median_ms"),
+ ci.get("record_median_ms", 0.0),
+ PRACTICAL_FLOOR_PCT["record_median"],
+ True,
+ spread.get("record_median_ms"),
+ )
+ if change:
+ paired = paired_record_delta(rng, baseline_runs, feature_runs, method)
+ if paired:
+ change["records"] = paired["records"]
+ change["paired_delta_ms"] = round(paired["delta_ms"], 6)
+ changes["record_median"] = change
+ return changes
+
+
+def parity_entry() -> dict:
+ return {
+ "matched": 0,
+ "content_mismatch": 0,
+ "kind_mismatch": 0,
+ "missing": 0,
+ "nondeterministic": 0,
+ "excluded": 0,
+ "divergent_records": [],
+ }
+
+
+def compare_responses(
+ reference: dict[int, dict],
+ other: dict[int, dict],
+ exclude: frozenset[int] = frozenset(),
+) -> dict:
+ """Compare one run's digests against the reference run, per method.
+
+ Records in `exclude` are unstable in the baseline itself and are only counted,
+ never judged.
+ """
+ per_method: dict[str, dict] = {}
+ for index, expected in reference.items():
+ method = expected["method"]
+ entry = per_method.setdefault(method, parity_entry())
+ if index in exclude:
+ entry["excluded"] += 1
+ continue
+ actual = other.get(index)
+ if actual is None:
+ entry["missing"] += 1
+ elif actual["kind"] != expected["kind"]:
+ entry["kind_mismatch"] += 1
+ elif actual["digest"] != expected["digest"]:
+ entry["content_mismatch"] += 1
+ else:
+ entry["matched"] += 1
+ continue
+ if len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED:
+ entry["divergent_records"].append(index)
+ # A record the reference run never answered but this run did is just as divergent.
+ for index, actual in other.items():
+ if index in reference:
+ continue
+ entry = per_method.setdefault(actual["method"], parity_entry())
+ if index in exclude:
+ entry["excluded"] += 1
+ continue
+ entry["missing"] += 1
+ if len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED:
+ entry["divergent_records"].append(index)
+ return per_method
+
+
+def merge_parity(target: dict, addition: dict) -> None:
+ for method, counts in addition.items():
+ entry = target.setdefault(method, parity_entry())
+ for key in ("matched", "content_mismatch", "kind_mismatch", "missing", "nondeterministic", "excluded"):
+ entry[key] += counts.get(key, 0)
+ for index in counts["divergent_records"]:
+ if index not in entry["divergent_records"] and len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED:
+ entry["divergent_records"].append(index)
+
+
+def parity_totals(per_method: dict) -> dict:
+ totals = {
+ "matched": 0,
+ "content_mismatch": 0,
+ "kind_mismatch": 0,
+ "missing": 0,
+ "nondeterministic": 0,
+ "excluded": 0,
+ }
+ for counts in per_method.values():
+ for key in totals:
+ totals[key] += counts.get(key, 0)
+ # A record answered in only one of the two runs, or answered differently
+ # within the feature runs alone, counts as a divergence.
+ totals["mismatched"] = (
+ totals["content_mismatch"] + totals["kind_mismatch"] + totals["missing"] + totals["nondeterministic"]
+ )
+ return totals
+
+
+def nondeterministic_count(runs: list[dict]) -> int:
+ total = 0
+ for run in runs:
+ entries = report_field(run["report"], "nondeterministic")
+ if isinstance(entries, list):
+ total += len(entries)
+ elif isinstance(entries, int):
+ total += entries
+ return total
+
+
+def nondeterministic_records(runs: list[dict]) -> tuple[set[int], bool]:
+ """Record indexes a run answered differently within itself.
+
+ The second value is true when a run only reported a count, so the records
+ cannot be identified.
+ """
+ indexes: set[int] = set()
+ unknown = False
+ for run in runs:
+ entries = report_field(run["report"], "nondeterministic")
+ if isinstance(entries, list):
+ indexes.update(int(index) for index in entries)
+ total = report_field(run["report"], "nondeterministic_total")
+ if isinstance(total, int) and total > len(entries):
+ unknown = True
+ elif isinstance(entries, int) and entries > 0:
+ unknown = True
+ return indexes, unknown
+
+
+def compute_parity(baseline_runs: list[dict], feature_runs: list[dict]) -> dict:
+ """Compare every run's digests against the first baseline run.
+
+ Records that are unstable in the baseline itself, answered differently
+ between baseline runs or within one, are excluded from the decision and
+ reported; the feature is judged on the remaining stable records, where its
+ own nondeterminism counts as a divergence.
+ """
+ reference = baseline_runs[0]["responses"]
+
+ baseline_parity: dict[str, dict] = {}
+ for run in baseline_runs[1:]:
+ merge_parity(baseline_parity, compare_responses(reference, run["responses"]))
+ baseline_nondeterministic, baseline_unknown = nondeterministic_records(baseline_runs)
+ unstable = set(baseline_nondeterministic)
+ for counts in baseline_parity.values():
+ unstable.update(counts["divergent_records"])
+ baseline_totals = parity_totals(baseline_parity)
+ if baseline_totals["mismatched"] > sum(len(c["divergent_records"]) for c in baseline_parity.values()):
+ # More baseline divergences than we could list: the unstable set is incomplete.
+ baseline_unknown = True
+
+ feature_nondeterministic, feature_unknown = nondeterministic_records(feature_runs)
+ excluded = frozenset(unstable)
+ feature_parity: dict[str, dict] = {}
+ for run in feature_runs:
+ merge_parity(feature_parity, compare_responses(reference, run["responses"], excluded))
+ for index in sorted(feature_nondeterministic - unstable):
+ expected = reference.get(index)
+ if expected is None:
+ continue
+ entry = feature_parity.setdefault(expected["method"], parity_entry())
+ entry["nondeterministic"] += 1
+ if index not in entry["divergent_records"] and len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED:
+ entry["divergent_records"].append(index)
+ feature_totals = parity_totals(feature_parity)
+ nondeterministic = nondeterministic_count(baseline_runs + feature_runs)
+
+ records = len(reference)
+ stable_records = records - len(unstable & set(reference))
+ if baseline_unknown or feature_unknown or (unstable and stable_records == 0):
+ status = "inconclusive"
+ elif feature_totals["mismatched"]:
+ status = "mismatch"
+ else:
+ status = "matched"
+
+ excluded_list = sorted(unstable)
+ return {
+ "status": status,
+ "records": records,
+ "feature": {"totals": feature_totals, "methods": feature_parity},
+ "baseline": {"totals": baseline_totals, "methods": baseline_parity},
+ "nondeterministic": nondeterministic,
+ "excluded": excluded_list[:MAX_DIVERGENT_REPORTED],
+ "excluded_count": len(excluded_list),
+ "line": parity_line(
+ status, records, feature_totals, baseline_totals, nondeterministic, feature_parity, excluded_list
+ ),
+ }
+
+
+def parity_line(
+ status: str,
+ records: int,
+ feature_totals: dict,
+ baseline_totals: dict,
+ nondeterministic: int,
+ feature_parity: dict,
+ excluded: list[int] | None = None,
+) -> str:
+ excluded = excluded or []
+ excluded_note = ""
+ if excluded:
+ shown = ", ".join(str(index) for index in excluded[:MAX_DIVERGENT_REPORTED])
+ excluded_note = f"; {len(excluded)} unstable in the baseline, excluded (records {shown})"
+ checked = feature_totals["matched"] + feature_totals["mismatched"]
+ if status == "matched":
+ return (
+ f"✅ matched {feature_totals['matched']}/{checked} responses "
+ f"over {records} records{excluded_note}"
+ )
+
+ details = []
+ for method, counts in sorted(feature_parity.items()):
+ mismatched = (
+ counts["content_mismatch"] + counts["kind_mismatch"] + counts["missing"] + counts.get("nondeterministic", 0)
+ )
+ if not mismatched:
+ continue
+ indexes = ", ".join(str(index) for index in counts["divergent_records"])
+ details.append(f"`{method}`: {mismatched} (records {indexes})")
+
+ if status == "mismatch":
+ return "❌ {} content, {} kind mismatches, {} answered in one arm only, {} nondeterministic in the feature only — {}{}".format(
+ feature_totals["content_mismatch"],
+ feature_totals["kind_mismatch"],
+ feature_totals["missing"],
+ feature_totals.get("nondeterministic", 0),
+ "; ".join(details) if details else "no per-method detail",
+ excluded_note,
+ )
+
+ reasons = []
+ if baseline_totals["mismatched"]:
+ reasons.append(f"{baseline_totals['mismatched']} baseline-vs-baseline mismatches")
+ if nondeterministic:
+ reasons.append(f"{nondeterministic} records answered differently within a run")
+ if not reasons:
+ reasons.append("no stable records to judge")
+ suffix = f"; feature mismatches: {feature_totals['mismatched']}" if feature_totals["mismatched"] else ""
+ return "⚠️ inconclusive — " + ", ".join(reasons) + suffix
+
+
+def parity_change(parity: dict) -> dict:
+ """Fold parity into `changes` so the shared verdict helper sees it."""
+ totals = parity["feature"]["totals"]
+ change = {
+ "pct": 0.0,
+ "ci_pct": 0.0,
+ "floor_pct": 0.0,
+ "sig": "bad" if parity["status"] == "mismatch" else "neutral",
+ "mismatched": totals["mismatched"],
+ "excluded": parity.get("excluded_count", 0),
+ }
+ if parity["status"] == "inconclusive":
+ change["informational"] = True
+ change["informational_reason"] = "informational, responses are not deterministic"
+ return change
+
+
+def corpus_methods(baseline_runs: list[dict], feature_runs: list[dict]) -> list[str]:
+ methods = set()
+ for run in baseline_runs + feature_runs:
+ for entry in run["responses"].values():
+ methods.add(entry["method"])
+ for row in run["requests"]:
+ methods.add(row["method"])
+ return sorted(methods)
+
+
+def assert_same_head(runs: list[dict]) -> str | None:
+ """Refuse to compare runs that replayed against different chain tips."""
+ hashes = {}
+ for run in runs:
+ head_hash = report_field(run["report"], "head_hash")
+ if head_hash:
+ hashes[run["name"]] = head_hash
+ distinct = set(hashes.values())
+ if len(distinct) > 1:
+ detail = ", ".join(f"{name}={value}" for name, value in sorted(hashes.items()))
+ raise SystemExit(f"Runs replayed against different chain tips: {detail}")
+ return next(iter(distinct), None)
+
+
+def fmt_opt_ms(value: float | None) -> str:
+ return fmt_ms(value) if isinstance(value, (int, float)) else "n/a"
+
+
+def fmt_opt_num(value: float | None, digits: int = 2) -> str:
+ return f"{value:.{digits}f}" if isinstance(value, (int, float)) else "n/a"
+
+
+def change_str(change: dict | None) -> str:
+ if not change:
+ return "n/a"
+ emoji = {"good": "✅", "bad": "❌", "neutral": "⚪"}[
+ "neutral" if change.get("informational") else change["sig"]
+ ]
+ details = [f"±{change['ci_pct']:.2f}%", f"floor {change['floor_pct']:.2f}%"]
+ if "aa_pct" in change:
+ details.append(f"A/A {change['aa_pct']:+.2f}%")
+ if change.get("informational"):
+ details.append("informational")
+ return f"{change['pct']:+.2f}% {emoji} ({', '.join(details)})"
+
+
+def generate_metric_table(summary: dict, baseline_label: str, feature_label: str) -> str:
+ baseline = summary["baseline"]["stats"]
+ feature = summary["feature"]["stats"]
+ changes = summary["changes"]
+ rows = [
+ ("Mean", "mean_ms", "mean", fmt_opt_ms),
+ ("P50", "p50_ms", "p50", fmt_opt_ms),
+ ("P90", "p90_ms", "p90", fmt_opt_ms),
+ ("P99", "p99_ms", "p99", fmt_opt_ms),
+ ("Record median", "record_median_ms", "record_median", fmt_opt_ms),
+ ("Closed-loop rps", "closed_loop_rps", "closed_loop_rps", fmt_opt_num),
+ ("CPU / request", "cpu_ms_per_request", "cpu_per_request", fmt_opt_ms),
+ ]
+ lines = [
+ f"| Metric | {baseline_label} | {feature_label} | Change |",
+ "|--------|------|--------|--------|",
+ ]
+ for label, stat_key, change_key, formatter in rows:
+ if stat_key not in baseline and stat_key not in feature:
+ continue
+ lines.append(
+ f"| {label} | {formatter(baseline.get(stat_key))} | "
+ f"{formatter(feature.get(stat_key))} | {change_str(changes.get(change_key))} |"
+ )
+ lines.append(
+ f"| Error rate | {fmt_opt_num(baseline.get('error_rate_pct'))}% | "
+ f"{fmt_opt_num(feature.get('error_rate_pct'))}% | |"
+ )
+ lines.append(
+ f"| Open-loop rps | {fmt_opt_num(baseline.get('open_loop_rps'))} | "
+ f"{fmt_opt_num(feature.get('open_loop_rps'))} | |"
+ )
+ lines.append("")
+ return "\n".join(lines)
+
+
+def generate_method_table(summary: dict) -> str:
+ methods = summary.get("methods") or {}
+ if len(methods) < 2:
+ return ""
+ lines = [
+ "### Per-method",
+ "",
+ "| Method | Records | Baseline mean | Feature mean | Change | Response bytes | Parity |",
+ "|--------|---------|---------------|--------------|--------|----------------|--------|",
+ ]
+ for method in sorted(methods):
+ entry = methods[method]
+ baseline = entry["baseline"]
+ feature = entry["feature"]
+ totals = entry["parity"]
+ mismatched = totals["content_mismatch"] + totals["kind_mismatch"]
+ parity = f"{totals['matched']} matched" if not mismatched else f"❌ {mismatched} mismatched"
+ lines.append(
+ "| `{}` | {} | {} | {} | {} | {} | {} |".format(
+ method,
+ baseline.get("records", baseline.get("requests", 0)),
+ fmt_opt_ms(baseline.get("mean_ms")),
+ fmt_opt_ms(feature.get("mean_ms")),
+ change_str(entry["changes"].get("mean")),
+ fmt_opt_num(baseline.get("response_bytes_median"), 0),
+ parity,
+ )
+ )
+ lines.append("")
+ return "\n".join(lines)
+
+
+def generate_markdown(
+ summary: dict,
+ baseline_label: str,
+ feature_label: str,
+ behind_baseline: int = 0,
+ repo: str = "",
+ baseline_ref: str = "",
+ baseline_name: str = "",
+ derek_command: str | None = None,
+) -> str:
+ corpus = summary["corpus"]
+ lines = ["## Benchmark Results", "", "## Configuration"]
+ if derek_command:
+ lines.append(f"- Derek command: `{derek_command}`")
+ methods = ", ".join(corpus.get("methods") or summary.get("replayed_methods") or []) or "all"
+ top_gas_note = f" (top-gas {corpus['top_gas']})" if corpus.get("top_gas") else ""
+ tracer_notes = []
+ if corpus.get("tracer"):
+ tracer_notes.append("tracer " + ", ".join(corpus["tracer"]))
+ for key in ("tracer_config", "trace_options"):
+ if corpus.get(key):
+ tracer_notes.append(f"{key.replace('_', '-')} `{json.dumps(corpus[key], separators=(',', ':'))}`")
+ tracer_note = f", {'; '.join(tracer_notes)}" if tracer_notes else ""
+ blocks_note = ""
+ if corpus.get("blocks_source"):
+ blocks_note = f", blocks {corpus.get('corpus_from')}..{corpus.get('corpus_to')} from {corpus['blocks_source']}"
+ if corpus.get("block_param") == "parent":
+ blocks_note += " at parent-block state"
+ lines.append(
+ f"- Corpus: `{corpus.get('name', 'static')}` ({corpus.get('source', 'static')}), "
+ f"class `{corpus.get('class', 'call')}`{top_gas_note}{tracer_note}{blocks_note}, {corpus.get('records', summary['records'])} records, "
+ f"methods: {methods}"
+ )
+ lines.append(
+ f"- Replay: {summary['rps']} rps for {summary['duration']}, "
+ f"{summary['passes']} passes x {summary['concurrency']} workers, "
+ f"{summary['warmup_seconds']}s warmup, "
+ f"max tracing requests: {summary.get('max_tracing_requests') or 'default'}"
+ )
+ if corpus.get("tip"):
+ lines.append(f"- Tip: {corpus['tip']} (`{corpus.get('tip_hash', 'unknown')}`)")
+ lines.append("")
+ if behind_baseline > 0:
+ plural = "s" if behind_baseline > 1 else ""
+ diff_link = f"https://github.com/{repo}/compare/{baseline_ref[:12]}...{baseline_name}"
+ lines.append(
+ f"> ⚠️ Feature is [**{behind_baseline} commit{plural} behind `{baseline_name}`**]({diff_link}). "
+ "Consider rebasing for accurate results."
+ )
+ lines.append("")
+ lines.append(generate_metric_table(summary, baseline_label, feature_label))
+ meta_parts = [
+ f"{summary['records']} records",
+ "mode: call",
+ f"{summary['run_pairs']} run pairs" if summary.get("run_pairs") else "",
+ ]
+ lines.append(f"*{', '.join(part for part in meta_parts if part)}*")
+ lines.append("")
+ lines.append(f"**Parity:** {summary['parity']['line']}")
+ method_table = generate_method_table(summary)
+ if method_table:
+ lines.append("")
+ lines.append(method_table)
+ lines.extend(generate_observability_section(summary))
+ return "\n".join(lines)
+
+
+def build_summary(args, baseline_runs: list[dict], feature_runs: list[dict]) -> dict:
+ rng = random.Random(42)
+ head_hash = assert_same_head(baseline_runs + feature_runs)
+
+ baseline_stats = arm_stats(baseline_runs)
+ feature_stats = arm_stats(feature_runs)
+ changes = compute_changes(rng, baseline_runs, feature_runs, baseline_stats, feature_stats)
+ parity = compute_parity(baseline_runs, feature_runs)
+ changes["parity"] = parity_change(parity)
+
+ corpus = {"name": "static", "source": "static", "class": args.klass}
+ if args.corpus_meta:
+ meta_path = Path(args.corpus_meta)
+ if meta_path.is_file():
+ with meta_path.open() as f:
+ corpus.update(json.load(f))
+ if head_hash and not corpus.get("tip_hash"):
+ corpus["tip_hash"] = head_hash
+
+ methods = corpus_methods(baseline_runs, feature_runs)
+ method_entries = {}
+ if len(methods) > 1:
+ for method in methods:
+ method_baseline = arm_stats(baseline_runs, method)
+ method_feature = arm_stats(feature_runs, method)
+ if not method_baseline or not method_feature:
+ continue
+ method_entries[method] = {
+ "baseline": method_baseline,
+ "feature": method_feature,
+ "changes": compute_changes(
+ rng, baseline_runs, feature_runs, method_baseline, method_feature, method
+ ),
+ "parity": parity["feature"]["methods"].get(
+ method,
+ {"matched": 0, "content_mismatch": 0, "kind_mismatch": 0, "missing": 0, "divergent_records": []},
+ ),
+ }
+
+ records = parity["records"] or corpus.get("records") or 0
+ summary = {
+ "mode": "call",
+ "blocks": records,
+ "records": records,
+ "corpus": corpus,
+ "replayed_methods": methods,
+ "rps": args.rps,
+ "duration": args.duration,
+ "passes": args.passes,
+ "concurrency": args.concurrency,
+ "warmup_seconds": args.warmup_seconds,
+ "run_pairs": args.run_pairs,
+ "baseline": {
+ "name": args.baseline_name or "baseline",
+ "ref": args.baseline_ref or "main",
+ "stats": baseline_stats,
+ },
+ "feature": {
+ "name": args.feature_name or "feature",
+ "ref": args.feature_ref or "unknown",
+ "stats": feature_stats,
+ },
+ "changes": changes,
+ "methods": method_entries,
+ "parity": parity,
+ }
+ if args.max_tracing_requests:
+ summary["max_tracing_requests"] = args.max_tracing_requests
+ if args.benchmark_id:
+ summary["benchmark_id"] = args.benchmark_id
+
+ from_ms, to_ms = resolve_observability_range(
+ args.observability_from_ms, args.observability_to_ms
+ )
+ logs_url = args.logs_url
+ traces_url = args.traces_url
+ if args.benchmark_id and from_ms and to_ms:
+ logs_url = logs_url or build_grafana_logs_url(args.benchmark_id, from_ms, to_ms)
+ traces_url = traces_url or build_grafana_traces_url(args.benchmark_id, from_ms, to_ms)
+ observability = {
+ key: value
+ for key, value in {
+ "benchmark_id": args.benchmark_id,
+ "from_ms": from_ms,
+ "to_ms": to_ms,
+ "grafana_url": args.grafana_url,
+ "logs_url": logs_url,
+ "traces_url": traces_url,
+ }.items()
+ if value
+ }
+ if observability:
+ summary["observability"] = observability
+ return summary
+
+
+def parse_args(argv: list[str] | None = None):
+ parser = argparse.ArgumentParser(description="Summarize an RPC replay benchmark")
+ parser.add_argument("--baseline-dir", nargs="+", required=True, help="Baseline run directories")
+ parser.add_argument("--feature-dir", nargs="+", required=True, help="Feature run directories")
+ parser.add_argument("--output-summary", required=True, help="Output JSON summary path")
+ parser.add_argument("--output-markdown", required=True, help="Output markdown path")
+ parser.add_argument("--repo", default="paradigmxyz/reth", help="GitHub repo (owner/name)")
+ parser.add_argument("--baseline-ref", default=None, help="Baseline commit SHA")
+ parser.add_argument("--baseline-name", default=None, help="Baseline display name")
+ parser.add_argument("--feature-name", default=None, help="Feature branch name")
+ parser.add_argument("--feature-ref", default=None, help="Feature commit SHA")
+ parser.add_argument("--behind-baseline", type=int, default=0, help="Commits behind baseline")
+ parser.add_argument("--corpus-meta", default=None, help="corpus.meta.json written by the extract step")
+ parser.add_argument("--class", dest="klass", default="call", help="Replayed method class")
+ parser.add_argument("--rps", default="", help="Configured open-loop request rate")
+ parser.add_argument("--duration", default="", help="Configured open-loop duration")
+ parser.add_argument("--passes", default="", help="Configured closed-loop passes")
+ parser.add_argument("--concurrency", default="", help="Configured closed-loop workers")
+ parser.add_argument("--warmup-seconds", default="", help="Warmup duration in seconds")
+ parser.add_argument("--run-pairs", type=int, default=None, help="Configured number of run pairs")
+ parser.add_argument("--max-tracing-requests", default=None, help="Node tracing request limit")
+ parser.add_argument("--benchmark-id", default=None, help="Benchmark ID used for OTLP labels")
+ parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL")
+ parser.add_argument("--logs-url", default=None, help="Grafana Explore URL for benchmark logs")
+ parser.add_argument("--traces-url", default=None, help="Grafana Explore URL for benchmark traces")
+ parser.add_argument("--observability-from-ms", type=int, default=None, help="Observability range start")
+ parser.add_argument("--observability-to-ms", type=int, default=None, help="Observability range end")
+ parser.add_argument(
+ "--derek-command",
+ default=os.environ.get("DEREK_BENCH_COMMAND"),
+ help="Full derek bench command",
+ )
+ return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = parse_args(argv)
+ if args.run_pairs is not None and args.run_pairs < 1:
+ print("--run-pairs must be greater than zero", file=sys.stderr)
+ return 1
+ baseline_runs = [load_run(Path(path)) for path in args.baseline_dir]
+ feature_runs = [load_run(Path(path)) for path in args.feature_dir]
+
+ summary = build_summary(args, baseline_runs, feature_runs)
+
+ base_url = f"https://github.com/{args.repo}/commit"
+ baseline_label = f"[`{summary['baseline']['name']}`]({base_url}/{summary['baseline']['ref']})"
+ feature_label = f"[`{summary['feature']['name']}`]({base_url}/{summary['feature']['ref']})"
+ markdown = generate_markdown(
+ summary,
+ baseline_label,
+ feature_label,
+ behind_baseline=args.behind_baseline,
+ repo=args.repo,
+ baseline_ref=summary["baseline"]["ref"],
+ baseline_name=summary["baseline"]["name"],
+ derek_command=args.derek_command,
+ )
+
+ with open(args.output_summary, "w") as f:
+ json.dump(summary, f, indent=2)
+ print(f"Summary written to {args.output_summary}")
+ with open(args.output_markdown, "w") as f:
+ f.write(markdown)
+ print(f"Markdown written to {args.output_markdown}")
+ print(f"Parity: {summary['parity']['line']}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/bench-job-summary.js b/.github/scripts/bench-job-summary.js
index e79085eb2fd..129507b7afa 100644
--- a/.github/scripts/bench-job-summary.js
+++ b/.github/scripts/bench-job-summary.js
@@ -82,6 +82,11 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl,
}
md += '\n';
+ // Response parity (call mode)
+ if (summary.parity) {
+ md += `**Parity:** ${summary.parity.line}\n\n`;
+ }
+
// Wait time breakdown
const wtRows = waitTimeRows(summary);
if (wtRows.length > 0) {
@@ -111,8 +116,8 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl,
md += '\n';
}
- // Charts
- if (chartSha) {
+ // Charts (call mode does not generate any)
+ if (chartSha && summary.mode !== 'call') {
const prNum = prNumber || '0';
const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/pr/${prNum}/${runId}`;
const charts = [
diff --git a/.github/scripts/bench-txgen-extract.sh b/.github/scripts/bench-txgen-extract.sh
index a513dcfd1d4..8d1c9953cbc 100755
--- a/.github/scripts/bench-txgen-extract.sh
+++ b/.github/scripts/bench-txgen-extract.sh
@@ -15,8 +15,28 @@
# /benchmark-blocks.ndjson
# In RPC mode, the corresponding names end in `-transactions.ndjson`.
#
+# In call mode it instead writes a request corpus replayed by `bench call`:
+# /corpus.jsonl[.gz] one JSON-RPC request per line
+# /corpus.meta.json record counts and the tip it was built for
+#
+# Corpus sources
+# --------------
+# `corpus=static` generates the corpus from the chain around the snapshot tip
+# for the requested class. `corpus=` replays a corpus staged on the runner
+# under BENCH_CORPUS_DIR (default /reth-bench/corpora), which holds:
+#
+# .jsonl.gz the corpus itself, gzip optional
+# .sha256 optional checksum, verified before the corpus is used
+# tracers/.js JavaScript tracers selectable as tracer=js:
+#
+# Provisioning that directory is manual (copy the file onto the runner). Names
+# must not contain path separators. Corpus contents are never logged and never
+# uploaded as build artifacts; only the record counts in corpus.meta.json are.
+#
# Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS
-# Optional env: BENCH_EXECUTION_MODE, BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL
+# Optional env: BENCH_EXECUTION_MODE, BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL,
+# BENCH_CORPUS, BENCH_CORPUS_DIR, BENCH_CALL_CLASS, BENCH_CALL_METHODS, BENCH_CALL_TOP_GAS,
+# BENCH_CALL_TRACER, BENCH_CALL_TRACER_CONFIG, BENCH_CALL_TRACE_OPTIONS, BENCH_CALL_NAMESPACE
set -euxo pipefail
BINARY="$1"
@@ -69,6 +89,11 @@ RETH_ARGS=(
--no-persist-peers
)
+# Call mode probes the snapshot's tracing range before the node is stopped.
+if [ "$EXECUTION_MODE" = "call" ]; then
+ RETH_ARGS+=(--http.api "eth,net,web3,debug,trace")
+fi
+
if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then
RETH_ARGS+=(--debug.startup-sync-state-idle)
fi
@@ -104,12 +129,304 @@ HEAD_HEX=$(jq -r '.result' <<< "$HEAD_JSON")
HEAD_DEC=$((16#${HEAD_HEX#0x}))
echo "Snapshot chain tip: ${HEAD_DEC}"
-# Stop the throwaway node and recover snapshot before extraction.
-sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true
-sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true
-kill "${TAIL_PID:-}" 2>/dev/null || true
-TAIL_PID=
-sudo schelk recover -y --kill || true
+HEAD_HASH=$(curl -sf http://127.0.0.1:8545 -X POST \
+ -H 'Content-Type: application/json' \
+ -d '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}' \
+ | jq -r '.result.hash')
+
+# Returns non-zero when the snapshot can no longer trace the given block, which
+# happens once the block falls out of the retained account and storage history.
+# The archive node serves the blocks after the snapshot tip for the call
+# classes; probe it once so an outage falls back instead of failing the run.
+archive_rpc_reachable() {
+ local response
+ response=$(curl -sf -m 15 "$BENCH_RPC_URL" -X POST -H 'Content-Type: application/json' \
+ --data '{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}' 2>/dev/null) || return 1
+ [[ "$response" == *'"result"'* ]]
+}
+
+block_is_traceable() {
+ local block="$1"
+ local block_hex tx_hash response
+ block_hex=$(printf '0x%x' "$block")
+ tx_hash=$(curl -sf http://127.0.0.1:8545 -X POST \
+ -H 'Content-Type: application/json' \
+ -d "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"${block_hex}\",false],\"id\":1}" \
+ | jq -r '.result.transactions[0] // empty')
+ if [ -z "$tx_hash" ]; then
+ echo "Block ${block} has no transactions, skipping traceability probe"
+ return 0
+ fi
+ # The block class never resolves hashes, so it probes the block-addressed
+ # method it will replay; the transaction class probes by hash, which also
+ # detects a pruned transaction lookup.
+ local probe
+ if [ "$CALL_CLASS" = "traceblock" ]; then
+ probe="{\"jsonrpc\":\"2.0\",\"method\":\"debug_traceBlockByNumber\",\"params\":[\"${block_hex}\",{\"tracer\":\"callTracer\",\"tracerConfig\":{\"onlyTopCall\":true}}],\"id\":1}"
+ else
+ probe="{\"jsonrpc\":\"2.0\",\"method\":\"debug_traceTransaction\",\"params\":[\"${tx_hash}\",{\"tracer\":\"callTracer\"}],\"id\":1}"
+ fi
+ if ! response=$(curl -sf http://127.0.0.1:8545 -X POST \
+ -H 'Content-Type: application/json' \
+ -d "$probe"); then
+ echo "traceability probe request for block ${block} failed"
+ return 1
+ fi
+ if jq -e 'has("error")' <<< "$response" > /dev/null 2>&1; then
+ local message
+ message="$(jq -r '.error.message' <<< "$response")"
+ # A transaction the node just listed in the block but cannot find by hash
+ # means the transaction lookup index is pruned, so no block in the range
+ # will do any better; report that instead of shrinking the range.
+ if [[ "$message" == *"transaction not found"* ]]; then
+ echo "::error::Snapshot has no transaction hash index (transaction lookup is pruned); hash-addressed classes cannot run on it. Use class=tracecall or class=traceblock, or a snapshot that keeps the transaction lookup."
+ exit 1
+ fi
+ echo "Block ${block} is not traceable: ${message}"
+ return 1
+ fi
+ return 0
+}
+
+CALL_BLOCKS="${BENCH_BLOCKS:-20}"
+CALL_CLASS="${BENCH_CALL_CLASS:-call}"
+CORPUS_SOURCE="${BENCH_CORPUS:-static}"
+if [ "$EXECUTION_MODE" = "call" ] && [ "$CORPUS_SOURCE" = "static" ]; then
+ case "$CALL_CLASS" in
+ tracetx|traceblock)
+ # The snapshot is a pruned full node whose persisted state can lag its
+ # header tip, so the newest blocks may trace against stale state ("nonce
+ # too high"). Walk down from the tip to the newest block that actually
+ # traces, then window below it; also shrink the window if its oldest
+ # block has fallen past the retained history floor.
+ CORPUS_TOP="$HEAD_DEC"
+ probe_budget="${BENCH_TRACE_PROBE_DEPTH:-1024}"
+ while ! block_is_traceable "$CORPUS_TOP"; do
+ probe_budget=$(( probe_budget - 1 ))
+ if [ "$probe_budget" -le 0 ] || [ "$CORPUS_TOP" -le 1 ]; then
+ echo "::error::Snapshot has no traceable block near its tip ${HEAD_DEC} (searched down to ${CORPUS_TOP}); its persisted state lags too far behind the header for class ${CALL_CLASS}."
+ exit 1
+ fi
+ CORPUS_TOP=$(( CORPUS_TOP - 1 ))
+ done
+ if [ "$CORPUS_TOP" -ne "$HEAD_DEC" ]; then
+ echo "Newest traceable block is ${CORPUS_TOP}, $(( HEAD_DEC - CORPUS_TOP )) below the tip"
+ fi
+ while [ "$CALL_BLOCKS" -gt 1 ] && ! block_is_traceable $(( CORPUS_TOP - CALL_BLOCKS + 1 )); do
+ CALL_BLOCKS=$(( CALL_BLOCKS / 2 ))
+ echo "Retrying traceability probe with ${CALL_BLOCKS} source blocks"
+ done
+ echo "Snapshot traces blocks $(( CORPUS_TOP - CALL_BLOCKS + 1 ))..${CORPUS_TOP}"
+ ;;
+ esac
+fi
+
+# Stop the throwaway node and recover the snapshot before extraction. Call mode
+# keeps it up: the trace classes read their blocks from it, and the exit trap
+# stops it once the corpus is staged.
+if [ "$EXECUTION_MODE" != "call" ]; then
+ sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true
+ sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true
+ kill "${TAIL_PID:-}" 2>/dev/null || true
+ TAIL_PID=
+ sudo schelk recover -y --kill || true
+fi
+
+# --- Stage the call corpus ---
+
+if [ "$EXECUTION_MODE" = "call" ]; then
+ CALL_METHODS="${BENCH_CALL_METHODS:-}"
+ CORPUS_FILE="$OUTPUT_DIR/corpus.jsonl"
+
+ read_corpus() {
+ case "$1" in
+ *.gz) gzip -dc "$1" ;;
+ *) cat "$1" ;;
+ esac
+ }
+
+ if [ "$CORPUS_SOURCE" = "static" ]; then
+ TXGEN_ETHEREUM="$(which txgen-ethereum)"
+ if [ "$CALL_BLOCKS" -le 0 ]; then
+ echo "::error::BENCH_BLOCKS must be greater than 0"
+ exit 1
+ fi
+ case "$CALL_CLASS" in
+ call|tracecall)
+ # Transactions of the blocks after the tip are exactly the calls a
+ # client would have served against the tip state; only the archive
+ # node has those blocks. When it is unreachable, the blocks just below
+ # the tip on the snapshot stand in: their calls replay against the tip
+ # state too, only with more reverts.
+ CORPUS_FORMAT=calls
+ if archive_rpc_reachable; then
+ CORPUS_FROM=$(( HEAD_DEC + 1 ))
+ CORPUS_TO=$(( HEAD_DEC + CALL_BLOCKS ))
+ CORPUS_RPC="$BENCH_RPC_URL"
+ CORPUS_BLOCKS_SOURCE=archive
+ else
+ echo "::warning::Archive node ${BENCH_RPC_URL} is unreachable; building the ${CALL_CLASS} corpus from snapshot blocks below the tip"
+ CORPUS_FROM=$(( HEAD_DEC - CALL_BLOCKS + 1 ))
+ CORPUS_TO="$HEAD_DEC"
+ CORPUS_RPC="http://127.0.0.1:8545"
+ CORPUS_BLOCKS_SOURCE=snapshot
+ # These transactions already executed, so replay each against the
+ # state before its block instead of the tip, where its preconditions
+ # no longer hold.
+ CORPUS_BLOCK_PARAM=parent
+ fi
+ ;;
+ tracetx|traceblock)
+ # These blocks are on the snapshot, so the throwaway node serves them
+ # and the archive node is not needed. CORPUS_TOP is the newest block the
+ # probe found traceable, which may sit below the header tip.
+ CORPUS_FROM=$(( CORPUS_TOP - CALL_BLOCKS + 1 ))
+ CORPUS_TO="$CORPUS_TOP"
+ CORPUS_FORMAT=traces
+ CORPUS_RPC="http://127.0.0.1:8545"
+ CORPUS_BLOCKS_SOURCE=snapshot
+ ;;
+ *)
+ echo "::error::Unknown call class: ${CALL_CLASS}"
+ exit 1
+ ;;
+ esac
+ CORPUS_METHOD_ARGS=()
+ if [ -n "$CALL_METHODS" ]; then
+ CORPUS_METHOD_ARGS+=(--methods "$CALL_METHODS")
+ fi
+ if [ -n "${BENCH_CALL_TOP_GAS:-}" ]; then
+ CORPUS_METHOD_ARGS+=(--top-gas "$BENCH_CALL_TOP_GAS")
+ fi
+ if [ -n "${BENCH_CALL_TRACER:-}" ]; then
+ IFS=',' read -r -a tracer_specs <<< "$BENCH_CALL_TRACER"
+ for spec in "${tracer_specs[@]}"; do
+ spec="${spec// /}"
+ [ -z "$spec" ] && continue
+ case "$spec" in
+ js:*)
+ tracer_name="${spec#js:}"
+ case "$tracer_name" in
+ */*|.*|"")
+ echo "::error::Invalid tracer name: ${spec}"
+ exit 1
+ ;;
+ esac
+ tracer_file="${BENCH_CORPUS_DIR:-/reth-bench/corpora}/tracers/${tracer_name}.js"
+ if [ ! -f "$tracer_file" ]; then
+ echo "::error::JS tracer ${tracer_name} not found at ${tracer_file}"
+ exit 1
+ fi
+ CORPUS_METHOD_ARGS+=(--tracer "js:${tracer_file}")
+ ;;
+ *)
+ CORPUS_METHOD_ARGS+=(--tracer "$spec")
+ ;;
+ esac
+ done
+ fi
+ if [ -n "${BENCH_CALL_TRACER_CONFIG:-}" ]; then
+ CORPUS_METHOD_ARGS+=(--tracer-config "$BENCH_CALL_TRACER_CONFIG")
+ fi
+ if [ -n "${CORPUS_BLOCK_PARAM:-}" ]; then
+ CORPUS_METHOD_ARGS+=(--block-param "$CORPUS_BLOCK_PARAM")
+ fi
+ if [ -n "${BENCH_CALL_TRACE_OPTIONS:-}" ]; then
+ CORPUS_METHOD_ARGS+=(--trace-options "$BENCH_CALL_TRACE_OPTIONS")
+ fi
+ echo "Generating ${CALL_CLASS} corpus from blocks ${CORPUS_FROM}..${CORPUS_TO} (methods: ${CALL_METHODS:-default})"
+ "$TXGEN_ETHEREUM" extract \
+ --rpc "$CORPUS_RPC" \
+ --from "$CORPUS_FROM" \
+ --to "$CORPUS_TO" \
+ --format "$CORPUS_FORMAT" \
+ "${CORPUS_METHOD_ARGS[@]}" \
+ -o "$CORPUS_FILE"
+ else
+ case "$CORPUS_SOURCE" in
+ */*|.*)
+ echo "::error::Invalid corpus name: ${CORPUS_SOURCE}"
+ exit 1
+ ;;
+ esac
+ CORPUS_DIR="${BENCH_CORPUS_DIR:-/reth-bench/corpora}"
+ CORPUS_SRC=""
+ for candidate in "$CORPUS_DIR/$CORPUS_SOURCE.jsonl.gz" "$CORPUS_DIR/$CORPUS_SOURCE.jsonl"; do
+ if [ -f "$candidate" ]; then
+ CORPUS_SRC="$candidate"
+ break
+ fi
+ done
+ if [ -z "$CORPUS_SRC" ]; then
+ echo "::error::Corpus ${CORPUS_SOURCE} not found in ${CORPUS_DIR} (expected ${CORPUS_SOURCE}.jsonl.gz or ${CORPUS_SOURCE}.jsonl)"
+ exit 1
+ fi
+ CORPUS_SHA256="$CORPUS_DIR/$CORPUS_SOURCE.sha256"
+ if [ -f "$CORPUS_SHA256" ]; then
+ EXPECTED_SHA="$(awk '{print $1; exit}' "$CORPUS_SHA256")"
+ ACTUAL_SHA="$(sha256sum "$CORPUS_SRC" | awk '{print $1}')"
+ if [ "$EXPECTED_SHA" != "$ACTUAL_SHA" ]; then
+ echo "::error::Checksum mismatch for corpus ${CORPUS_SOURCE}"
+ exit 1
+ fi
+ echo "Corpus ${CORPUS_SOURCE} checksum verified"
+ fi
+ case "$CORPUS_SRC" in
+ *.gz) CORPUS_FILE="$OUTPUT_DIR/corpus.jsonl.gz" ;;
+ esac
+ cp "$CORPUS_SRC" "$CORPUS_FILE"
+ fi
+
+ CORPUS_RECORDS=$(read_corpus "$CORPUS_FILE" | wc -l | tr -d ' ')
+ if [ "$CORPUS_RECORDS" -le 0 ]; then
+ echo "::error::Corpus ${CORPUS_SOURCE} is empty"
+ exit 1
+ fi
+ CORPUS_RECORDS_PER_METHOD=$(read_corpus "$CORPUS_FILE" | jq -r '.method' | sort | uniq -c \
+ | jq -R -s 'split("\n")
+ | map(select(length > 0) | capture("^\\s*(?\\d+)\\s+(?.+)$") | {(.method): (.count | tonumber)})
+ | add // {}')
+ jq -n \
+ --arg source "$CORPUS_SOURCE" \
+ --arg name "$CORPUS_SOURCE" \
+ --arg class "$CALL_CLASS" \
+ --arg methods "$CALL_METHODS" \
+ --argjson records "$CORPUS_RECORDS" \
+ --argjson records_per_method "$CORPUS_RECORDS_PER_METHOD" \
+ --argjson tip "$HEAD_DEC" \
+ --arg tip_hash "$HEAD_HASH" \
+ --arg blocks_source "${CORPUS_BLOCKS_SOURCE:-}" \
+ --arg block_param "${CORPUS_BLOCK_PARAM:-}" \
+ --argjson corpus_from "${CORPUS_FROM:-null}" \
+ --argjson corpus_to "${CORPUS_TO:-null}" \
+ --argjson top_gas "${BENCH_CALL_TOP_GAS:-null}" \
+ --arg tracer "${BENCH_CALL_TRACER:-}" \
+ --arg namespace "${BENCH_CALL_NAMESPACE:-}" \
+ --argjson tracer_config "${BENCH_CALL_TRACER_CONFIG:-null}" \
+ --argjson trace_options "${BENCH_CALL_TRACE_OPTIONS:-null}" \
+ '{
+ source: (if $source == "static" then "static" else "custom" end),
+ name: $name,
+ class: $class,
+ methods: (if $methods == "" then [] else ($methods | split(",")) end),
+ records: $records,
+ records_per_method: $records_per_method,
+ tip: $tip,
+ tip_hash: $tip_hash,
+ blocks_source: (if $blocks_source == "" then null else $blocks_source end),
+ block_param: (if $block_param == "" then null else $block_param end),
+ corpus_from: $corpus_from,
+ corpus_to: $corpus_to,
+ top_gas: $top_gas,
+ tracer: (if $tracer == "" then null else ($tracer | split(",")) end),
+ namespace: (if $namespace == "" then null else $namespace end),
+ tracer_config: $tracer_config,
+ trace_options: $trace_options,
+ }' > "$OUTPUT_DIR/corpus.meta.json"
+
+ echo "Corpus staged: ${CORPUS_RECORDS} records in ${CORPUS_FILE}"
+ exit 0
+fi
# --- Extract payloads from the remote RPC ---
diff --git a/.github/scripts/bench-txgen-install.sh b/.github/scripts/bench-txgen-install.sh
index 6ac1d31897c..366a01d01eb 100755
--- a/.github/scripts/bench-txgen-install.sh
+++ b/.github/scripts/bench-txgen-install.sh
@@ -4,6 +4,7 @@
#
# Optional env:
# TXGEN_REPO – txgen repository URL (default: https://github.com/tempoxyz/txgen)
+# TXGEN_REF – branch or commit to install instead of the default branch
set -euxo pipefail
TXGEN_REPO="${TXGEN_REPO:-https://github.com/tempoxyz/txgen}"
@@ -31,4 +32,14 @@ elif [ -n "${TXGEN_TOKEN:-${GH_PROJECT_TOKEN:-${DEREK_PAT:-${DEREK_TOKEN:-}}}}"
fi
export CARGO_NET_GIT_FETCH_WITH_CLI=true
-cargo install --git "$TXGEN_REPO" --locked txgen-ethereum bench-cli
+INSTALL_ARGS=()
+if [ -n "${TXGEN_REF:-}" ]; then
+ if [[ "$TXGEN_REF" =~ ^[0-9a-f]{7,40}$ ]]; then
+ INSTALL_ARGS+=(--rev "$TXGEN_REF")
+ else
+ INSTALL_ARGS+=(--branch "$TXGEN_REF")
+ fi
+ # A pinned ref must replace whatever build of the same version is installed.
+ INSTALL_ARGS+=(--force)
+fi
+cargo install --git "$TXGEN_REPO" ${INSTALL_ARGS[@]+"${INSTALL_ARGS[@]}"} --locked txgen-ethereum bench-cli
diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh
index b709cf7c544..902f4ebb174 100755
--- a/.github/scripts/bench-txgen-run.sh
+++ b/.github/scripts/bench-txgen-run.sh
@@ -4,10 +4,16 @@
# mount snapshot → start node → extract source blocks → warmup → replay →
# convert txgen JSON report into legacy benchmark CSVs.
#
+# In call mode it replays the request corpus staged by bench-txgen-extract.sh
+# instead, and writes responses.ndjson, record_timings.csv, requests.csv,
+# report.json and cpu.json for bench-call-summary.py.
+#
# Usage: bench-txgen-run.sh