diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..4ed70d500ab --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,243 @@ +version: 2.1 + +# Runs this fork's validation workflows through pinned act because GitHub Actions is disabled. +# Pull requests and branch pushes run the PR checks; merge queues add stage.yml. Event metadata is +# generated once per CircleCI revision and shared by all shards. +# +# The read-only GitHub token is exposed to PR code, so its CircleCI context must remain +# restricted to trusted contributors. Keep CircleCI auto-cancellation enabled. +# +# The act binary, checksum, and runner image are pinned. Compiler caches are no-ops because +# act's local cache is not reusable across these machines; Pages plumbing is skipped after the +# validation build. Lint and unit jobs are distributed; other workflows run up to four jobs. +# +# Excluded: pr-title.yml, label-pr.yml, and the privileged cyclops-audit.yml integration. + +jobs: + # Replaces lint.yml's `deny` job, which calls tempoxyz/ci's reusable deny.yml. That workflow starts + # with tempoxyz's secure-runner, which needs a GitHub OIDC token that act never receives, so the + # check itself runs natively here. Its policy comes from deny.toml in this repo. + # + # When re-applying this CI patch onto a new reth release, diff lint.yml's `deny:` job and the + # tempoxyz/ci deny.yml it calls against this job (toolchain, cargo-deny version, `check` + # arguments, deny flags), and update both pins here. + cargo-deny: + docker: + - image: rust:1.95-bookworm@sha256:6258907abe69656e41cd992e0b705cdcfabcbbe3db374f92ed2d47121282d4a1 + resource_class: medium + steps: + - checkout + - run: + name: Install pinned cargo-deny + command: | + set -euo pipefail + readonly DENY_VERSION="0.20.2" + readonly DENY_SHA256="9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f" + readonly DENY_ARCHIVE="cargo-deny-${DENY_VERSION}-x86_64-unknown-linux-musl.tar.gz" + curl --fail --silent --show-error --location \ + "https://github.com/EmbarkStudios/cargo-deny/releases/download/${DENY_VERSION}/${DENY_ARCHIVE}" \ + --output "/tmp/${DENY_ARCHIVE}" + echo "${DENY_SHA256} /tmp/${DENY_ARCHIVE}" | sha256sum --check + tar -xzf "/tmp/${DENY_ARCHIVE}" -C /usr/local/bin --strip-components=1 \ + "cargo-deny-${DENY_VERSION}-x86_64-unknown-linux-musl/cargo-deny" + test "$(cargo-deny --version)" = "cargo-deny ${DENY_VERSION}" + - run: + name: cargo deny check + command: cargo deny --log-level warn --all-features check all + + prepare-act-event: + machine: + image: ubuntu-2404:2025.09.1 + resource_class: medium + steps: + - checkout + - run: + name: Snapshot GitHub event + command: | + mkdir -p /tmp/act-event + if ! .circleci/prepare-act-event.sh; then + echo "GitHub event preparation failed; act shards will report the failure" | + tee /tmp/act-event/failed + fi + - persist_to_workspace: + root: /tmp/act-event + paths: + - . + + act: + parameters: + wf: + type: string + job: + type: string + default: "" + concurrent_jobs: + type: integer + default: 4 + machine: + image: ubuntu-2404:2025.09.1 + resource_class: xlarge + steps: + - checkout + - attach_workspace: + at: /tmp/act-event + - run: + name: Check event snapshot + command: | + if [ -f /tmp/act-event/failed ]; then + cat /tmp/act-event/failed >&2 + exit 1 + fi + - run: + name: Install pinned act + command: | + set -euo pipefail + readonly ACT_VERSION="0.2.89-action-cache-fix.1" + readonly ACT_RELEASE="v0.2.89-action-cache-fix.4" + readonly ACT_BINARY="act_Linux_x86_64" + readonly ACT_SHA256="28bfcf1502f59a834b14f2fb23c39e5b253c2234d875592b5aa0a175f9674dce" + curl --fail --silent --show-error --location \ + "https://github.com/ethereum-optimism/act/releases/download/${ACT_RELEASE}/${ACT_BINARY}" \ + --output "/tmp/${ACT_BINARY}" + echo "${ACT_SHA256} /tmp/${ACT_BINARY}" | sha256sum --check + sudo install -m 0755 "/tmp/${ACT_BINARY}" /usr/local/bin/act + test "$(act --version)" = "act version ${ACT_VERSION}" + - run: + name: Stub unsupported non-check actions + command: | + mkdir -p /tmp/noop-action + printf '%s\n' \ + 'name: noop' \ + 'description: disabled service plumbing under act' \ + 'inputs:' \ + ' cache-bin: {required: false}' \ + ' cache-on-failure: {required: false}' \ + ' cache-targets: {required: false}' \ + ' prefix-key: {required: false}' \ + 'runs:' \ + ' using: composite' \ + ' steps:' \ + ' - run: echo "Service plumbing disabled under act"' \ + ' shell: bash' \ + > /tmp/noop-action/action.yml + - run: + name: Run << parameters.wf >> via act + no_output_timeout: 45m + command: | + set -euo pipefail + event_path=/tmp/act-event/github-event.json + event_name="$(jq -er 'if .pull_request then "pull_request" elif .merge_group then "merge_group" elif .ref and .after then "push" else error("unsupported event") end' "$event_path")" + base_ref="$(jq -r '(.pull_request.base.ref // .merge_group.base_ref // "") | sub("^refs/heads/"; "")' "$event_path")" + target_job="<< parameters.job >>" + # Validate documentation without entering Pages deployment on branch pushes. + if [ "<< parameters.wf >>" = "book.yml" ]; then + target_job=build + fi + act_job=() + if [ -n "$target_job" ]; then + case "$target_job" in + crate-checks-*) + partition="${target_job#crate-checks-}" + if [[ ! "$partition" =~ ^[1-9][0-9]*$ ]]; then + echo "invalid crate-checks partition: $partition" >&2 + exit 1 + fi + act_job=(-j crate-checks --matrix "partition:${partition}") + ;; + *) + act_job=(-j "$target_job") + ;; + esac + fi + + act_env=(--env "RUSTC_WRAPPER=" --env "GITHUB_BASE_REF=${base_ref}") + if [ "<< parameters.wf >>" = "unit.yml" ] && [ "$target_job" = "state" ]; then + # EF tests use Rayon internally, so unrestricted nextest processes multiply + # runnable threads and starve valid_blocks past its upstream timeout. + act_env+=(--env "NEXTEST_TEST_THREADS=2") + fi + if [ "$event_name" != "pull_request" ]; then + act_env+=( + --env "GITHUB_REF=refs/heads/${CIRCLE_BRANCH}" + --env "GITHUB_SHA=${CIRCLE_SHA1}" + ) + fi + + act "$event_name" -W ".github/workflows/<< parameters.wf >>" \ + "${act_job[@]}" \ + --eventpath "$event_path" \ + -P "ubuntu-latest=catthehacker/ubuntu@sha256:b839c14c4410998529ec18f951262bdf87a2b23bc1467304d07b491b9455e074" \ + --concurrent-jobs << parameters.concurrent_jobs >> \ + "${act_env[@]}" \ + -s GITHUB_TOKEN="${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \ + --local-repository "mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba=/tmp/noop-action" \ + --local-repository "Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6=/tmp/noop-action" \ + --local-repository "actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d=/tmp/noop-action" \ + --local-repository "actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9=/tmp/noop-action" + +workflows: + reth-act: + jobs: + - prepare-act-event: + context: + - circleci-repo-readonly-authenticated-github-token + - act: + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-<< matrix.wf >>" + requires: + - prepare-act-event + matrix: + parameters: + wf: + - book.yml + - compact.yml + - e2e.yml + - grafana.yml + - integration.yml + - lint-actions.yml + - stage.yml + - act: + wf: lint.yml + concurrent_jobs: 1 + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-lint-<< matrix.job >>" + requires: + - prepare-act-event + matrix: + parameters: + job: + - clippy-binaries + - clippy + - wasm + - riscv + - crate-checks-1 + - crate-checks-2 + - crate-checks-3 + - msrv + - docs + - fmt + - udeps + - book + - typos + - check-toml + - grafana + - no-test-deps + - feature-propagation + - cargo-deny: + name: lint-deny + - act: + wf: unit.yml + concurrent_jobs: 1 + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-unit-<< matrix.job >>" + requires: + - prepare-act-event + matrix: + parameters: + job: + - test + - state + - doc diff --git a/.circleci/prepare-act-event.sh b/.circleci/prepare-act-event.sh new file mode 100755 index 00000000000..99cccb27f3c --- /dev/null +++ b/.circleci/prepare-act-event.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly OUTPUT_DIR="${ACT_EVENT_OUTPUT_DIR:-/tmp/act-event}" +readonly EVENT_PATH="${OUTPUT_DIR}/github-event.json" + +mkdir -p "$OUTPUT_DIR" +rm -f "$EVENT_PATH" + +: "${CIRCLE_SHA1:?missing CircleCI revision}" +: "${CIRCLE_PROJECT_USERNAME:?missing CircleCI project owner}" +: "${CIRCLE_PROJECT_REPONAME:?missing CircleCI project repository}" + +repository="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" + +if [[ "${CIRCLE_BRANCH:-}" == gh-readonly-queue/* ]]; then + event_name="merge_group" +elif [[ -n "${CIRCLE_PULL_REQUEST:-}" ]]; then + event_name="pull_request" +else + event_name="push" +fi + +case "$event_name" in + pull_request) + : "${CIRCLE_PULL_REQUEST:?pull-request trigger must provide CIRCLE_PULL_REQUEST}" + pr_number="${CIRCLE_PULL_REQUEST##*/}" + if [[ ! "$pr_number" =~ ^[1-9][0-9]*$ ]]; then + echo "invalid pull-request URL: ${CIRCLE_PULL_REQUEST}" >&2 + exit 1 + fi + + pull_request_path="${OUTPUT_DIR}/pull-request.json" + curl --fail --silent --show-error --location \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${repository}/pulls/${pr_number}" > "$pull_request_path" + + event_head_sha="$(jq -er '.head.sha' "$pull_request_path")" + if [[ "$event_head_sha" != "$CIRCLE_SHA1" ]]; then + echo "PR head does not match CircleCI revision: ${CIRCLE_SHA1} != ${event_head_sha}" >&2 + exit 1 + fi + + jq '{ + action: "synchronize", + number: .number, + pull_request: ., + repository: .base.repo, + sender: .user + }' "$pull_request_path" > "$EVENT_PATH" + base_ref="$(jq -er '.base.ref | select(length > 0)' "$pull_request_path")" + head_ref="$(jq -er '.head.ref | select(length > 0)' "$pull_request_path")" + echo "act event: pull_request #${pr_number}, ${head_ref} -> ${base_ref}" + ;; + push) + : "${CIRCLE_BRANCH:?push trigger must provide CIRCLE_BRANCH}" + jq -n \ + --arg ref "refs/heads/${CIRCLE_BRANCH}" \ + --arg after "$CIRCLE_SHA1" \ + '{ref: $ref, after: $after}' > "$EVENT_PATH" + echo "act event: push ${CIRCLE_BRANCH} at ${CIRCLE_SHA1}" + ;; + merge_group) + : "${CIRCLE_BRANCH:?merge-queue trigger must provide CIRCLE_BRANCH}" + if [[ ! "$CIRCLE_BRANCH" =~ ^gh-readonly-queue/(.+)/pr-([1-9][0-9]*)-[^/]+$ ]]; then + echo "merge_group requires a gh-readonly-queue//pr-- branch, got: ${CIRCLE_BRANCH}" >&2 + exit 1 + fi + base_ref="${BASH_REMATCH[1]}" + + if ! base_sha="$(git rev-parse "${CIRCLE_SHA1}^1" 2>/dev/null)"; then + git fetch --no-tags --depth=2 origin "$CIRCLE_SHA1" + base_sha="$(git rev-parse "${CIRCLE_SHA1}^1")" + fi + + jq -n \ + --arg head_sha "$CIRCLE_SHA1" \ + --arg head_ref "refs/heads/${CIRCLE_BRANCH}" \ + --arg base_sha "$base_sha" \ + --arg base_ref "refs/heads/${base_ref}" \ + --arg repository "$repository" \ + --arg repository_name "$CIRCLE_PROJECT_REPONAME" \ + --arg repository_owner "$CIRCLE_PROJECT_USERNAME" \ + '{ + action: "checks_requested", + merge_group: { + head_sha: $head_sha, + head_ref: $head_ref, + base_sha: $base_sha, + base_ref: $base_ref + }, + repository: { + full_name: $repository, + name: $repository_name, + owner: {login: $repository_owner} + } + }' > "$EVENT_PATH" + echo "act event: merge_group ${CIRCLE_BRANCH} -> ${base_ref} at ${base_sha}" + ;; +esac diff --git a/.config/nextest.toml b/.config/nextest.toml index 01cb0158619..d07545e2400 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -6,6 +6,12 @@ slow-timeout = { period = "30s", terminate-after = 2 } filter = "test(general_state_tests)" slow-timeout = { period = "1m", terminate-after = 10 } +# valid_blocks covers a full fixture directory, like the grouped state tests. +# Keep its CircleCI runtime bounded by the same ten-minute budget. +[[profile.default.overrides]] +filter = "package(=ef-tests) and test(=valid_blocks)" +slow-timeout = { period = "1m", terminate-after = 10 } + [[profile.default.overrides]] filter = "test(eest_fixtures)" slow-timeout = { period = "2m", terminate-after = 10 } diff --git a/.config/zepter.yaml b/.config/zepter.yaml index 7aa1bd725ef..d2d5dfb5255 100644 --- a/.config/zepter.yaml +++ b/.config/zepter.yaml @@ -12,7 +12,7 @@ workflows: # Check that `A` activates the features of `B`. "propagate-feature", # These are the features to check: - "--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global,trie-debug,secp256k1", + "--features=std,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global,trie-debug,secp256k1", # Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually. "--left-side-feature-missing=ignore", # Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on. diff --git a/.dockerignore b/.dockerignore index f4baa18a38b..e9d3ac4167b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,7 +7,6 @@ !/pkg !/testing !/.github/scripts/install_llvm_ubuntu.sh -!book.toml !Cargo.lock !Cargo.toml !Cross.toml diff --git a/.github/scripts/bench-call-summary.py b/.github/scripts/bench-call-summary.py new file mode 100755 index 00000000000..49eeba43069 --- /dev/null +++ b/.github/scripts/bench-call-summary.py @@ -0,0 +1,1097 @@ +#!/usr/bin/env python3 +"""Summarize an RPC replay benchmark and generate summary JSON + markdown. + +Usage: + bench-call-summary.py \ + --output-summary \ + --output-markdown \ + --baseline-dir [ ...] \ + --feature-dir [ ...] + +Each run directory holds the output of one `bench call` measured phase: + + report.json run metadata, identity and per-method aggregates + requests.csv open-loop, one row per request + record_timings.csv closed-loop, one row per record and pass + responses.ndjson one response digest per corpus record + cpu.json CPU time the node's cgroup spent on the measured phase + +Latency, throughput and CPU per request are compared with whole-run cluster +bootstrap confidence intervals, and every response digest is compared against +the first baseline run. Any digest that differs between the arms fails the +verdict regardless of the timings; digests that already differ between two +baseline runs make the comparison inconclusive instead. +""" + +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import os +from pathlib import Path +import random +import sys + +# Latency floors below which a delta is treated as noise. These are starting +# values for the first runs on the bench runner and are meant to be tightened +# once the A/A spread of each class is known. +PRACTICAL_FLOOR_PCT = { + "mean": 2.5, + "p50": 2.5, + "p90": 2.5, + "p99": 5.0, + "record_median": 2.5, + "closed_loop_rps": 2.5, + "cpu_per_request": 2.5, +} +MAX_DIVERGENT_REPORTED = 40 +OK_STATUS = "ok" + + +def _load_engine_summary() -> object: + """Import the engine summary script for its statistics helpers.""" + path = Path(__file__).resolve().with_name("bench-reth-summary.py") + spec = importlib.util.spec_from_file_location("bench_reth_summary", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_engine = _load_engine_summary() +BOOTSTRAP_ITERATIONS = _engine.BOOTSTRAP_ITERATIONS +_bootstrap_ci = _engine._bootstrap_ci +_ci_half_width = _engine._ci_half_width +_mean = _engine._mean +build_grafana_logs_url = _engine.build_grafana_logs_url +build_grafana_traces_url = _engine.build_grafana_traces_url +fmt_ms = _engine.fmt_ms +generate_observability_section = _engine.generate_observability_section +percentile = _engine.percentile +resolve_observability_range = _engine.resolve_observability_range +significance = _engine.significance + + +def load_run(directory: Path) -> dict: + """Load one run directory into the per-run shape used by the comparison.""" + report_path = directory / "report.json" + if not report_path.is_file(): + raise SystemExit(f"Missing {report_path}") + with report_path.open() as f: + report = json.load(f) + + responses_path = directory / "responses.ndjson" + if not responses_path.is_file(): + raise SystemExit(f"Missing {responses_path}") + + requests = parse_requests_csv(directory / "requests.csv") + records = parse_record_timings_csv(directory / "record_timings.csv") + if not requests and not records: + raise SystemExit(f"No open-loop or closed-loop timings in {directory}") + + cpu = {} + cpu_path = directory / "cpu.json" + if cpu_path.is_file(): + with cpu_path.open() as f: + cpu = json.load(f) + + return { + "name": directory.name, + "report": report, + "requests": requests, + "records": records, + "responses": parse_responses(responses_path), + "cpu": cpu, + } + + +def parse_requests_csv(path: Path) -> list[dict]: + """Parse the open-loop per-request CSV.""" + if not path.is_file(): + return [] + rows = [] + with path.open() as f: + for row in csv.DictReader(f): + rows.append( + { + "offset_ms": float(row["offset_ms"]), + "record_index": int(row["record_index"]), + "method": row["method"], + "latency_us": int(row["latency_us"]), + "status": row["status"], + } + ) + return rows + + +def parse_record_timings_csv(path: Path) -> list[dict]: + """Parse the closed-loop per-record CSV.""" + if not path.is_file(): + return [] + rows = [] + with path.open() as f: + for row in csv.DictReader(f): + rows.append( + { + "record_index": int(row["record_index"]), + "method": row["method"], + "pass": int(row["pass"]), + "latency_us": int(row["latency_us"]), + "status": row["status"], + } + ) + return rows + + +def parse_responses(path: Path) -> dict[int, dict]: + """Parse response digests, keyed by record index.""" + responses = {} + with path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + entry = json.loads(line) + responses[int(entry["record_index"])] = { + "method": entry.get("method", "unknown"), + "kind": entry.get("kind", "unknown"), + "digest": entry.get("digest", ""), + "len": entry.get("len"), + } + return responses + + +def call_section(report: dict) -> dict: + """The replay section of a report; `bench call` nests its fields under `call`.""" + section = report.get("call") + return section if isinstance(section, dict) else report + + +def report_field(report: dict, key: str): + """Look a field up in the report, tolerating the nested `call` and identity blocks.""" + for scope in (report, call_section(report)): + if key in scope: + return scope[key] + for nested in ("identity", "metadata", "meta"): + block = scope.get(nested) + if isinstance(block, dict) and key in block: + return block[key] + return None + + +def median(values: list[float]) -> float: + if not values: + return 0.0 + ordered = sorted(values) + mid = len(ordered) // 2 + if len(ordered) % 2: + return ordered[mid] + return (ordered[mid - 1] + ordered[mid]) / 2.0 + + +def open_loop_stats(rows: list[dict]) -> dict: + """Latency distribution, achieved rate and error rate of one open-loop run.""" + if not rows: + return {} + ok_rows = [row for row in rows if row["status"] == OK_STATUS] + latencies_ms = sorted(row["latency_us"] / 1_000 for row in ok_rows) + offsets = [row["offset_ms"] for row in rows] + span_s = (max(offsets) - min(offsets)) / 1_000 if len(rows) > 1 else 0.0 + return { + "requests": len(rows), + "ok": len(ok_rows), + "errors": len(rows) - len(ok_rows), + "error_rate_pct": (len(rows) - len(ok_rows)) / len(rows) * 100.0, + "mean_ms": _mean(latencies_ms), + "p50_ms": percentile(latencies_ms, 50), + "p90_ms": percentile(latencies_ms, 90), + "p99_ms": percentile(latencies_ms, 99), + "open_loop_rps": len(rows) / span_s if span_s > 0 else 0.0, + } + + +def record_medians(rows: list[dict]) -> dict[int, float]: + """Median closed-loop latency in ms per record, over the passes of one run.""" + by_record: dict[int, list[float]] = {} + for row in rows: + if row["status"] != OK_STATUS: + continue + by_record.setdefault(row["record_index"], []).append(row["latency_us"] / 1_000) + return {index: median(values) for index, values in by_record.items()} + + +def arm_record_medians(runs: list[dict], method: str | None = None) -> dict[int, float]: + """Median closed-loop latency per record, over all passes and runs of one arm.""" + by_record: dict[int, list[float]] = {} + for run in runs: + for row in run["records"]: + if row["status"] != OK_STATUS: + continue + if method is not None and row["method"] != method: + continue + by_record.setdefault(row["record_index"], []).append(row["latency_us"] / 1_000) + return {index: median(values) for index, values in by_record.items()} + + +def run_stats(run: dict, method: str | None = None) -> dict: + """Every displayed metric for one run, optionally restricted to one method.""" + requests = run["requests"] + records = run["records"] + if method is not None: + requests = [row for row in requests if row["method"] == method] + records = [row for row in records if row["method"] == method] + + stats = open_loop_stats(requests) + medians = record_medians(records) + if medians: + stats["record_median_ms"] = median(list(medians.values())) + stats["records"] = len(medians) + + report = run["report"] + if method is None: + closed_loop_rps = report_field(report, "closed_loop_rps") + dropped = report_field(report, "dropped") + else: + methods = call_section(report).get("methods") or {} + entry = methods.get(method) or {} + closed_loop_rps = entry.get("closed_loop_rps") + if closed_loop_rps is None and isinstance(entry.get("closed_loop"), dict): + closed_loop_rps = entry["closed_loop"].get("rps") + dropped = entry.get("dropped") + if isinstance(closed_loop_rps, (int, float)): + stats["closed_loop_rps"] = float(closed_loop_rps) + if isinstance(dropped, int): + stats["dropped"] = dropped + + if method is None: + cpu_ms = cpu_ms_per_request(run) + if cpu_ms is not None: + stats["cpu_ms_per_request"] = cpu_ms + + lengths = [ + entry["len"] + for entry in run["responses"].values() + if isinstance(entry.get("len"), int) and (method is None or entry["method"] == method) + ] + if lengths: + stats["response_bytes_median"] = median([float(v) for v in lengths]) + return stats + + +def cpu_ms_per_request(run: dict) -> float | None: + """CPU milliseconds the node spent per successful request of one run.""" + cpu = run["cpu"] + usage_usec = cpu.get("usage_usec_delta") + if not isinstance(usage_usec, (int, float)) or usage_usec <= 0: + return None + requests_ok = cpu.get("requests_ok") + if not isinstance(requests_ok, int) or requests_ok <= 0: + requests_ok = sum( + 1 for row in run["requests"] + run["records"] if row["status"] == OK_STATUS + ) + if requests_ok <= 0: + return None + return usage_usec / 1_000 / requests_ok + + +def arm_stats(runs: list[dict], method: str | None = None) -> dict: + """Point estimates for one arm, averaged over its runs.""" + per_run = [run_stats(run, method) for run in runs] + per_run = [stats for stats in per_run if stats] + if not per_run: + return {} + + stats = {"runs": len(per_run)} + for key in ( + "mean_ms", + "p50_ms", + "p90_ms", + "p99_ms", + "open_loop_rps", + "closed_loop_rps", + "record_median_ms", + "cpu_ms_per_request", + "error_rate_pct", + "response_bytes_median", + ): + values = [run[key] for run in per_run if key in run] + if values: + stats[key] = _mean(values) + for key in ("requests", "ok", "errors", "dropped"): + values = [run[key] for run in per_run if key in run] + if values: + stats[key] = sum(values) + records = [run["records"] for run in per_run if "records" in run] + if records: + stats["records"] = max(records) + return stats + + +def per_run_values(runs: list[dict], method: str | None = None) -> dict[str, list[float]]: + """One value per run and metric, the input of the cluster bootstrap.""" + values: dict[str, list[float]] = {} + for run in runs: + stats = run_stats(run, method) + for key, value in stats.items(): + if isinstance(value, (int, float)): + values.setdefault(key, []).append(float(value)) + return values + + +def cluster_bootstrap_ci( + rng: random.Random, + baseline_values: dict[str, list[float]], + feature_values: dict[str, list[float]], + metrics: tuple[str, ...], + n_iter: int = BOOTSTRAP_ITERATIONS, +) -> dict[str, float]: + """Resample whole runs to estimate run-to-run noise per metric. + + Mirrors the engine summary's cluster bootstrap, which resamples block rows + and cannot be reused for request shaped runs. + """ + empty = {metric: 0.0 for metric in metrics} + usable = [ + metric + for metric in metrics + if len(baseline_values.get(metric, [])) >= 2 and len(feature_values.get(metric, [])) >= 2 + ] + if not usable: + return empty + + samples: dict[str, list[float]] = {metric: [] for metric in usable} + for metric in usable: + baseline_metric = baseline_values[metric] + feature_metric = feature_values[metric] + baseline_count = len(baseline_metric) + feature_count = len(feature_metric) + for _ in range(n_iter): + baseline_sample = [baseline_metric[rng.randrange(baseline_count)] for _ in range(baseline_count)] + feature_sample = [feature_metric[rng.randrange(feature_count)] for _ in range(feature_count)] + samples[metric].append(_mean(feature_sample) - _mean(baseline_sample)) + + result = dict(empty) + for metric in usable: + result[metric] = _ci_half_width(samples[metric]) + return result + + +def paired_record_delta( + rng: random.Random, + baseline_runs: list[dict], + feature_runs: list[dict], + method: str | None = None, +) -> dict | None: + """Per-record closed-loop delta with a bootstrap CI over records.""" + baseline_medians = arm_record_medians(baseline_runs, method) + feature_medians = arm_record_medians(feature_runs, method) + common = sorted(set(baseline_medians) & set(feature_medians)) + if not common: + return None + diffs = [feature_medians[index] - baseline_medians[index] for index in common] + base = _mean([baseline_medians[index] for index in common]) + return { + "records": len(common), + "baseline_ms": base, + "feature_ms": _mean([feature_medians[index] for index in common]), + "delta_ms": _mean(diffs), + "ci_ms": _bootstrap_ci(rng, diffs), + } + + +def make_change( + baseline_value: float | None, + feature_value: float | None, + ci_value: float, + floor_pct: float, + lower_is_better: bool, + aa_pct: float | None = None, +) -> dict | None: + """Build one `changes` entry in the shape the comment renderers consume.""" + if baseline_value is None or feature_value is None or baseline_value <= 0: + return None + pct = (feature_value - baseline_value) / baseline_value * 100.0 + ci_pct = ci_value / baseline_value * 100.0 if ci_value else 0.0 + change = { + "pct": round(pct, 4), + "ci_pct": round(ci_pct, 4), + "floor_pct": round(floor_pct, 4), + "sig": significance(pct, ci_pct, floor_pct, lower_is_better), + "baseline": baseline_value, + "feature": feature_value, + } + if aa_pct is not None: + change["aa_pct"] = round(aa_pct, 4) + return change + + +def aa_spread(runs: list[dict], method: str | None = None) -> dict[str, float]: + """Percent difference between the first two baseline runs, the noise floor.""" + if len(runs) < 2: + return {} + first = run_stats(runs[0], method) + second = run_stats(runs[1], method) + spread = {} + for key, value in first.items(): + if not isinstance(value, (int, float)) or value <= 0: + continue + other = second.get(key) + if isinstance(other, (int, float)): + spread[key] = (other - value) / value * 100.0 + return spread + + +def compute_changes( + rng: random.Random, + baseline_runs: list[dict], + feature_runs: list[dict], + baseline_stats: dict, + feature_stats: dict, + method: str | None = None, +) -> dict: + """Compare one corpus or one method between the two arms.""" + metrics = ( + ("mean", "mean_ms", True), + ("p50", "p50_ms", True), + ("p90", "p90_ms", True), + ("p99", "p99_ms", True), + ("closed_loop_rps", "closed_loop_rps", False), + ("cpu_per_request", "cpu_ms_per_request", True), + ) + ci = cluster_bootstrap_ci( + rng, + per_run_values(baseline_runs, method), + per_run_values(feature_runs, method), + tuple(stat_key for _, stat_key, _ in metrics) + ("record_median_ms",), + ) + spread = aa_spread(baseline_runs, method) + + changes = {} + for name, stat_key, lower_is_better in metrics: + change = make_change( + baseline_stats.get(stat_key), + feature_stats.get(stat_key), + ci.get(stat_key, 0.0), + PRACTICAL_FLOOR_PCT.get(name, 0.0), + lower_is_better, + spread.get(stat_key), + ) + if change: + changes[name] = change + + # The point estimate is the per-run record median averaged over runs, so its + # interval comes from the same whole-run bootstrap as the other metrics; the + # per-record pairing is kept as supplementary detail only. + change = make_change( + baseline_stats.get("record_median_ms"), + feature_stats.get("record_median_ms"), + ci.get("record_median_ms", 0.0), + PRACTICAL_FLOOR_PCT["record_median"], + True, + spread.get("record_median_ms"), + ) + if change: + paired = paired_record_delta(rng, baseline_runs, feature_runs, method) + if paired: + change["records"] = paired["records"] + change["paired_delta_ms"] = round(paired["delta_ms"], 6) + changes["record_median"] = change + return changes + + +def parity_entry() -> dict: + return { + "matched": 0, + "content_mismatch": 0, + "kind_mismatch": 0, + "missing": 0, + "nondeterministic": 0, + "excluded": 0, + "divergent_records": [], + } + + +def compare_responses( + reference: dict[int, dict], + other: dict[int, dict], + exclude: frozenset[int] = frozenset(), +) -> dict: + """Compare one run's digests against the reference run, per method. + + Records in `exclude` are unstable in the baseline itself and are only counted, + never judged. + """ + per_method: dict[str, dict] = {} + for index, expected in reference.items(): + method = expected["method"] + entry = per_method.setdefault(method, parity_entry()) + if index in exclude: + entry["excluded"] += 1 + continue + actual = other.get(index) + if actual is None: + entry["missing"] += 1 + elif actual["kind"] != expected["kind"]: + entry["kind_mismatch"] += 1 + elif actual["digest"] != expected["digest"]: + entry["content_mismatch"] += 1 + else: + entry["matched"] += 1 + continue + if len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + # A record the reference run never answered but this run did is just as divergent. + for index, actual in other.items(): + if index in reference: + continue + entry = per_method.setdefault(actual["method"], parity_entry()) + if index in exclude: + entry["excluded"] += 1 + continue + entry["missing"] += 1 + if len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + return per_method + + +def merge_parity(target: dict, addition: dict) -> None: + for method, counts in addition.items(): + entry = target.setdefault(method, parity_entry()) + for key in ("matched", "content_mismatch", "kind_mismatch", "missing", "nondeterministic", "excluded"): + entry[key] += counts.get(key, 0) + for index in counts["divergent_records"]: + if index not in entry["divergent_records"] and len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + + +def parity_totals(per_method: dict) -> dict: + totals = { + "matched": 0, + "content_mismatch": 0, + "kind_mismatch": 0, + "missing": 0, + "nondeterministic": 0, + "excluded": 0, + } + for counts in per_method.values(): + for key in totals: + totals[key] += counts.get(key, 0) + # A record answered in only one of the two runs, or answered differently + # within the feature runs alone, counts as a divergence. + totals["mismatched"] = ( + totals["content_mismatch"] + totals["kind_mismatch"] + totals["missing"] + totals["nondeterministic"] + ) + return totals + + +def nondeterministic_count(runs: list[dict]) -> int: + total = 0 + for run in runs: + entries = report_field(run["report"], "nondeterministic") + if isinstance(entries, list): + total += len(entries) + elif isinstance(entries, int): + total += entries + return total + + +def nondeterministic_records(runs: list[dict]) -> tuple[set[int], bool]: + """Record indexes a run answered differently within itself. + + The second value is true when a run only reported a count, so the records + cannot be identified. + """ + indexes: set[int] = set() + unknown = False + for run in runs: + entries = report_field(run["report"], "nondeterministic") + if isinstance(entries, list): + indexes.update(int(index) for index in entries) + total = report_field(run["report"], "nondeterministic_total") + if isinstance(total, int) and total > len(entries): + unknown = True + elif isinstance(entries, int) and entries > 0: + unknown = True + return indexes, unknown + + +def compute_parity(baseline_runs: list[dict], feature_runs: list[dict]) -> dict: + """Compare every run's digests against the first baseline run. + + Records that are unstable in the baseline itself, answered differently + between baseline runs or within one, are excluded from the decision and + reported; the feature is judged on the remaining stable records, where its + own nondeterminism counts as a divergence. + """ + reference = baseline_runs[0]["responses"] + + baseline_parity: dict[str, dict] = {} + for run in baseline_runs[1:]: + merge_parity(baseline_parity, compare_responses(reference, run["responses"])) + baseline_nondeterministic, baseline_unknown = nondeterministic_records(baseline_runs) + unstable = set(baseline_nondeterministic) + for counts in baseline_parity.values(): + unstable.update(counts["divergent_records"]) + baseline_totals = parity_totals(baseline_parity) + if baseline_totals["mismatched"] > sum(len(c["divergent_records"]) for c in baseline_parity.values()): + # More baseline divergences than we could list: the unstable set is incomplete. + baseline_unknown = True + + feature_nondeterministic, feature_unknown = nondeterministic_records(feature_runs) + excluded = frozenset(unstable) + feature_parity: dict[str, dict] = {} + for run in feature_runs: + merge_parity(feature_parity, compare_responses(reference, run["responses"], excluded)) + for index in sorted(feature_nondeterministic - unstable): + expected = reference.get(index) + if expected is None: + continue + entry = feature_parity.setdefault(expected["method"], parity_entry()) + entry["nondeterministic"] += 1 + if index not in entry["divergent_records"] and len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + feature_totals = parity_totals(feature_parity) + nondeterministic = nondeterministic_count(baseline_runs + feature_runs) + + records = len(reference) + stable_records = records - len(unstable & set(reference)) + if baseline_unknown or feature_unknown or (unstable and stable_records == 0): + status = "inconclusive" + elif feature_totals["mismatched"]: + status = "mismatch" + else: + status = "matched" + + excluded_list = sorted(unstable) + return { + "status": status, + "records": records, + "feature": {"totals": feature_totals, "methods": feature_parity}, + "baseline": {"totals": baseline_totals, "methods": baseline_parity}, + "nondeterministic": nondeterministic, + "excluded": excluded_list[:MAX_DIVERGENT_REPORTED], + "excluded_count": len(excluded_list), + "line": parity_line( + status, records, feature_totals, baseline_totals, nondeterministic, feature_parity, excluded_list + ), + } + + +def parity_line( + status: str, + records: int, + feature_totals: dict, + baseline_totals: dict, + nondeterministic: int, + feature_parity: dict, + excluded: list[int] | None = None, +) -> str: + excluded = excluded or [] + excluded_note = "" + if excluded: + shown = ", ".join(str(index) for index in excluded[:MAX_DIVERGENT_REPORTED]) + excluded_note = f"; {len(excluded)} unstable in the baseline, excluded (records {shown})" + checked = feature_totals["matched"] + feature_totals["mismatched"] + if status == "matched": + return ( + f"✅ matched {feature_totals['matched']}/{checked} responses " + f"over {records} records{excluded_note}" + ) + + details = [] + for method, counts in sorted(feature_parity.items()): + mismatched = ( + counts["content_mismatch"] + counts["kind_mismatch"] + counts["missing"] + counts.get("nondeterministic", 0) + ) + if not mismatched: + continue + indexes = ", ".join(str(index) for index in counts["divergent_records"]) + details.append(f"`{method}`: {mismatched} (records {indexes})") + + if status == "mismatch": + return "❌ {} content, {} kind mismatches, {} answered in one arm only, {} nondeterministic in the feature only — {}{}".format( + feature_totals["content_mismatch"], + feature_totals["kind_mismatch"], + feature_totals["missing"], + feature_totals.get("nondeterministic", 0), + "; ".join(details) if details else "no per-method detail", + excluded_note, + ) + + reasons = [] + if baseline_totals["mismatched"]: + reasons.append(f"{baseline_totals['mismatched']} baseline-vs-baseline mismatches") + if nondeterministic: + reasons.append(f"{nondeterministic} records answered differently within a run") + if not reasons: + reasons.append("no stable records to judge") + suffix = f"; feature mismatches: {feature_totals['mismatched']}" if feature_totals["mismatched"] else "" + return "⚠️ inconclusive — " + ", ".join(reasons) + suffix + + +def parity_change(parity: dict) -> dict: + """Fold parity into `changes` so the shared verdict helper sees it.""" + totals = parity["feature"]["totals"] + change = { + "pct": 0.0, + "ci_pct": 0.0, + "floor_pct": 0.0, + "sig": "bad" if parity["status"] == "mismatch" else "neutral", + "mismatched": totals["mismatched"], + "excluded": parity.get("excluded_count", 0), + } + if parity["status"] == "inconclusive": + change["informational"] = True + change["informational_reason"] = "informational, responses are not deterministic" + return change + + +def corpus_methods(baseline_runs: list[dict], feature_runs: list[dict]) -> list[str]: + methods = set() + for run in baseline_runs + feature_runs: + for entry in run["responses"].values(): + methods.add(entry["method"]) + for row in run["requests"]: + methods.add(row["method"]) + return sorted(methods) + + +def assert_same_head(runs: list[dict]) -> str | None: + """Refuse to compare runs that replayed against different chain tips.""" + hashes = {} + for run in runs: + head_hash = report_field(run["report"], "head_hash") + if head_hash: + hashes[run["name"]] = head_hash + distinct = set(hashes.values()) + if len(distinct) > 1: + detail = ", ".join(f"{name}={value}" for name, value in sorted(hashes.items())) + raise SystemExit(f"Runs replayed against different chain tips: {detail}") + return next(iter(distinct), None) + + +def fmt_opt_ms(value: float | None) -> str: + return fmt_ms(value) if isinstance(value, (int, float)) else "n/a" + + +def fmt_opt_num(value: float | None, digits: int = 2) -> str: + return f"{value:.{digits}f}" if isinstance(value, (int, float)) else "n/a" + + +def change_str(change: dict | None) -> str: + if not change: + return "n/a" + emoji = {"good": "✅", "bad": "❌", "neutral": "⚪"}[ + "neutral" if change.get("informational") else change["sig"] + ] + details = [f"±{change['ci_pct']:.2f}%", f"floor {change['floor_pct']:.2f}%"] + if "aa_pct" in change: + details.append(f"A/A {change['aa_pct']:+.2f}%") + if change.get("informational"): + details.append("informational") + return f"{change['pct']:+.2f}% {emoji} ({', '.join(details)})" + + +def generate_metric_table(summary: dict, baseline_label: str, feature_label: str) -> str: + baseline = summary["baseline"]["stats"] + feature = summary["feature"]["stats"] + changes = summary["changes"] + rows = [ + ("Mean", "mean_ms", "mean", fmt_opt_ms), + ("P50", "p50_ms", "p50", fmt_opt_ms), + ("P90", "p90_ms", "p90", fmt_opt_ms), + ("P99", "p99_ms", "p99", fmt_opt_ms), + ("Record median", "record_median_ms", "record_median", fmt_opt_ms), + ("Closed-loop rps", "closed_loop_rps", "closed_loop_rps", fmt_opt_num), + ("CPU / request", "cpu_ms_per_request", "cpu_per_request", fmt_opt_ms), + ] + lines = [ + f"| Metric | {baseline_label} | {feature_label} | Change |", + "|--------|------|--------|--------|", + ] + for label, stat_key, change_key, formatter in rows: + if stat_key not in baseline and stat_key not in feature: + continue + lines.append( + f"| {label} | {formatter(baseline.get(stat_key))} | " + f"{formatter(feature.get(stat_key))} | {change_str(changes.get(change_key))} |" + ) + lines.append( + f"| Error rate | {fmt_opt_num(baseline.get('error_rate_pct'))}% | " + f"{fmt_opt_num(feature.get('error_rate_pct'))}% | |" + ) + lines.append( + f"| Open-loop rps | {fmt_opt_num(baseline.get('open_loop_rps'))} | " + f"{fmt_opt_num(feature.get('open_loop_rps'))} | |" + ) + lines.append("") + return "\n".join(lines) + + +def generate_method_table(summary: dict) -> str: + methods = summary.get("methods") or {} + if len(methods) < 2: + return "" + lines = [ + "### Per-method", + "", + "| Method | Records | Baseline mean | Feature mean | Change | Response bytes | Parity |", + "|--------|---------|---------------|--------------|--------|----------------|--------|", + ] + for method in sorted(methods): + entry = methods[method] + baseline = entry["baseline"] + feature = entry["feature"] + totals = entry["parity"] + mismatched = totals["content_mismatch"] + totals["kind_mismatch"] + parity = f"{totals['matched']} matched" if not mismatched else f"❌ {mismatched} mismatched" + lines.append( + "| `{}` | {} | {} | {} | {} | {} | {} |".format( + method, + baseline.get("records", baseline.get("requests", 0)), + fmt_opt_ms(baseline.get("mean_ms")), + fmt_opt_ms(feature.get("mean_ms")), + change_str(entry["changes"].get("mean")), + fmt_opt_num(baseline.get("response_bytes_median"), 0), + parity, + ) + ) + lines.append("") + return "\n".join(lines) + + +def generate_markdown( + summary: dict, + baseline_label: str, + feature_label: str, + behind_baseline: int = 0, + repo: str = "", + baseline_ref: str = "", + baseline_name: str = "", + derek_command: str | None = None, +) -> str: + corpus = summary["corpus"] + lines = ["## Benchmark Results", "", "## Configuration"] + if derek_command: + lines.append(f"- Derek command: `{derek_command}`") + methods = ", ".join(corpus.get("methods") or summary.get("replayed_methods") or []) or "all" + top_gas_note = f" (top-gas {corpus['top_gas']})" if corpus.get("top_gas") else "" + tracer_notes = [] + if corpus.get("tracer"): + tracer_notes.append("tracer " + ", ".join(corpus["tracer"])) + for key in ("tracer_config", "trace_options"): + if corpus.get(key): + tracer_notes.append(f"{key.replace('_', '-')} `{json.dumps(corpus[key], separators=(',', ':'))}`") + tracer_note = f", {'; '.join(tracer_notes)}" if tracer_notes else "" + blocks_note = "" + if corpus.get("blocks_source"): + blocks_note = f", blocks {corpus.get('corpus_from')}..{corpus.get('corpus_to')} from {corpus['blocks_source']}" + if corpus.get("block_param") == "parent": + blocks_note += " at parent-block state" + lines.append( + f"- Corpus: `{corpus.get('name', 'static')}` ({corpus.get('source', 'static')}), " + f"class `{corpus.get('class', 'call')}`{top_gas_note}{tracer_note}{blocks_note}, {corpus.get('records', summary['records'])} records, " + f"methods: {methods}" + ) + lines.append( + f"- Replay: {summary['rps']} rps for {summary['duration']}, " + f"{summary['passes']} passes x {summary['concurrency']} workers, " + f"{summary['warmup_seconds']}s warmup, " + f"max tracing requests: {summary.get('max_tracing_requests') or 'default'}" + ) + if corpus.get("tip"): + lines.append(f"- Tip: {corpus['tip']} (`{corpus.get('tip_hash', 'unknown')}`)") + lines.append("") + if behind_baseline > 0: + plural = "s" if behind_baseline > 1 else "" + diff_link = f"https://github.com/{repo}/compare/{baseline_ref[:12]}...{baseline_name}" + lines.append( + f"> ⚠️ Feature is [**{behind_baseline} commit{plural} behind `{baseline_name}`**]({diff_link}). " + "Consider rebasing for accurate results." + ) + lines.append("") + lines.append(generate_metric_table(summary, baseline_label, feature_label)) + meta_parts = [ + f"{summary['records']} records", + "mode: call", + f"{summary['run_pairs']} run pairs" if summary.get("run_pairs") else "", + ] + lines.append(f"*{', '.join(part for part in meta_parts if part)}*") + lines.append("") + lines.append(f"**Parity:** {summary['parity']['line']}") + method_table = generate_method_table(summary) + if method_table: + lines.append("") + lines.append(method_table) + lines.extend(generate_observability_section(summary)) + return "\n".join(lines) + + +def build_summary(args, baseline_runs: list[dict], feature_runs: list[dict]) -> dict: + rng = random.Random(42) + head_hash = assert_same_head(baseline_runs + feature_runs) + + baseline_stats = arm_stats(baseline_runs) + feature_stats = arm_stats(feature_runs) + changes = compute_changes(rng, baseline_runs, feature_runs, baseline_stats, feature_stats) + parity = compute_parity(baseline_runs, feature_runs) + changes["parity"] = parity_change(parity) + + corpus = {"name": "static", "source": "static", "class": args.klass} + if args.corpus_meta: + meta_path = Path(args.corpus_meta) + if meta_path.is_file(): + with meta_path.open() as f: + corpus.update(json.load(f)) + if head_hash and not corpus.get("tip_hash"): + corpus["tip_hash"] = head_hash + + methods = corpus_methods(baseline_runs, feature_runs) + method_entries = {} + if len(methods) > 1: + for method in methods: + method_baseline = arm_stats(baseline_runs, method) + method_feature = arm_stats(feature_runs, method) + if not method_baseline or not method_feature: + continue + method_entries[method] = { + "baseline": method_baseline, + "feature": method_feature, + "changes": compute_changes( + rng, baseline_runs, feature_runs, method_baseline, method_feature, method + ), + "parity": parity["feature"]["methods"].get( + method, + {"matched": 0, "content_mismatch": 0, "kind_mismatch": 0, "missing": 0, "divergent_records": []}, + ), + } + + records = parity["records"] or corpus.get("records") or 0 + summary = { + "mode": "call", + "blocks": records, + "records": records, + "corpus": corpus, + "replayed_methods": methods, + "rps": args.rps, + "duration": args.duration, + "passes": args.passes, + "concurrency": args.concurrency, + "warmup_seconds": args.warmup_seconds, + "run_pairs": args.run_pairs, + "baseline": { + "name": args.baseline_name or "baseline", + "ref": args.baseline_ref or "main", + "stats": baseline_stats, + }, + "feature": { + "name": args.feature_name or "feature", + "ref": args.feature_ref or "unknown", + "stats": feature_stats, + }, + "changes": changes, + "methods": method_entries, + "parity": parity, + } + if args.max_tracing_requests: + summary["max_tracing_requests"] = args.max_tracing_requests + if args.benchmark_id: + summary["benchmark_id"] = args.benchmark_id + + from_ms, to_ms = resolve_observability_range( + args.observability_from_ms, args.observability_to_ms + ) + logs_url = args.logs_url + traces_url = args.traces_url + if args.benchmark_id and from_ms and to_ms: + logs_url = logs_url or build_grafana_logs_url(args.benchmark_id, from_ms, to_ms) + traces_url = traces_url or build_grafana_traces_url(args.benchmark_id, from_ms, to_ms) + observability = { + key: value + for key, value in { + "benchmark_id": args.benchmark_id, + "from_ms": from_ms, + "to_ms": to_ms, + "grafana_url": args.grafana_url, + "logs_url": logs_url, + "traces_url": traces_url, + }.items() + if value + } + if observability: + summary["observability"] = observability + return summary + + +def parse_args(argv: list[str] | None = None): + parser = argparse.ArgumentParser(description="Summarize an RPC replay benchmark") + parser.add_argument("--baseline-dir", nargs="+", required=True, help="Baseline run directories") + parser.add_argument("--feature-dir", nargs="+", required=True, help="Feature run directories") + parser.add_argument("--output-summary", required=True, help="Output JSON summary path") + parser.add_argument("--output-markdown", required=True, help="Output markdown path") + parser.add_argument("--repo", default="paradigmxyz/reth", help="GitHub repo (owner/name)") + parser.add_argument("--baseline-ref", default=None, help="Baseline commit SHA") + parser.add_argument("--baseline-name", default=None, help="Baseline display name") + parser.add_argument("--feature-name", default=None, help="Feature branch name") + parser.add_argument("--feature-ref", default=None, help="Feature commit SHA") + parser.add_argument("--behind-baseline", type=int, default=0, help="Commits behind baseline") + parser.add_argument("--corpus-meta", default=None, help="corpus.meta.json written by the extract step") + parser.add_argument("--class", dest="klass", default="call", help="Replayed method class") + parser.add_argument("--rps", default="", help="Configured open-loop request rate") + parser.add_argument("--duration", default="", help="Configured open-loop duration") + parser.add_argument("--passes", default="", help="Configured closed-loop passes") + parser.add_argument("--concurrency", default="", help="Configured closed-loop workers") + parser.add_argument("--warmup-seconds", default="", help="Warmup duration in seconds") + parser.add_argument("--run-pairs", type=int, default=None, help="Configured number of run pairs") + parser.add_argument("--max-tracing-requests", default=None, help="Node tracing request limit") + parser.add_argument("--benchmark-id", default=None, help="Benchmark ID used for OTLP labels") + parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL") + parser.add_argument("--logs-url", default=None, help="Grafana Explore URL for benchmark logs") + parser.add_argument("--traces-url", default=None, help="Grafana Explore URL for benchmark traces") + parser.add_argument("--observability-from-ms", type=int, default=None, help="Observability range start") + parser.add_argument("--observability-to-ms", type=int, default=None, help="Observability range end") + parser.add_argument( + "--derek-command", + default=os.environ.get("DEREK_BENCH_COMMAND"), + help="Full derek bench command", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + if args.run_pairs is not None and args.run_pairs < 1: + print("--run-pairs must be greater than zero", file=sys.stderr) + return 1 + baseline_runs = [load_run(Path(path)) for path in args.baseline_dir] + feature_runs = [load_run(Path(path)) for path in args.feature_dir] + + summary = build_summary(args, baseline_runs, feature_runs) + + base_url = f"https://github.com/{args.repo}/commit" + baseline_label = f"[`{summary['baseline']['name']}`]({base_url}/{summary['baseline']['ref']})" + feature_label = f"[`{summary['feature']['name']}`]({base_url}/{summary['feature']['ref']})" + markdown = generate_markdown( + summary, + baseline_label, + feature_label, + behind_baseline=args.behind_baseline, + repo=args.repo, + baseline_ref=summary["baseline"]["ref"], + baseline_name=summary["baseline"]["name"], + derek_command=args.derek_command, + ) + + with open(args.output_summary, "w") as f: + json.dump(summary, f, indent=2) + print(f"Summary written to {args.output_summary}") + with open(args.output_markdown, "w") as f: + f.write(markdown) + print(f"Markdown written to {args.output_markdown}") + print(f"Parity: {summary['parity']['line']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/bench-job-summary.js b/.github/scripts/bench-job-summary.js index e79085eb2fd..129507b7afa 100644 --- a/.github/scripts/bench-job-summary.js +++ b/.github/scripts/bench-job-summary.js @@ -82,6 +82,11 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, } md += '\n'; + // Response parity (call mode) + if (summary.parity) { + md += `**Parity:** ${summary.parity.line}\n\n`; + } + // Wait time breakdown const wtRows = waitTimeRows(summary); if (wtRows.length > 0) { @@ -111,8 +116,8 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, md += '\n'; } - // Charts - if (chartSha) { + // Charts (call mode does not generate any) + if (chartSha && summary.mode !== 'call') { const prNum = prNumber || '0'; const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/pr/${prNum}/${runId}`; const charts = [ diff --git a/.github/scripts/bench-txgen-extract.sh b/.github/scripts/bench-txgen-extract.sh index a513dcfd1d4..8d1c9953cbc 100755 --- a/.github/scripts/bench-txgen-extract.sh +++ b/.github/scripts/bench-txgen-extract.sh @@ -15,8 +15,28 @@ # /benchmark-blocks.ndjson # In RPC mode, the corresponding names end in `-transactions.ndjson`. # +# In call mode it instead writes a request corpus replayed by `bench call`: +# /corpus.jsonl[.gz] one JSON-RPC request per line +# /corpus.meta.json record counts and the tip it was built for +# +# Corpus sources +# -------------- +# `corpus=static` generates the corpus from the chain around the snapshot tip +# for the requested class. `corpus=` replays a corpus staged on the runner +# under BENCH_CORPUS_DIR (default /reth-bench/corpora), which holds: +# +# .jsonl.gz the corpus itself, gzip optional +# .sha256 optional checksum, verified before the corpus is used +# tracers/.js JavaScript tracers selectable as tracer=js: +# +# Provisioning that directory is manual (copy the file onto the runner). Names +# must not contain path separators. Corpus contents are never logged and never +# uploaded as build artifacts; only the record counts in corpus.meta.json are. +# # Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS -# Optional env: BENCH_EXECUTION_MODE, BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL +# Optional env: BENCH_EXECUTION_MODE, BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL, +# BENCH_CORPUS, BENCH_CORPUS_DIR, BENCH_CALL_CLASS, BENCH_CALL_METHODS, BENCH_CALL_TOP_GAS, +# BENCH_CALL_TRACER, BENCH_CALL_TRACER_CONFIG, BENCH_CALL_TRACE_OPTIONS, BENCH_CALL_NAMESPACE set -euxo pipefail BINARY="$1" @@ -69,6 +89,11 @@ RETH_ARGS=( --no-persist-peers ) +# Call mode probes the snapshot's tracing range before the node is stopped. +if [ "$EXECUTION_MODE" = "call" ]; then + RETH_ARGS+=(--http.api "eth,net,web3,debug,trace") +fi + if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then RETH_ARGS+=(--debug.startup-sync-state-idle) fi @@ -104,12 +129,304 @@ HEAD_HEX=$(jq -r '.result' <<< "$HEAD_JSON") HEAD_DEC=$((16#${HEAD_HEX#0x})) echo "Snapshot chain tip: ${HEAD_DEC}" -# Stop the throwaway node and recover snapshot before extraction. -sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true -sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true -kill "${TAIL_PID:-}" 2>/dev/null || true -TAIL_PID= -sudo schelk recover -y --kill || true +HEAD_HASH=$(curl -sf http://127.0.0.1:8545 -X POST \ + -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}' \ + | jq -r '.result.hash') + +# Returns non-zero when the snapshot can no longer trace the given block, which +# happens once the block falls out of the retained account and storage history. +# The archive node serves the blocks after the snapshot tip for the call +# classes; probe it once so an outage falls back instead of failing the run. +archive_rpc_reachable() { + local response + response=$(curl -sf -m 15 "$BENCH_RPC_URL" -X POST -H 'Content-Type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}' 2>/dev/null) || return 1 + [[ "$response" == *'"result"'* ]] +} + +block_is_traceable() { + local block="$1" + local block_hex tx_hash response + block_hex=$(printf '0x%x' "$block") + tx_hash=$(curl -sf http://127.0.0.1:8545 -X POST \ + -H 'Content-Type: application/json' \ + -d "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"${block_hex}\",false],\"id\":1}" \ + | jq -r '.result.transactions[0] // empty') + if [ -z "$tx_hash" ]; then + echo "Block ${block} has no transactions, skipping traceability probe" + return 0 + fi + # The block class never resolves hashes, so it probes the block-addressed + # method it will replay; the transaction class probes by hash, which also + # detects a pruned transaction lookup. + local probe + if [ "$CALL_CLASS" = "traceblock" ]; then + probe="{\"jsonrpc\":\"2.0\",\"method\":\"debug_traceBlockByNumber\",\"params\":[\"${block_hex}\",{\"tracer\":\"callTracer\",\"tracerConfig\":{\"onlyTopCall\":true}}],\"id\":1}" + else + probe="{\"jsonrpc\":\"2.0\",\"method\":\"debug_traceTransaction\",\"params\":[\"${tx_hash}\",{\"tracer\":\"callTracer\"}],\"id\":1}" + fi + if ! response=$(curl -sf http://127.0.0.1:8545 -X POST \ + -H 'Content-Type: application/json' \ + -d "$probe"); then + echo "traceability probe request for block ${block} failed" + return 1 + fi + if jq -e 'has("error")' <<< "$response" > /dev/null 2>&1; then + local message + message="$(jq -r '.error.message' <<< "$response")" + # A transaction the node just listed in the block but cannot find by hash + # means the transaction lookup index is pruned, so no block in the range + # will do any better; report that instead of shrinking the range. + if [[ "$message" == *"transaction not found"* ]]; then + echo "::error::Snapshot has no transaction hash index (transaction lookup is pruned); hash-addressed classes cannot run on it. Use class=tracecall or class=traceblock, or a snapshot that keeps the transaction lookup." + exit 1 + fi + echo "Block ${block} is not traceable: ${message}" + return 1 + fi + return 0 +} + +CALL_BLOCKS="${BENCH_BLOCKS:-20}" +CALL_CLASS="${BENCH_CALL_CLASS:-call}" +CORPUS_SOURCE="${BENCH_CORPUS:-static}" +if [ "$EXECUTION_MODE" = "call" ] && [ "$CORPUS_SOURCE" = "static" ]; then + case "$CALL_CLASS" in + tracetx|traceblock) + # The snapshot is a pruned full node whose persisted state can lag its + # header tip, so the newest blocks may trace against stale state ("nonce + # too high"). Walk down from the tip to the newest block that actually + # traces, then window below it; also shrink the window if its oldest + # block has fallen past the retained history floor. + CORPUS_TOP="$HEAD_DEC" + probe_budget="${BENCH_TRACE_PROBE_DEPTH:-1024}" + while ! block_is_traceable "$CORPUS_TOP"; do + probe_budget=$(( probe_budget - 1 )) + if [ "$probe_budget" -le 0 ] || [ "$CORPUS_TOP" -le 1 ]; then + echo "::error::Snapshot has no traceable block near its tip ${HEAD_DEC} (searched down to ${CORPUS_TOP}); its persisted state lags too far behind the header for class ${CALL_CLASS}." + exit 1 + fi + CORPUS_TOP=$(( CORPUS_TOP - 1 )) + done + if [ "$CORPUS_TOP" -ne "$HEAD_DEC" ]; then + echo "Newest traceable block is ${CORPUS_TOP}, $(( HEAD_DEC - CORPUS_TOP )) below the tip" + fi + while [ "$CALL_BLOCKS" -gt 1 ] && ! block_is_traceable $(( CORPUS_TOP - CALL_BLOCKS + 1 )); do + CALL_BLOCKS=$(( CALL_BLOCKS / 2 )) + echo "Retrying traceability probe with ${CALL_BLOCKS} source blocks" + done + echo "Snapshot traces blocks $(( CORPUS_TOP - CALL_BLOCKS + 1 ))..${CORPUS_TOP}" + ;; + esac +fi + +# Stop the throwaway node and recover the snapshot before extraction. Call mode +# keeps it up: the trace classes read their blocks from it, and the exit trap +# stops it once the corpus is staged. +if [ "$EXECUTION_MODE" != "call" ]; then + sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true + sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true + kill "${TAIL_PID:-}" 2>/dev/null || true + TAIL_PID= + sudo schelk recover -y --kill || true +fi + +# --- Stage the call corpus --- + +if [ "$EXECUTION_MODE" = "call" ]; then + CALL_METHODS="${BENCH_CALL_METHODS:-}" + CORPUS_FILE="$OUTPUT_DIR/corpus.jsonl" + + read_corpus() { + case "$1" in + *.gz) gzip -dc "$1" ;; + *) cat "$1" ;; + esac + } + + if [ "$CORPUS_SOURCE" = "static" ]; then + TXGEN_ETHEREUM="$(which txgen-ethereum)" + if [ "$CALL_BLOCKS" -le 0 ]; then + echo "::error::BENCH_BLOCKS must be greater than 0" + exit 1 + fi + case "$CALL_CLASS" in + call|tracecall) + # Transactions of the blocks after the tip are exactly the calls a + # client would have served against the tip state; only the archive + # node has those blocks. When it is unreachable, the blocks just below + # the tip on the snapshot stand in: their calls replay against the tip + # state too, only with more reverts. + CORPUS_FORMAT=calls + if archive_rpc_reachable; then + CORPUS_FROM=$(( HEAD_DEC + 1 )) + CORPUS_TO=$(( HEAD_DEC + CALL_BLOCKS )) + CORPUS_RPC="$BENCH_RPC_URL" + CORPUS_BLOCKS_SOURCE=archive + else + echo "::warning::Archive node ${BENCH_RPC_URL} is unreachable; building the ${CALL_CLASS} corpus from snapshot blocks below the tip" + CORPUS_FROM=$(( HEAD_DEC - CALL_BLOCKS + 1 )) + CORPUS_TO="$HEAD_DEC" + CORPUS_RPC="http://127.0.0.1:8545" + CORPUS_BLOCKS_SOURCE=snapshot + # These transactions already executed, so replay each against the + # state before its block instead of the tip, where its preconditions + # no longer hold. + CORPUS_BLOCK_PARAM=parent + fi + ;; + tracetx|traceblock) + # These blocks are on the snapshot, so the throwaway node serves them + # and the archive node is not needed. CORPUS_TOP is the newest block the + # probe found traceable, which may sit below the header tip. + CORPUS_FROM=$(( CORPUS_TOP - CALL_BLOCKS + 1 )) + CORPUS_TO="$CORPUS_TOP" + CORPUS_FORMAT=traces + CORPUS_RPC="http://127.0.0.1:8545" + CORPUS_BLOCKS_SOURCE=snapshot + ;; + *) + echo "::error::Unknown call class: ${CALL_CLASS}" + exit 1 + ;; + esac + CORPUS_METHOD_ARGS=() + if [ -n "$CALL_METHODS" ]; then + CORPUS_METHOD_ARGS+=(--methods "$CALL_METHODS") + fi + if [ -n "${BENCH_CALL_TOP_GAS:-}" ]; then + CORPUS_METHOD_ARGS+=(--top-gas "$BENCH_CALL_TOP_GAS") + fi + if [ -n "${BENCH_CALL_TRACER:-}" ]; then + IFS=',' read -r -a tracer_specs <<< "$BENCH_CALL_TRACER" + for spec in "${tracer_specs[@]}"; do + spec="${spec// /}" + [ -z "$spec" ] && continue + case "$spec" in + js:*) + tracer_name="${spec#js:}" + case "$tracer_name" in + */*|.*|"") + echo "::error::Invalid tracer name: ${spec}" + exit 1 + ;; + esac + tracer_file="${BENCH_CORPUS_DIR:-/reth-bench/corpora}/tracers/${tracer_name}.js" + if [ ! -f "$tracer_file" ]; then + echo "::error::JS tracer ${tracer_name} not found at ${tracer_file}" + exit 1 + fi + CORPUS_METHOD_ARGS+=(--tracer "js:${tracer_file}") + ;; + *) + CORPUS_METHOD_ARGS+=(--tracer "$spec") + ;; + esac + done + fi + if [ -n "${BENCH_CALL_TRACER_CONFIG:-}" ]; then + CORPUS_METHOD_ARGS+=(--tracer-config "$BENCH_CALL_TRACER_CONFIG") + fi + if [ -n "${CORPUS_BLOCK_PARAM:-}" ]; then + CORPUS_METHOD_ARGS+=(--block-param "$CORPUS_BLOCK_PARAM") + fi + if [ -n "${BENCH_CALL_TRACE_OPTIONS:-}" ]; then + CORPUS_METHOD_ARGS+=(--trace-options "$BENCH_CALL_TRACE_OPTIONS") + fi + echo "Generating ${CALL_CLASS} corpus from blocks ${CORPUS_FROM}..${CORPUS_TO} (methods: ${CALL_METHODS:-default})" + "$TXGEN_ETHEREUM" extract \ + --rpc "$CORPUS_RPC" \ + --from "$CORPUS_FROM" \ + --to "$CORPUS_TO" \ + --format "$CORPUS_FORMAT" \ + "${CORPUS_METHOD_ARGS[@]}" \ + -o "$CORPUS_FILE" + else + case "$CORPUS_SOURCE" in + */*|.*) + echo "::error::Invalid corpus name: ${CORPUS_SOURCE}" + exit 1 + ;; + esac + CORPUS_DIR="${BENCH_CORPUS_DIR:-/reth-bench/corpora}" + CORPUS_SRC="" + for candidate in "$CORPUS_DIR/$CORPUS_SOURCE.jsonl.gz" "$CORPUS_DIR/$CORPUS_SOURCE.jsonl"; do + if [ -f "$candidate" ]; then + CORPUS_SRC="$candidate" + break + fi + done + if [ -z "$CORPUS_SRC" ]; then + echo "::error::Corpus ${CORPUS_SOURCE} not found in ${CORPUS_DIR} (expected ${CORPUS_SOURCE}.jsonl.gz or ${CORPUS_SOURCE}.jsonl)" + exit 1 + fi + CORPUS_SHA256="$CORPUS_DIR/$CORPUS_SOURCE.sha256" + if [ -f "$CORPUS_SHA256" ]; then + EXPECTED_SHA="$(awk '{print $1; exit}' "$CORPUS_SHA256")" + ACTUAL_SHA="$(sha256sum "$CORPUS_SRC" | awk '{print $1}')" + if [ "$EXPECTED_SHA" != "$ACTUAL_SHA" ]; then + echo "::error::Checksum mismatch for corpus ${CORPUS_SOURCE}" + exit 1 + fi + echo "Corpus ${CORPUS_SOURCE} checksum verified" + fi + case "$CORPUS_SRC" in + *.gz) CORPUS_FILE="$OUTPUT_DIR/corpus.jsonl.gz" ;; + esac + cp "$CORPUS_SRC" "$CORPUS_FILE" + fi + + CORPUS_RECORDS=$(read_corpus "$CORPUS_FILE" | wc -l | tr -d ' ') + if [ "$CORPUS_RECORDS" -le 0 ]; then + echo "::error::Corpus ${CORPUS_SOURCE} is empty" + exit 1 + fi + CORPUS_RECORDS_PER_METHOD=$(read_corpus "$CORPUS_FILE" | jq -r '.method' | sort | uniq -c \ + | jq -R -s 'split("\n") + | map(select(length > 0) | capture("^\\s*(?\\d+)\\s+(?.+)$") | {(.method): (.count | tonumber)}) + | add // {}') + jq -n \ + --arg source "$CORPUS_SOURCE" \ + --arg name "$CORPUS_SOURCE" \ + --arg class "$CALL_CLASS" \ + --arg methods "$CALL_METHODS" \ + --argjson records "$CORPUS_RECORDS" \ + --argjson records_per_method "$CORPUS_RECORDS_PER_METHOD" \ + --argjson tip "$HEAD_DEC" \ + --arg tip_hash "$HEAD_HASH" \ + --arg blocks_source "${CORPUS_BLOCKS_SOURCE:-}" \ + --arg block_param "${CORPUS_BLOCK_PARAM:-}" \ + --argjson corpus_from "${CORPUS_FROM:-null}" \ + --argjson corpus_to "${CORPUS_TO:-null}" \ + --argjson top_gas "${BENCH_CALL_TOP_GAS:-null}" \ + --arg tracer "${BENCH_CALL_TRACER:-}" \ + --arg namespace "${BENCH_CALL_NAMESPACE:-}" \ + --argjson tracer_config "${BENCH_CALL_TRACER_CONFIG:-null}" \ + --argjson trace_options "${BENCH_CALL_TRACE_OPTIONS:-null}" \ + '{ + source: (if $source == "static" then "static" else "custom" end), + name: $name, + class: $class, + methods: (if $methods == "" then [] else ($methods | split(",")) end), + records: $records, + records_per_method: $records_per_method, + tip: $tip, + tip_hash: $tip_hash, + blocks_source: (if $blocks_source == "" then null else $blocks_source end), + block_param: (if $block_param == "" then null else $block_param end), + corpus_from: $corpus_from, + corpus_to: $corpus_to, + top_gas: $top_gas, + tracer: (if $tracer == "" then null else ($tracer | split(",")) end), + namespace: (if $namespace == "" then null else $namespace end), + tracer_config: $tracer_config, + trace_options: $trace_options, + }' > "$OUTPUT_DIR/corpus.meta.json" + + echo "Corpus staged: ${CORPUS_RECORDS} records in ${CORPUS_FILE}" + exit 0 +fi # --- Extract payloads from the remote RPC --- diff --git a/.github/scripts/bench-txgen-install.sh b/.github/scripts/bench-txgen-install.sh index 6ac1d31897c..366a01d01eb 100755 --- a/.github/scripts/bench-txgen-install.sh +++ b/.github/scripts/bench-txgen-install.sh @@ -4,6 +4,7 @@ # # Optional env: # TXGEN_REPO – txgen repository URL (default: https://github.com/tempoxyz/txgen) +# TXGEN_REF – branch or commit to install instead of the default branch set -euxo pipefail TXGEN_REPO="${TXGEN_REPO:-https://github.com/tempoxyz/txgen}" @@ -31,4 +32,14 @@ elif [ -n "${TXGEN_TOKEN:-${GH_PROJECT_TOKEN:-${DEREK_PAT:-${DEREK_TOKEN:-}}}}" fi export CARGO_NET_GIT_FETCH_WITH_CLI=true -cargo install --git "$TXGEN_REPO" --locked txgen-ethereum bench-cli +INSTALL_ARGS=() +if [ -n "${TXGEN_REF:-}" ]; then + if [[ "$TXGEN_REF" =~ ^[0-9a-f]{7,40}$ ]]; then + INSTALL_ARGS+=(--rev "$TXGEN_REF") + else + INSTALL_ARGS+=(--branch "$TXGEN_REF") + fi + # A pinned ref must replace whatever build of the same version is installed. + INSTALL_ARGS+=(--force) +fi +cargo install --git "$TXGEN_REPO" ${INSTALL_ARGS[@]+"${INSTALL_ARGS[@]}"} --locked txgen-ethereum bench-cli diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh index b709cf7c544..902f4ebb174 100755 --- a/.github/scripts/bench-txgen-run.sh +++ b/.github/scripts/bench-txgen-run.sh @@ -4,10 +4,16 @@ # mount snapshot → start node → extract source blocks → warmup → replay → # convert txgen JSON report into legacy benchmark CSVs. # +# In call mode it replays the request corpus staged by bench-txgen-extract.sh +# instead, and writes responses.ndjson, record_timings.csv, requests.csv, +# report.json and cpu.json for bench-call-summary.py. +# # Usage: bench-txgen-run.sh