diff --git a/.config/zepter.yaml b/.config/zepter.yaml index 7aa1bd725ef..d2d5dfb5255 100644 --- a/.config/zepter.yaml +++ b/.config/zepter.yaml @@ -12,7 +12,7 @@ workflows: # Check that `A` activates the features of `B`. "propagate-feature", # These are the features to check: - "--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global,trie-debug,secp256k1", + "--features=std,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global,trie-debug,secp256k1", # Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually. "--left-side-feature-missing=ignore", # Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on. diff --git a/.dockerignore b/.dockerignore index f4baa18a38b..e9d3ac4167b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,7 +7,6 @@ !/pkg !/testing !/.github/scripts/install_llvm_ubuntu.sh -!book.toml !Cargo.lock !Cargo.toml !Cross.toml diff --git a/.github/scripts/bench-call-summary.py b/.github/scripts/bench-call-summary.py new file mode 100755 index 00000000000..49eeba43069 --- /dev/null +++ b/.github/scripts/bench-call-summary.py @@ -0,0 +1,1097 @@ +#!/usr/bin/env python3 +"""Summarize an RPC replay benchmark and generate summary JSON + markdown. + +Usage: + bench-call-summary.py \ + --output-summary \ + --output-markdown \ + --baseline-dir [ ...] \ + --feature-dir [ ...] + +Each run directory holds the output of one `bench call` measured phase: + + report.json run metadata, identity and per-method aggregates + requests.csv open-loop, one row per request + record_timings.csv closed-loop, one row per record and pass + responses.ndjson one response digest per corpus record + cpu.json CPU time the node's cgroup spent on the measured phase + +Latency, throughput and CPU per request are compared with whole-run cluster +bootstrap confidence intervals, and every response digest is compared against +the first baseline run. Any digest that differs between the arms fails the +verdict regardless of the timings; digests that already differ between two +baseline runs make the comparison inconclusive instead. +""" + +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import os +from pathlib import Path +import random +import sys + +# Latency floors below which a delta is treated as noise. These are starting +# values for the first runs on the bench runner and are meant to be tightened +# once the A/A spread of each class is known. +PRACTICAL_FLOOR_PCT = { + "mean": 2.5, + "p50": 2.5, + "p90": 2.5, + "p99": 5.0, + "record_median": 2.5, + "closed_loop_rps": 2.5, + "cpu_per_request": 2.5, +} +MAX_DIVERGENT_REPORTED = 40 +OK_STATUS = "ok" + + +def _load_engine_summary() -> object: + """Import the engine summary script for its statistics helpers.""" + path = Path(__file__).resolve().with_name("bench-reth-summary.py") + spec = importlib.util.spec_from_file_location("bench_reth_summary", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_engine = _load_engine_summary() +BOOTSTRAP_ITERATIONS = _engine.BOOTSTRAP_ITERATIONS +_bootstrap_ci = _engine._bootstrap_ci +_ci_half_width = _engine._ci_half_width +_mean = _engine._mean +build_grafana_logs_url = _engine.build_grafana_logs_url +build_grafana_traces_url = _engine.build_grafana_traces_url +fmt_ms = _engine.fmt_ms +generate_observability_section = _engine.generate_observability_section +percentile = _engine.percentile +resolve_observability_range = _engine.resolve_observability_range +significance = _engine.significance + + +def load_run(directory: Path) -> dict: + """Load one run directory into the per-run shape used by the comparison.""" + report_path = directory / "report.json" + if not report_path.is_file(): + raise SystemExit(f"Missing {report_path}") + with report_path.open() as f: + report = json.load(f) + + responses_path = directory / "responses.ndjson" + if not responses_path.is_file(): + raise SystemExit(f"Missing {responses_path}") + + requests = parse_requests_csv(directory / "requests.csv") + records = parse_record_timings_csv(directory / "record_timings.csv") + if not requests and not records: + raise SystemExit(f"No open-loop or closed-loop timings in {directory}") + + cpu = {} + cpu_path = directory / "cpu.json" + if cpu_path.is_file(): + with cpu_path.open() as f: + cpu = json.load(f) + + return { + "name": directory.name, + "report": report, + "requests": requests, + "records": records, + "responses": parse_responses(responses_path), + "cpu": cpu, + } + + +def parse_requests_csv(path: Path) -> list[dict]: + """Parse the open-loop per-request CSV.""" + if not path.is_file(): + return [] + rows = [] + with path.open() as f: + for row in csv.DictReader(f): + rows.append( + { + "offset_ms": float(row["offset_ms"]), + "record_index": int(row["record_index"]), + "method": row["method"], + "latency_us": int(row["latency_us"]), + "status": row["status"], + } + ) + return rows + + +def parse_record_timings_csv(path: Path) -> list[dict]: + """Parse the closed-loop per-record CSV.""" + if not path.is_file(): + return [] + rows = [] + with path.open() as f: + for row in csv.DictReader(f): + rows.append( + { + "record_index": int(row["record_index"]), + "method": row["method"], + "pass": int(row["pass"]), + "latency_us": int(row["latency_us"]), + "status": row["status"], + } + ) + return rows + + +def parse_responses(path: Path) -> dict[int, dict]: + """Parse response digests, keyed by record index.""" + responses = {} + with path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + entry = json.loads(line) + responses[int(entry["record_index"])] = { + "method": entry.get("method", "unknown"), + "kind": entry.get("kind", "unknown"), + "digest": entry.get("digest", ""), + "len": entry.get("len"), + } + return responses + + +def call_section(report: dict) -> dict: + """The replay section of a report; `bench call` nests its fields under `call`.""" + section = report.get("call") + return section if isinstance(section, dict) else report + + +def report_field(report: dict, key: str): + """Look a field up in the report, tolerating the nested `call` and identity blocks.""" + for scope in (report, call_section(report)): + if key in scope: + return scope[key] + for nested in ("identity", "metadata", "meta"): + block = scope.get(nested) + if isinstance(block, dict) and key in block: + return block[key] + return None + + +def median(values: list[float]) -> float: + if not values: + return 0.0 + ordered = sorted(values) + mid = len(ordered) // 2 + if len(ordered) % 2: + return ordered[mid] + return (ordered[mid - 1] + ordered[mid]) / 2.0 + + +def open_loop_stats(rows: list[dict]) -> dict: + """Latency distribution, achieved rate and error rate of one open-loop run.""" + if not rows: + return {} + ok_rows = [row for row in rows if row["status"] == OK_STATUS] + latencies_ms = sorted(row["latency_us"] / 1_000 for row in ok_rows) + offsets = [row["offset_ms"] for row in rows] + span_s = (max(offsets) - min(offsets)) / 1_000 if len(rows) > 1 else 0.0 + return { + "requests": len(rows), + "ok": len(ok_rows), + "errors": len(rows) - len(ok_rows), + "error_rate_pct": (len(rows) - len(ok_rows)) / len(rows) * 100.0, + "mean_ms": _mean(latencies_ms), + "p50_ms": percentile(latencies_ms, 50), + "p90_ms": percentile(latencies_ms, 90), + "p99_ms": percentile(latencies_ms, 99), + "open_loop_rps": len(rows) / span_s if span_s > 0 else 0.0, + } + + +def record_medians(rows: list[dict]) -> dict[int, float]: + """Median closed-loop latency in ms per record, over the passes of one run.""" + by_record: dict[int, list[float]] = {} + for row in rows: + if row["status"] != OK_STATUS: + continue + by_record.setdefault(row["record_index"], []).append(row["latency_us"] / 1_000) + return {index: median(values) for index, values in by_record.items()} + + +def arm_record_medians(runs: list[dict], method: str | None = None) -> dict[int, float]: + """Median closed-loop latency per record, over all passes and runs of one arm.""" + by_record: dict[int, list[float]] = {} + for run in runs: + for row in run["records"]: + if row["status"] != OK_STATUS: + continue + if method is not None and row["method"] != method: + continue + by_record.setdefault(row["record_index"], []).append(row["latency_us"] / 1_000) + return {index: median(values) for index, values in by_record.items()} + + +def run_stats(run: dict, method: str | None = None) -> dict: + """Every displayed metric for one run, optionally restricted to one method.""" + requests = run["requests"] + records = run["records"] + if method is not None: + requests = [row for row in requests if row["method"] == method] + records = [row for row in records if row["method"] == method] + + stats = open_loop_stats(requests) + medians = record_medians(records) + if medians: + stats["record_median_ms"] = median(list(medians.values())) + stats["records"] = len(medians) + + report = run["report"] + if method is None: + closed_loop_rps = report_field(report, "closed_loop_rps") + dropped = report_field(report, "dropped") + else: + methods = call_section(report).get("methods") or {} + entry = methods.get(method) or {} + closed_loop_rps = entry.get("closed_loop_rps") + if closed_loop_rps is None and isinstance(entry.get("closed_loop"), dict): + closed_loop_rps = entry["closed_loop"].get("rps") + dropped = entry.get("dropped") + if isinstance(closed_loop_rps, (int, float)): + stats["closed_loop_rps"] = float(closed_loop_rps) + if isinstance(dropped, int): + stats["dropped"] = dropped + + if method is None: + cpu_ms = cpu_ms_per_request(run) + if cpu_ms is not None: + stats["cpu_ms_per_request"] = cpu_ms + + lengths = [ + entry["len"] + for entry in run["responses"].values() + if isinstance(entry.get("len"), int) and (method is None or entry["method"] == method) + ] + if lengths: + stats["response_bytes_median"] = median([float(v) for v in lengths]) + return stats + + +def cpu_ms_per_request(run: dict) -> float | None: + """CPU milliseconds the node spent per successful request of one run.""" + cpu = run["cpu"] + usage_usec = cpu.get("usage_usec_delta") + if not isinstance(usage_usec, (int, float)) or usage_usec <= 0: + return None + requests_ok = cpu.get("requests_ok") + if not isinstance(requests_ok, int) or requests_ok <= 0: + requests_ok = sum( + 1 for row in run["requests"] + run["records"] if row["status"] == OK_STATUS + ) + if requests_ok <= 0: + return None + return usage_usec / 1_000 / requests_ok + + +def arm_stats(runs: list[dict], method: str | None = None) -> dict: + """Point estimates for one arm, averaged over its runs.""" + per_run = [run_stats(run, method) for run in runs] + per_run = [stats for stats in per_run if stats] + if not per_run: + return {} + + stats = {"runs": len(per_run)} + for key in ( + "mean_ms", + "p50_ms", + "p90_ms", + "p99_ms", + "open_loop_rps", + "closed_loop_rps", + "record_median_ms", + "cpu_ms_per_request", + "error_rate_pct", + "response_bytes_median", + ): + values = [run[key] for run in per_run if key in run] + if values: + stats[key] = _mean(values) + for key in ("requests", "ok", "errors", "dropped"): + values = [run[key] for run in per_run if key in run] + if values: + stats[key] = sum(values) + records = [run["records"] for run in per_run if "records" in run] + if records: + stats["records"] = max(records) + return stats + + +def per_run_values(runs: list[dict], method: str | None = None) -> dict[str, list[float]]: + """One value per run and metric, the input of the cluster bootstrap.""" + values: dict[str, list[float]] = {} + for run in runs: + stats = run_stats(run, method) + for key, value in stats.items(): + if isinstance(value, (int, float)): + values.setdefault(key, []).append(float(value)) + return values + + +def cluster_bootstrap_ci( + rng: random.Random, + baseline_values: dict[str, list[float]], + feature_values: dict[str, list[float]], + metrics: tuple[str, ...], + n_iter: int = BOOTSTRAP_ITERATIONS, +) -> dict[str, float]: + """Resample whole runs to estimate run-to-run noise per metric. + + Mirrors the engine summary's cluster bootstrap, which resamples block rows + and cannot be reused for request shaped runs. + """ + empty = {metric: 0.0 for metric in metrics} + usable = [ + metric + for metric in metrics + if len(baseline_values.get(metric, [])) >= 2 and len(feature_values.get(metric, [])) >= 2 + ] + if not usable: + return empty + + samples: dict[str, list[float]] = {metric: [] for metric in usable} + for metric in usable: + baseline_metric = baseline_values[metric] + feature_metric = feature_values[metric] + baseline_count = len(baseline_metric) + feature_count = len(feature_metric) + for _ in range(n_iter): + baseline_sample = [baseline_metric[rng.randrange(baseline_count)] for _ in range(baseline_count)] + feature_sample = [feature_metric[rng.randrange(feature_count)] for _ in range(feature_count)] + samples[metric].append(_mean(feature_sample) - _mean(baseline_sample)) + + result = dict(empty) + for metric in usable: + result[metric] = _ci_half_width(samples[metric]) + return result + + +def paired_record_delta( + rng: random.Random, + baseline_runs: list[dict], + feature_runs: list[dict], + method: str | None = None, +) -> dict | None: + """Per-record closed-loop delta with a bootstrap CI over records.""" + baseline_medians = arm_record_medians(baseline_runs, method) + feature_medians = arm_record_medians(feature_runs, method) + common = sorted(set(baseline_medians) & set(feature_medians)) + if not common: + return None + diffs = [feature_medians[index] - baseline_medians[index] for index in common] + base = _mean([baseline_medians[index] for index in common]) + return { + "records": len(common), + "baseline_ms": base, + "feature_ms": _mean([feature_medians[index] for index in common]), + "delta_ms": _mean(diffs), + "ci_ms": _bootstrap_ci(rng, diffs), + } + + +def make_change( + baseline_value: float | None, + feature_value: float | None, + ci_value: float, + floor_pct: float, + lower_is_better: bool, + aa_pct: float | None = None, +) -> dict | None: + """Build one `changes` entry in the shape the comment renderers consume.""" + if baseline_value is None or feature_value is None or baseline_value <= 0: + return None + pct = (feature_value - baseline_value) / baseline_value * 100.0 + ci_pct = ci_value / baseline_value * 100.0 if ci_value else 0.0 + change = { + "pct": round(pct, 4), + "ci_pct": round(ci_pct, 4), + "floor_pct": round(floor_pct, 4), + "sig": significance(pct, ci_pct, floor_pct, lower_is_better), + "baseline": baseline_value, + "feature": feature_value, + } + if aa_pct is not None: + change["aa_pct"] = round(aa_pct, 4) + return change + + +def aa_spread(runs: list[dict], method: str | None = None) -> dict[str, float]: + """Percent difference between the first two baseline runs, the noise floor.""" + if len(runs) < 2: + return {} + first = run_stats(runs[0], method) + second = run_stats(runs[1], method) + spread = {} + for key, value in first.items(): + if not isinstance(value, (int, float)) or value <= 0: + continue + other = second.get(key) + if isinstance(other, (int, float)): + spread[key] = (other - value) / value * 100.0 + return spread + + +def compute_changes( + rng: random.Random, + baseline_runs: list[dict], + feature_runs: list[dict], + baseline_stats: dict, + feature_stats: dict, + method: str | None = None, +) -> dict: + """Compare one corpus or one method between the two arms.""" + metrics = ( + ("mean", "mean_ms", True), + ("p50", "p50_ms", True), + ("p90", "p90_ms", True), + ("p99", "p99_ms", True), + ("closed_loop_rps", "closed_loop_rps", False), + ("cpu_per_request", "cpu_ms_per_request", True), + ) + ci = cluster_bootstrap_ci( + rng, + per_run_values(baseline_runs, method), + per_run_values(feature_runs, method), + tuple(stat_key for _, stat_key, _ in metrics) + ("record_median_ms",), + ) + spread = aa_spread(baseline_runs, method) + + changes = {} + for name, stat_key, lower_is_better in metrics: + change = make_change( + baseline_stats.get(stat_key), + feature_stats.get(stat_key), + ci.get(stat_key, 0.0), + PRACTICAL_FLOOR_PCT.get(name, 0.0), + lower_is_better, + spread.get(stat_key), + ) + if change: + changes[name] = change + + # The point estimate is the per-run record median averaged over runs, so its + # interval comes from the same whole-run bootstrap as the other metrics; the + # per-record pairing is kept as supplementary detail only. + change = make_change( + baseline_stats.get("record_median_ms"), + feature_stats.get("record_median_ms"), + ci.get("record_median_ms", 0.0), + PRACTICAL_FLOOR_PCT["record_median"], + True, + spread.get("record_median_ms"), + ) + if change: + paired = paired_record_delta(rng, baseline_runs, feature_runs, method) + if paired: + change["records"] = paired["records"] + change["paired_delta_ms"] = round(paired["delta_ms"], 6) + changes["record_median"] = change + return changes + + +def parity_entry() -> dict: + return { + "matched": 0, + "content_mismatch": 0, + "kind_mismatch": 0, + "missing": 0, + "nondeterministic": 0, + "excluded": 0, + "divergent_records": [], + } + + +def compare_responses( + reference: dict[int, dict], + other: dict[int, dict], + exclude: frozenset[int] = frozenset(), +) -> dict: + """Compare one run's digests against the reference run, per method. + + Records in `exclude` are unstable in the baseline itself and are only counted, + never judged. + """ + per_method: dict[str, dict] = {} + for index, expected in reference.items(): + method = expected["method"] + entry = per_method.setdefault(method, parity_entry()) + if index in exclude: + entry["excluded"] += 1 + continue + actual = other.get(index) + if actual is None: + entry["missing"] += 1 + elif actual["kind"] != expected["kind"]: + entry["kind_mismatch"] += 1 + elif actual["digest"] != expected["digest"]: + entry["content_mismatch"] += 1 + else: + entry["matched"] += 1 + continue + if len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + # A record the reference run never answered but this run did is just as divergent. + for index, actual in other.items(): + if index in reference: + continue + entry = per_method.setdefault(actual["method"], parity_entry()) + if index in exclude: + entry["excluded"] += 1 + continue + entry["missing"] += 1 + if len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + return per_method + + +def merge_parity(target: dict, addition: dict) -> None: + for method, counts in addition.items(): + entry = target.setdefault(method, parity_entry()) + for key in ("matched", "content_mismatch", "kind_mismatch", "missing", "nondeterministic", "excluded"): + entry[key] += counts.get(key, 0) + for index in counts["divergent_records"]: + if index not in entry["divergent_records"] and len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + + +def parity_totals(per_method: dict) -> dict: + totals = { + "matched": 0, + "content_mismatch": 0, + "kind_mismatch": 0, + "missing": 0, + "nondeterministic": 0, + "excluded": 0, + } + for counts in per_method.values(): + for key in totals: + totals[key] += counts.get(key, 0) + # A record answered in only one of the two runs, or answered differently + # within the feature runs alone, counts as a divergence. + totals["mismatched"] = ( + totals["content_mismatch"] + totals["kind_mismatch"] + totals["missing"] + totals["nondeterministic"] + ) + return totals + + +def nondeterministic_count(runs: list[dict]) -> int: + total = 0 + for run in runs: + entries = report_field(run["report"], "nondeterministic") + if isinstance(entries, list): + total += len(entries) + elif isinstance(entries, int): + total += entries + return total + + +def nondeterministic_records(runs: list[dict]) -> tuple[set[int], bool]: + """Record indexes a run answered differently within itself. + + The second value is true when a run only reported a count, so the records + cannot be identified. + """ + indexes: set[int] = set() + unknown = False + for run in runs: + entries = report_field(run["report"], "nondeterministic") + if isinstance(entries, list): + indexes.update(int(index) for index in entries) + total = report_field(run["report"], "nondeterministic_total") + if isinstance(total, int) and total > len(entries): + unknown = True + elif isinstance(entries, int) and entries > 0: + unknown = True + return indexes, unknown + + +def compute_parity(baseline_runs: list[dict], feature_runs: list[dict]) -> dict: + """Compare every run's digests against the first baseline run. + + Records that are unstable in the baseline itself, answered differently + between baseline runs or within one, are excluded from the decision and + reported; the feature is judged on the remaining stable records, where its + own nondeterminism counts as a divergence. + """ + reference = baseline_runs[0]["responses"] + + baseline_parity: dict[str, dict] = {} + for run in baseline_runs[1:]: + merge_parity(baseline_parity, compare_responses(reference, run["responses"])) + baseline_nondeterministic, baseline_unknown = nondeterministic_records(baseline_runs) + unstable = set(baseline_nondeterministic) + for counts in baseline_parity.values(): + unstable.update(counts["divergent_records"]) + baseline_totals = parity_totals(baseline_parity) + if baseline_totals["mismatched"] > sum(len(c["divergent_records"]) for c in baseline_parity.values()): + # More baseline divergences than we could list: the unstable set is incomplete. + baseline_unknown = True + + feature_nondeterministic, feature_unknown = nondeterministic_records(feature_runs) + excluded = frozenset(unstable) + feature_parity: dict[str, dict] = {} + for run in feature_runs: + merge_parity(feature_parity, compare_responses(reference, run["responses"], excluded)) + for index in sorted(feature_nondeterministic - unstable): + expected = reference.get(index) + if expected is None: + continue + entry = feature_parity.setdefault(expected["method"], parity_entry()) + entry["nondeterministic"] += 1 + if index not in entry["divergent_records"] and len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + feature_totals = parity_totals(feature_parity) + nondeterministic = nondeterministic_count(baseline_runs + feature_runs) + + records = len(reference) + stable_records = records - len(unstable & set(reference)) + if baseline_unknown or feature_unknown or (unstable and stable_records == 0): + status = "inconclusive" + elif feature_totals["mismatched"]: + status = "mismatch" + else: + status = "matched" + + excluded_list = sorted(unstable) + return { + "status": status, + "records": records, + "feature": {"totals": feature_totals, "methods": feature_parity}, + "baseline": {"totals": baseline_totals, "methods": baseline_parity}, + "nondeterministic": nondeterministic, + "excluded": excluded_list[:MAX_DIVERGENT_REPORTED], + "excluded_count": len(excluded_list), + "line": parity_line( + status, records, feature_totals, baseline_totals, nondeterministic, feature_parity, excluded_list + ), + } + + +def parity_line( + status: str, + records: int, + feature_totals: dict, + baseline_totals: dict, + nondeterministic: int, + feature_parity: dict, + excluded: list[int] | None = None, +) -> str: + excluded = excluded or [] + excluded_note = "" + if excluded: + shown = ", ".join(str(index) for index in excluded[:MAX_DIVERGENT_REPORTED]) + excluded_note = f"; {len(excluded)} unstable in the baseline, excluded (records {shown})" + checked = feature_totals["matched"] + feature_totals["mismatched"] + if status == "matched": + return ( + f"✅ matched {feature_totals['matched']}/{checked} responses " + f"over {records} records{excluded_note}" + ) + + details = [] + for method, counts in sorted(feature_parity.items()): + mismatched = ( + counts["content_mismatch"] + counts["kind_mismatch"] + counts["missing"] + counts.get("nondeterministic", 0) + ) + if not mismatched: + continue + indexes = ", ".join(str(index) for index in counts["divergent_records"]) + details.append(f"`{method}`: {mismatched} (records {indexes})") + + if status == "mismatch": + return "❌ {} content, {} kind mismatches, {} answered in one arm only, {} nondeterministic in the feature only — {}{}".format( + feature_totals["content_mismatch"], + feature_totals["kind_mismatch"], + feature_totals["missing"], + feature_totals.get("nondeterministic", 0), + "; ".join(details) if details else "no per-method detail", + excluded_note, + ) + + reasons = [] + if baseline_totals["mismatched"]: + reasons.append(f"{baseline_totals['mismatched']} baseline-vs-baseline mismatches") + if nondeterministic: + reasons.append(f"{nondeterministic} records answered differently within a run") + if not reasons: + reasons.append("no stable records to judge") + suffix = f"; feature mismatches: {feature_totals['mismatched']}" if feature_totals["mismatched"] else "" + return "⚠️ inconclusive — " + ", ".join(reasons) + suffix + + +def parity_change(parity: dict) -> dict: + """Fold parity into `changes` so the shared verdict helper sees it.""" + totals = parity["feature"]["totals"] + change = { + "pct": 0.0, + "ci_pct": 0.0, + "floor_pct": 0.0, + "sig": "bad" if parity["status"] == "mismatch" else "neutral", + "mismatched": totals["mismatched"], + "excluded": parity.get("excluded_count", 0), + } + if parity["status"] == "inconclusive": + change["informational"] = True + change["informational_reason"] = "informational, responses are not deterministic" + return change + + +def corpus_methods(baseline_runs: list[dict], feature_runs: list[dict]) -> list[str]: + methods = set() + for run in baseline_runs + feature_runs: + for entry in run["responses"].values(): + methods.add(entry["method"]) + for row in run["requests"]: + methods.add(row["method"]) + return sorted(methods) + + +def assert_same_head(runs: list[dict]) -> str | None: + """Refuse to compare runs that replayed against different chain tips.""" + hashes = {} + for run in runs: + head_hash = report_field(run["report"], "head_hash") + if head_hash: + hashes[run["name"]] = head_hash + distinct = set(hashes.values()) + if len(distinct) > 1: + detail = ", ".join(f"{name}={value}" for name, value in sorted(hashes.items())) + raise SystemExit(f"Runs replayed against different chain tips: {detail}") + return next(iter(distinct), None) + + +def fmt_opt_ms(value: float | None) -> str: + return fmt_ms(value) if isinstance(value, (int, float)) else "n/a" + + +def fmt_opt_num(value: float | None, digits: int = 2) -> str: + return f"{value:.{digits}f}" if isinstance(value, (int, float)) else "n/a" + + +def change_str(change: dict | None) -> str: + if not change: + return "n/a" + emoji = {"good": "✅", "bad": "❌", "neutral": "⚪"}[ + "neutral" if change.get("informational") else change["sig"] + ] + details = [f"±{change['ci_pct']:.2f}%", f"floor {change['floor_pct']:.2f}%"] + if "aa_pct" in change: + details.append(f"A/A {change['aa_pct']:+.2f}%") + if change.get("informational"): + details.append("informational") + return f"{change['pct']:+.2f}% {emoji} ({', '.join(details)})" + + +def generate_metric_table(summary: dict, baseline_label: str, feature_label: str) -> str: + baseline = summary["baseline"]["stats"] + feature = summary["feature"]["stats"] + changes = summary["changes"] + rows = [ + ("Mean", "mean_ms", "mean", fmt_opt_ms), + ("P50", "p50_ms", "p50", fmt_opt_ms), + ("P90", "p90_ms", "p90", fmt_opt_ms), + ("P99", "p99_ms", "p99", fmt_opt_ms), + ("Record median", "record_median_ms", "record_median", fmt_opt_ms), + ("Closed-loop rps", "closed_loop_rps", "closed_loop_rps", fmt_opt_num), + ("CPU / request", "cpu_ms_per_request", "cpu_per_request", fmt_opt_ms), + ] + lines = [ + f"| Metric | {baseline_label} | {feature_label} | Change |", + "|--------|------|--------|--------|", + ] + for label, stat_key, change_key, formatter in rows: + if stat_key not in baseline and stat_key not in feature: + continue + lines.append( + f"| {label} | {formatter(baseline.get(stat_key))} | " + f"{formatter(feature.get(stat_key))} | {change_str(changes.get(change_key))} |" + ) + lines.append( + f"| Error rate | {fmt_opt_num(baseline.get('error_rate_pct'))}% | " + f"{fmt_opt_num(feature.get('error_rate_pct'))}% | |" + ) + lines.append( + f"| Open-loop rps | {fmt_opt_num(baseline.get('open_loop_rps'))} | " + f"{fmt_opt_num(feature.get('open_loop_rps'))} | |" + ) + lines.append("") + return "\n".join(lines) + + +def generate_method_table(summary: dict) -> str: + methods = summary.get("methods") or {} + if len(methods) < 2: + return "" + lines = [ + "### Per-method", + "", + "| Method | Records | Baseline mean | Feature mean | Change | Response bytes | Parity |", + "|--------|---------|---------------|--------------|--------|----------------|--------|", + ] + for method in sorted(methods): + entry = methods[method] + baseline = entry["baseline"] + feature = entry["feature"] + totals = entry["parity"] + mismatched = totals["content_mismatch"] + totals["kind_mismatch"] + parity = f"{totals['matched']} matched" if not mismatched else f"❌ {mismatched} mismatched" + lines.append( + "| `{}` | {} | {} | {} | {} | {} | {} |".format( + method, + baseline.get("records", baseline.get("requests", 0)), + fmt_opt_ms(baseline.get("mean_ms")), + fmt_opt_ms(feature.get("mean_ms")), + change_str(entry["changes"].get("mean")), + fmt_opt_num(baseline.get("response_bytes_median"), 0), + parity, + ) + ) + lines.append("") + return "\n".join(lines) + + +def generate_markdown( + summary: dict, + baseline_label: str, + feature_label: str, + behind_baseline: int = 0, + repo: str = "", + baseline_ref: str = "", + baseline_name: str = "", + derek_command: str | None = None, +) -> str: + corpus = summary["corpus"] + lines = ["## Benchmark Results", "", "## Configuration"] + if derek_command: + lines.append(f"- Derek command: `{derek_command}`") + methods = ", ".join(corpus.get("methods") or summary.get("replayed_methods") or []) or "all" + top_gas_note = f" (top-gas {corpus['top_gas']})" if corpus.get("top_gas") else "" + tracer_notes = [] + if corpus.get("tracer"): + tracer_notes.append("tracer " + ", ".join(corpus["tracer"])) + for key in ("tracer_config", "trace_options"): + if corpus.get(key): + tracer_notes.append(f"{key.replace('_', '-')} `{json.dumps(corpus[key], separators=(',', ':'))}`") + tracer_note = f", {'; '.join(tracer_notes)}" if tracer_notes else "" + blocks_note = "" + if corpus.get("blocks_source"): + blocks_note = f", blocks {corpus.get('corpus_from')}..{corpus.get('corpus_to')} from {corpus['blocks_source']}" + if corpus.get("block_param") == "parent": + blocks_note += " at parent-block state" + lines.append( + f"- Corpus: `{corpus.get('name', 'static')}` ({corpus.get('source', 'static')}), " + f"class `{corpus.get('class', 'call')}`{top_gas_note}{tracer_note}{blocks_note}, {corpus.get('records', summary['records'])} records, " + f"methods: {methods}" + ) + lines.append( + f"- Replay: {summary['rps']} rps for {summary['duration']}, " + f"{summary['passes']} passes x {summary['concurrency']} workers, " + f"{summary['warmup_seconds']}s warmup, " + f"max tracing requests: {summary.get('max_tracing_requests') or 'default'}" + ) + if corpus.get("tip"): + lines.append(f"- Tip: {corpus['tip']} (`{corpus.get('tip_hash', 'unknown')}`)") + lines.append("") + if behind_baseline > 0: + plural = "s" if behind_baseline > 1 else "" + diff_link = f"https://github.com/{repo}/compare/{baseline_ref[:12]}...{baseline_name}" + lines.append( + f"> ⚠️ Feature is [**{behind_baseline} commit{plural} behind `{baseline_name}`**]({diff_link}). " + "Consider rebasing for accurate results." + ) + lines.append("") + lines.append(generate_metric_table(summary, baseline_label, feature_label)) + meta_parts = [ + f"{summary['records']} records", + "mode: call", + f"{summary['run_pairs']} run pairs" if summary.get("run_pairs") else "", + ] + lines.append(f"*{', '.join(part for part in meta_parts if part)}*") + lines.append("") + lines.append(f"**Parity:** {summary['parity']['line']}") + method_table = generate_method_table(summary) + if method_table: + lines.append("") + lines.append(method_table) + lines.extend(generate_observability_section(summary)) + return "\n".join(lines) + + +def build_summary(args, baseline_runs: list[dict], feature_runs: list[dict]) -> dict: + rng = random.Random(42) + head_hash = assert_same_head(baseline_runs + feature_runs) + + baseline_stats = arm_stats(baseline_runs) + feature_stats = arm_stats(feature_runs) + changes = compute_changes(rng, baseline_runs, feature_runs, baseline_stats, feature_stats) + parity = compute_parity(baseline_runs, feature_runs) + changes["parity"] = parity_change(parity) + + corpus = {"name": "static", "source": "static", "class": args.klass} + if args.corpus_meta: + meta_path = Path(args.corpus_meta) + if meta_path.is_file(): + with meta_path.open() as f: + corpus.update(json.load(f)) + if head_hash and not corpus.get("tip_hash"): + corpus["tip_hash"] = head_hash + + methods = corpus_methods(baseline_runs, feature_runs) + method_entries = {} + if len(methods) > 1: + for method in methods: + method_baseline = arm_stats(baseline_runs, method) + method_feature = arm_stats(feature_runs, method) + if not method_baseline or not method_feature: + continue + method_entries[method] = { + "baseline": method_baseline, + "feature": method_feature, + "changes": compute_changes( + rng, baseline_runs, feature_runs, method_baseline, method_feature, method + ), + "parity": parity["feature"]["methods"].get( + method, + {"matched": 0, "content_mismatch": 0, "kind_mismatch": 0, "missing": 0, "divergent_records": []}, + ), + } + + records = parity["records"] or corpus.get("records") or 0 + summary = { + "mode": "call", + "blocks": records, + "records": records, + "corpus": corpus, + "replayed_methods": methods, + "rps": args.rps, + "duration": args.duration, + "passes": args.passes, + "concurrency": args.concurrency, + "warmup_seconds": args.warmup_seconds, + "run_pairs": args.run_pairs, + "baseline": { + "name": args.baseline_name or "baseline", + "ref": args.baseline_ref or "main", + "stats": baseline_stats, + }, + "feature": { + "name": args.feature_name or "feature", + "ref": args.feature_ref or "unknown", + "stats": feature_stats, + }, + "changes": changes, + "methods": method_entries, + "parity": parity, + } + if args.max_tracing_requests: + summary["max_tracing_requests"] = args.max_tracing_requests + if args.benchmark_id: + summary["benchmark_id"] = args.benchmark_id + + from_ms, to_ms = resolve_observability_range( + args.observability_from_ms, args.observability_to_ms + ) + logs_url = args.logs_url + traces_url = args.traces_url + if args.benchmark_id and from_ms and to_ms: + logs_url = logs_url or build_grafana_logs_url(args.benchmark_id, from_ms, to_ms) + traces_url = traces_url or build_grafana_traces_url(args.benchmark_id, from_ms, to_ms) + observability = { + key: value + for key, value in { + "benchmark_id": args.benchmark_id, + "from_ms": from_ms, + "to_ms": to_ms, + "grafana_url": args.grafana_url, + "logs_url": logs_url, + "traces_url": traces_url, + }.items() + if value + } + if observability: + summary["observability"] = observability + return summary + + +def parse_args(argv: list[str] | None = None): + parser = argparse.ArgumentParser(description="Summarize an RPC replay benchmark") + parser.add_argument("--baseline-dir", nargs="+", required=True, help="Baseline run directories") + parser.add_argument("--feature-dir", nargs="+", required=True, help="Feature run directories") + parser.add_argument("--output-summary", required=True, help="Output JSON summary path") + parser.add_argument("--output-markdown", required=True, help="Output markdown path") + parser.add_argument("--repo", default="paradigmxyz/reth", help="GitHub repo (owner/name)") + parser.add_argument("--baseline-ref", default=None, help="Baseline commit SHA") + parser.add_argument("--baseline-name", default=None, help="Baseline display name") + parser.add_argument("--feature-name", default=None, help="Feature branch name") + parser.add_argument("--feature-ref", default=None, help="Feature commit SHA") + parser.add_argument("--behind-baseline", type=int, default=0, help="Commits behind baseline") + parser.add_argument("--corpus-meta", default=None, help="corpus.meta.json written by the extract step") + parser.add_argument("--class", dest="klass", default="call", help="Replayed method class") + parser.add_argument("--rps", default="", help="Configured open-loop request rate") + parser.add_argument("--duration", default="", help="Configured open-loop duration") + parser.add_argument("--passes", default="", help="Configured closed-loop passes") + parser.add_argument("--concurrency", default="", help="Configured closed-loop workers") + parser.add_argument("--warmup-seconds", default="", help="Warmup duration in seconds") + parser.add_argument("--run-pairs", type=int, default=None, help="Configured number of run pairs") + parser.add_argument("--max-tracing-requests", default=None, help="Node tracing request limit") + parser.add_argument("--benchmark-id", default=None, help="Benchmark ID used for OTLP labels") + parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL") + parser.add_argument("--logs-url", default=None, help="Grafana Explore URL for benchmark logs") + parser.add_argument("--traces-url", default=None, help="Grafana Explore URL for benchmark traces") + parser.add_argument("--observability-from-ms", type=int, default=None, help="Observability range start") + parser.add_argument("--observability-to-ms", type=int, default=None, help="Observability range end") + parser.add_argument( + "--derek-command", + default=os.environ.get("DEREK_BENCH_COMMAND"), + help="Full derek bench command", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + if args.run_pairs is not None and args.run_pairs < 1: + print("--run-pairs must be greater than zero", file=sys.stderr) + return 1 + baseline_runs = [load_run(Path(path)) for path in args.baseline_dir] + feature_runs = [load_run(Path(path)) for path in args.feature_dir] + + summary = build_summary(args, baseline_runs, feature_runs) + + base_url = f"https://github.com/{args.repo}/commit" + baseline_label = f"[`{summary['baseline']['name']}`]({base_url}/{summary['baseline']['ref']})" + feature_label = f"[`{summary['feature']['name']}`]({base_url}/{summary['feature']['ref']})" + markdown = generate_markdown( + summary, + baseline_label, + feature_label, + behind_baseline=args.behind_baseline, + repo=args.repo, + baseline_ref=summary["baseline"]["ref"], + baseline_name=summary["baseline"]["name"], + derek_command=args.derek_command, + ) + + with open(args.output_summary, "w") as f: + json.dump(summary, f, indent=2) + print(f"Summary written to {args.output_summary}") + with open(args.output_markdown, "w") as f: + f.write(markdown) + print(f"Markdown written to {args.output_markdown}") + print(f"Parity: {summary['parity']['line']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/bench-job-summary-test.js b/.github/scripts/bench-job-summary-test.js new file mode 100644 index 00000000000..f24c0468522 --- /dev/null +++ b/.github/scripts/bench-job-summary-test.js @@ -0,0 +1,39 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const jobSummary = require('./bench-job-summary'); + +for (const mode of ['engine', 'call']) { + test(`${mode} summary preserves attachments and trace downloads`, async (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-summary-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const previous = { ...process.env }; + t.after(() => { process.env = previous; }); + process.env.BENCH_WORK_DIR = directory; + process.env.BENCH_TRACING_CHROME = 'true'; + process.env.BENCH_RESULTS_URL = 'https://github.com/owner/repo/actions/runs/1/artifacts/2'; + const stats = { mean_ms: 1, stddev_ms: 0, p50_ms: 1, p90_ms: 1, p99_ms: 1, mean_mgas_s: 1 }; + fs.writeFileSync(path.join(directory, 'summary.json'), JSON.stringify({ + mode, changes: {}, blocks: 20, + baseline: { name: 'main', ref: 'a'.repeat(40), stats }, + feature: { name: 'feature', ref: 'b'.repeat(40), stats }, + })); + const charts = '\n### Charts\n\n![Latency](https://github.com/user-attachments/assets/chart)\n'; + if (mode === 'engine') fs.writeFileSync(path.join(directory, 'charts.md'), charts); + let markdown; + await jobSummary({ + core: { summary: { addRaw(value) { markdown = value; return { async write() {} }; } } }, + context: { repo: { owner: 'owner', repo: 'repo' } }, + logsUrl: 'https://example.com/logs', + tracesUrl: 'https://example.com/traces', + runId: '1', + }); + assert.equal(markdown.includes(charts), mode === 'engine'); + assert.ok(markdown.includes(`[Download bench-results](${process.env.BENCH_RESULTS_URL})`)); + assert.ok(markdown.includes('[Logs](https://example.com/logs)')); + assert.ok(markdown.includes('[Traces](https://example.com/traces)')); + assert.ok(!markdown.includes('reth-bench-charts')); + }); +} diff --git a/.github/scripts/bench-job-summary.js b/.github/scripts/bench-job-summary.js index e79085eb2fd..7102a9b2055 100644 --- a/.github/scripts/bench-job-summary.js +++ b/.github/scripts/bench-job-summary.js @@ -12,13 +12,12 @@ // // Usage from actions/github-script: // const jobSummary = require('./.github/scripts/bench-job-summary.js'); -// await jobSummary({ core, context, chartSha, grafanaUrl, logsUrl, tracesUrl, runId }); +// await jobSummary({ core, context, grafanaUrl, logsUrl, tracesUrl, runId }); const fs = require('fs'); const { verdict, loadSamplyUrls, - loadTracingChromeUrls, blocksLabel, metricRows, waitTimeRows, @@ -39,7 +38,7 @@ function fmtTargetMetricValue(metric, v) { return fmtMetricValue(v); } -module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, tracesUrl, runId }) { +module.exports = async function ({ core, context, grafanaUrl, logsUrl, tracesUrl, runId }) { let summary; try { summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8')); @@ -82,6 +81,11 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, } md += '\n'; + // Response parity (call mode) + if (summary.parity) { + md += `**Parity:** ${summary.parity.line}\n\n`; + } + // Wait time breakdown const wtRows = waitTimeRows(summary); if (wtRows.length > 0) { @@ -111,21 +115,8 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, md += '\n'; } - // Charts - if (chartSha) { - const prNum = prNumber || '0'; - const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/pr/${prNum}/${runId}`; - const charts = [ - { file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' }, - { file: 'wait_breakdown.png', label: 'Wait Time Breakdown' }, - { file: 'gas_vs_latency.png', label: 'Gas vs Latency' }, - ]; - md += `### Charts\n\n`; - for (const chart of charts) { - md += `
${chart.label}\n\n`; - md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`; - md += `
\n\n`; - } + if (summary.mode !== 'call') { + md += fs.readFileSync(process.env.BENCH_WORK_DIR + '/charts.md', 'utf8'); } // Samply profiles @@ -135,11 +126,8 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, md += `### Samply Profiles\n\n${samplyLinks.join('\n')}\n\n`; } - const tracingChromeUrls = loadTracingChromeUrls(process.env.BENCH_WORK_DIR); - const tracingChromeLinks = Object.entries(tracingChromeUrls) - .map(([run, url]) => `- **${run}**: [Perfetto](${url})`); - if (tracingChromeLinks.length > 0) { - md += `### Chrome Traces\n\n${tracingChromeLinks.join('\n')}\n\n`; + if (process.env.BENCH_TRACING_CHROME === 'true') { + md += `### Chrome Traces\n\n[Download bench-results](${process.env.BENCH_RESULTS_URL}), extract the archive, and open each run's \`tracing-chrome-profile.json\` in [Perfetto](https://ui.perfetto.dev/). GitHub sign-in is required; downloads are available until the artifact expires.\n\n`; } // Observability diff --git a/.github/scripts/bench-reth-summary.py b/.github/scripts/bench-reth-summary.py index 7d1ad07a2f4..e766fbdda94 100755 --- a/.github/scripts/bench-reth-summary.py +++ b/.github/scripts/bench-reth-summary.py @@ -14,7 +14,8 @@ Generates a statistical comparison between baseline and feature. Point estimates use pooled baseline and feature rows. Confidence intervals use whole-run cluster bootstrapping when multiple runs are available. Fails if baseline or feature CSV -is missing or empty. +is missing or empty. Elapsed time and end-to-end throughput use per-run means +from sibling txgen report.json files; missing measured durations remain unavailable. """ from __future__ import annotations @@ -49,6 +50,7 @@ "p90": 1.35, "p99": 5.0, "mgas_s": 1.20, + "end_to_end_mgas_s": 1.20, "wall_clock": 0.70, "persist_wait": 5.0, } @@ -243,7 +245,7 @@ def compute_stats(combined: list[dict]) -> dict: mean_mgas_s = sum(mgas_s_values) / len(mgas_s_values) if mgas_s_values else 0 total_latencies_ms = [r["total_latency_us"] / 1_000 for r in combined] - wall_clock_s = sum(total_latencies_ms) / 1_000 + processing_time_s = sum(total_latencies_ms) / 1_000 mean_total_lat_ms = sum(total_latencies_ms) / n # Persistence wait mean (for main table) @@ -262,7 +264,9 @@ def compute_stats(combined: list[dict]) -> dict: "p90_ms": percentile(sorted_lat, 90), "p99_ms": percentile(sorted_lat, 99), "mean_mgas_s": mean_mgas_s, - "wall_clock_s": wall_clock_s, + "processing_time_s": processing_time_s, + "wall_clock_s": None, + "end_to_end_mgas_s": None, "mean_total_lat_ms": mean_total_lat_ms, "mean_persist_ms": mean_persist_ms, } @@ -286,6 +290,84 @@ def compute_point_stats(runs: list[list[dict]]) -> dict: return stats +def load_elapsed_run(csv_path: str, rows: list[dict]) -> dict | None: + """Read measured run totals, never reconstruct elapsed time from server timings.""" + report_path = Path(csv_path).with_name("report.json") + if not report_path.exists(): + return None + with report_path.open() as f: + report = json.load(f) + totals = report.get("run_stats", {}) + duration_ms = totals.get("duration_ms") + if duration_ms is None: + return None + if ( + isinstance(duration_ms, bool) + or not isinstance(duration_ms, (int, float)) + or not math.isfinite(duration_ms) + or duration_ms <= 0 + ): + raise ValueError(f"Invalid measured duration in {report_path}") + gas = sum(row["gas_used"] for row in rows) + expected = { + "total_blocks": len(rows), + "total_gas": gas, + "start_block": min(row["block_number"] for row in rows), + "end_block": max(row["block_number"] for row in rows), + } + if any(totals.get(key) != value for key, value in expected.items()): + raise ValueError(f"Run totals in {report_path} do not match {csv_path}") + duration_s = duration_ms / 1_000 + return { + "wall_clock_s": duration_s, + "end_to_end_mgas_s": gas / duration_s / 1_000_000, + } + + +def add_elapsed_stats( + baseline_paths: list[str], + feature_paths: list[str], + baseline_runs: list[list[dict]], + feature_runs: list[list[dict]], + baseline_stats: dict, + feature_stats: dict, + ci_stats: dict, +) -> None: + """Average measured run metrics and bootstrap whole runs in matching units. + + A single run has no run-to-run noise estimate. Its elapsed metrics remain + informational; block-level resampling cannot estimate that uncertainty. + """ + baseline = [ + load_elapsed_run(path, rows) + for path, rows in zip(baseline_paths, baseline_runs, strict=True) + ] + feature = [ + load_elapsed_run(path, rows) + for path, rows in zip(feature_paths, feature_runs, strict=True) + ] + if any(run is None for run in baseline + feature): + print("Measured run duration unavailable; elapsed metrics are n/a", file=sys.stderr) + return + ci_stats["elapsed_informational"] = len(baseline) < 2 or len(feature) < 2 + rng = random.Random(42) + for key, ci_key in ( + ("wall_clock_s", "wall_clock_ci_s"), + ("end_to_end_mgas_s", "end_to_end_mgas_ci"), + ): + b = [run[key] for run in baseline] + f = [run[key] for run in feature] + baseline_stats[key] = _mean(b) + feature_stats[key] = _mean(f) + ci_stats[ci_key] = 0.0 + if not ci_stats["elapsed_informational"]: + samples = [ + _mean(rng.choices(f, k=len(f))) - _mean(rng.choices(b, k=len(b))) + for _ in range(BOOTSTRAP_ITERATIONS) + ] + ci_stats[ci_key] = _ci_half_width(samples) + + def compute_wait_stats(combined: list[dict], field: str) -> dict: """Compute mean/p50/p95 for a wait time field (in ms).""" values_ms = [] @@ -407,7 +489,6 @@ def _per_run_metric_values(runs: list[list[dict]]) -> dict[str, list[float]]: "p90_ms": [], "p99_ms": [], "mgas": [], - "wall_clock_ms": [], "persist_ms": [], } for run in runs: @@ -417,7 +498,6 @@ def _per_run_metric_values(runs: list[list[dict]]) -> dict[str, list[float]]: values["p90_ms"].append(stats["p90_ms"]) values["p99_ms"].append(stats["p99_ms"]) values["mgas"].append(stats["mean_mgas_s"]) - values["wall_clock_ms"].append(stats["mean_total_lat_ms"]) values["persist_ms"].append(stats["mean_persist_ms"]) return values @@ -434,7 +514,7 @@ def _cluster_bootstrap_ci( replacement. This estimates run-to-run noise without expanding reused baseline/feature runs into independent block-level datapoints. """ - metrics = ("mean_ms", "p50_ms", "p90_ms", "p99_ms", "mgas", "wall_clock_ms", "persist_ms") + metrics = ("mean_ms", "p50_ms", "p90_ms", "p99_ms", "mgas", "persist_ms") empty = {metric: 0.0 for metric in metrics} if len(baseline_runs) < 2 or len(feature_runs) < 2: return empty @@ -479,10 +559,9 @@ def compute_ci_stats( p90_ci = cluster_ci["p90_ms"] p99_ci = cluster_ci["p99_ms"] mgas_ci = cluster_ci["mgas"] - wall_clock_ci_ms = cluster_ci["wall_clock_ms"] persist_ci_ms = cluster_ci["persist_ms"] else: - pairs, all_lat_diffs, all_mgas_diffs, all_total_lat_diffs, all_persist_diffs = ( + pairs, all_lat_diffs, all_mgas_diffs, _all_total_lat_diffs, all_persist_diffs = ( _paired_data(baseline_runs[0], feature_runs[0]) ) if not all_lat_diffs: @@ -492,7 +571,6 @@ def compute_ci_stats( p90_ci = _bootstrap_percentile_ci(rng, pairs, 90) p99_ci = _bootstrap_percentile_ci(rng, pairs, 99) mgas_ci = _bootstrap_ci(rng, all_mgas_diffs) if all_mgas_diffs else 0.0 - wall_clock_ci_ms = _bootstrap_ci(rng, all_total_lat_diffs) if all_total_lat_diffs else 0.0 persist_ci_ms = _bootstrap_ci(rng, all_persist_diffs) if all_persist_diffs else 0.0 return { @@ -501,7 +579,6 @@ def compute_ci_stats( "p90_ci_ms": p90_ci, "p99_ci_ms": p99_ci, "mgas_ci": mgas_ci, - "wall_clock_ci_ms": wall_clock_ci_ms, "persist_ci_ms": persist_ci_ms, "blocks": blocks, } @@ -578,6 +655,8 @@ def informational_reason( baseline_stats: dict, feature_stats: dict, ) -> str | None: + if metric in ("wall_clock", "end_to_end_mgas_s") and ci_stats.get("elapsed_informational"): + return "informational <2 measured runs per side" if metric == "p99" and ci_stats["blocks"] < P99_MIN_VERDICT_BLOCKS: return f"informational <{P99_MIN_VERDICT_BLOCKS} blocks" if ( @@ -627,9 +706,13 @@ def ci_pct(ci_ms: float, base_ms: float) -> float: ("p90", "p90_ms", "p90_ci_ms", "p90_ms", True), ("p99", "p99_ms", "p99_ci_ms", "p99_ms", True), ("mgas_s", "mean_mgas_s", "mgas_ci", "mean_mgas_s", False), - ("wall_clock", "wall_clock_s", "wall_clock_ci_ms", "mean_total_lat_ms", True), ("persist_wait", "mean_persist_ms", "persist_ci_ms", "mean_persist_ms", True), ] + if baseline_stats.get("wall_clock_s") is not None and feature_stats.get("wall_clock_s") is not None: + metrics.extend([ + ("wall_clock", "wall_clock_s", "wall_clock_ci_s", "wall_clock_s", True), + ("end_to_end_mgas_s", "end_to_end_mgas_s", "end_to_end_mgas_ci", "end_to_end_mgas_s", False), + ]) changes = {} for name, stat_key, ci_key, base_key, lower_is_better in metrics: p = pct(baseline_stats[stat_key], feature_stats[stat_key]) @@ -1671,7 +1754,6 @@ def pct(base: float, feat: float) -> float: return (feat - base) / base * 100.0 if base > 0 else 0.0 gas_pct = pct(run1["mean_mgas_s"], run2["mean_mgas_s"]) - wall_pct = pct(run1["wall_clock_s"], run2["wall_clock_s"]) mean_pct = pct(run1["mean_ms"], run2["mean_ms"]) p50_pct = pct(run1["p50_ms"], run2["p50_ms"]) @@ -1688,7 +1770,6 @@ def pct(base: float, feat: float) -> float: # CI as a percentage of baseline mgas_ci_pct = ci_stats["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0 - wall_ci_pct = ci_stats["wall_clock_ci_ms"] / run1["mean_total_lat_ms"] * 100.0 if run1["mean_total_lat_ms"] > 0 else 0.0 persist_ci_pct = ci_stats["persist_ci_ms"] / run1["mean_persist_ms"] * 100.0 if run1["mean_persist_ms"] > 0 else 0.0 mean_floor = practical_floor_pct("mean", run1["mean_ms"]) @@ -1696,7 +1777,6 @@ def pct(base: float, feat: float) -> float: p90_floor = practical_floor_pct("p90", run1["p90_ms"]) p99_floor = practical_floor_pct("p99", run1["p99_ms"]) mgas_floor = practical_floor_pct("mgas_s", run1["mean_mgas_s"]) - wall_floor = practical_floor_pct("wall_clock", run1["mean_total_lat_ms"]) persist_floor = practical_floor_pct("persist_wait", run1["mean_persist_ms"]) p99_informational = informational_reason("p99", ci_stats, run1, run2) persist_informational = informational_reason("persist_wait", ci_stats, run1, run2) @@ -1708,15 +1788,35 @@ def pct(base: float, feat: float) -> float: lines = [ f"| Metric | {baseline_label} | {feature_label} | Change |", "|--------|------|--------|--------|", - f"| Mean | {fmt_ms(run1['mean_ms'])} | {fmt_ms(run2['mean_ms'])} | {change_str(mean_pct, mean_ci_pct, mean_floor, lower_is_better=True)} |", - f"| P50 | {fmt_ms(run1['p50_ms'])} | {fmt_ms(run2['p50_ms'])} | {change_str(p50_pct, p50_ci_pct, p50_floor, lower_is_better=True)} |", - f"| P90 | {fmt_ms(run1['p90_ms'])} | {fmt_ms(run2['p90_ms'])} | {change_str(p90_pct, p90_ci_pct, p90_floor, lower_is_better=True)} |", - f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, p99_floor, lower_is_better=True, informational=p99_informational)} |", - f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, mgas_floor, lower_is_better=False)} |", - f"| Wall Clock | {fmt_s(run1['wall_clock_s'])} | {fmt_s(run2['wall_clock_s'])} | {change_str(wall_pct, wall_ci_pct, wall_floor, lower_is_better=True)} |", + f"| Execution Mean | {fmt_ms(run1['mean_ms'])} | {fmt_ms(run2['mean_ms'])} | {change_str(mean_pct, mean_ci_pct, mean_floor, lower_is_better=True)} |", + f"| Execution P50 | {fmt_ms(run1['p50_ms'])} | {fmt_ms(run2['p50_ms'])} | {change_str(p50_pct, p50_ci_pct, p50_floor, lower_is_better=True)} |", + f"| Execution P90 | {fmt_ms(run1['p90_ms'])} | {fmt_ms(run2['p90_ms'])} | {change_str(p90_pct, p90_ci_pct, p90_floor, lower_is_better=True)} |", + f"| Execution P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, p99_floor, lower_is_better=True, informational=p99_informational)} |", + f"| Execution Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, mgas_floor, lower_is_better=False)} |", f"| Persist Wait | {fmt_ms(run1['mean_persist_ms'])} | {fmt_ms(run2['mean_persist_ms'])} | {change_str(persist_pct, persist_ci_pct, persist_floor, lower_is_better=True, informational=persist_informational)} |", - "", ] + changes = compute_changes(run1, run2, ci_stats) + for label, key, metric, formatter, lower_is_better in ( + ("Wall Clock", "wall_clock_s", "wall_clock", fmt_s, True), + ("End-to-end Mgas/s", "end_to_end_mgas_s", "end_to_end_mgas_s", fmt_mgas, False), + ): + if metric not in changes: + lines.append(f"| {label} | n/a | n/a | |") + continue + change = changes[metric] + description = change_str( + change["pct"], change["ci_pct"], change["floor_pct"], lower_is_better, + change.get("informational_reason"), + ) + lines.append(f"| {label} | {formatter(run1[key])} | {formatter(run2[key])} | {description} |") + lines.extend([ + "", + "Execution metrics use server processing time (local block time in RPC mode). " + "Wall Clock and end-to-end throughput use measured run duration, including persistence " + "waits and any configured pacing; values are averaged per run. " + "Missing measured durations are shown as n/a.", + "", + ]) meta_parts = [f"{n} {'big blocks' if big_blocks else 'blocks'}", f"mode: {mode}"] if warmup_blocks: meta_parts.append(f"{warmup_blocks} warmup") @@ -2021,6 +2121,11 @@ def main(): print("No comparable baseline and feature results", file=sys.stderr) sys.exit(1) + add_elapsed_stats( + args.baseline_csv, args.feature_csv, baseline_runs, feature_runs, + baseline_stats, feature_stats, ci_stats, + ) + baseline_ref = args.baseline_ref or "main" baseline_name = args.baseline_name or "baseline" feature_name = args.feature_name or "feature" diff --git a/.github/scripts/bench-scheduled-refs.sh b/.github/scripts/bench-scheduled-refs.sh index 6040ed30c8a..44122a0be47 100755 --- a/.github/scripts/bench-scheduled-refs.sh +++ b/.github/scripts/bench-scheduled-refs.sh @@ -26,12 +26,8 @@ # nightly-created — ISO timestamp of the nightly build (nightly only) # release-tag — release tag name (release mode only, e.g. "v2.0.0") # -# Reads: -# state/nightly-last-feature-ref (nightly, from decofe/reth-bench-charts repo) -# state/hourly-last-feature-ref (hourly, from decofe/reth-bench-charts repo) -# state/release-last-feature-ref (release, from decofe/reth-bench-charts repo) -# -# Requires: gh (GitHub CLI), jq, date, git (hourly mode), curl, DEREK_TOKEN env +# Reads bench-state- Actions artifacts from successful runs on this branch. +# Requires: gh, jq, unzip, date, git (hourly mode), GH_TOKEN with actions:read. set -euxo pipefail FORCE="${1:-false}" @@ -74,16 +70,10 @@ if [ "$MODE" = "hourly" ]; then fi echo "::endgroup::" - # --- Step 3: Read last successful feature ref from charts repo --- + # --- Step 3: Read last successful feature ref from Actions artifacts --- echo "::group::Reading persisted state" - LAST_FEATURE_REF="" - STATE_URL="https://raw.githubusercontent.com/decofe/reth-bench-charts/state/state/hourly-last-feature-ref" - if RAW=$(curl -sfL -H "Authorization: token ${DEREK_TOKEN}" "$STATE_URL"); then - LAST_FEATURE_REF=$(echo "$RAW" | tr -d '[:space:]') - echo "Previous feature ref: $LAST_FEATURE_REF" - else - echo "No persisted state found (first run)" - fi + LAST_FEATURE_REF=$(bash "$(dirname "$0")/bench-state.sh" "$REPO" hourly) + echo "Previous feature ref: ${LAST_FEATURE_REF:-none}" echo "::endgroup::" # --- Step 4: Determine baseline and skip logic --- @@ -181,16 +171,10 @@ if [ "$MODE" = "release" ]; then echo "Release commit (baseline): $BASELINE_REF" echo "::endgroup::" - # --- Step 3: Read last successful feature ref from charts repo --- + # --- Step 3: Read last successful feature ref from Actions artifacts --- echo "::group::Reading persisted state" - LAST_FEATURE_REF="" - STATE_URL="https://raw.githubusercontent.com/decofe/reth-bench-charts/state/state/release-last-feature-ref" - if RAW=$(curl -sfL -H "Authorization: token ${DEREK_TOKEN}" "$STATE_URL"); then - LAST_FEATURE_REF=$(echo "$RAW" | tr -d '[:space:]') - echo "Previous feature ref: $LAST_FEATURE_REF" - else - echo "No persisted state found (first run)" - fi + LAST_FEATURE_REF=$(bash "$(dirname "$0")/bench-state.sh" "$REPO" release) + echo "Previous feature ref: ${LAST_FEATURE_REF:-none}" echo "::endgroup::" # --- Step 4: Skip logic --- @@ -278,16 +262,10 @@ else fi echo "::endgroup::" -# --- Step 3: Read last successful feature ref from charts repo --- +# --- Step 3: Read last successful feature ref from Actions artifacts --- echo "::group::Reading persisted state" -LAST_FEATURE_REF="" -STATE_URL="https://raw.githubusercontent.com/decofe/reth-bench-charts/state/state/nightly-last-feature-ref" -if RAW=$(curl -sfL -H "Authorization: token ${DEREK_TOKEN}" "$STATE_URL"); then - LAST_FEATURE_REF=$(echo "$RAW" | tr -d '[:space:]') - echo "Previous feature ref: $LAST_FEATURE_REF" -else - echo "No persisted state found (first run)" -fi +LAST_FEATURE_REF=$(bash "$(dirname "$0")/bench-state.sh" "$REPO" nightly) + echo "Previous feature ref: ${LAST_FEATURE_REF:-none}" echo "::endgroup::" # --- Step 4: Determine baseline and skip logic --- diff --git a/.github/scripts/bench-slack-notify.js b/.github/scripts/bench-slack-notify.js index e71a9b8aa68..ffd631f9d7f 100644 --- a/.github/scripts/bench-slack-notify.js +++ b/.github/scripts/bench-slack-notify.js @@ -25,7 +25,6 @@ const { verdict, isWin, loadSamplyUrls, - loadTracingChromeUrls, blocksLabel, metricRows, } = require('./bench-utils'); @@ -82,16 +81,6 @@ function profileLinks(samplyUrls, prefix) { }); } -function chromeTraceLinks(tracingChromeUrls, prefix) { - return Object.entries(tracingChromeUrls) - .filter(([run]) => run.startsWith(`${prefix}-`)) - .sort(([a], [b]) => a.localeCompare(b, undefined, { numeric: true })) - .map(([run, url]) => { - const index = run.slice(prefix.length + 1); - return `<${url}|Chrome ${index}>`; - }); -} - // Slack shortcodes for verdict (Block Kit header doesn't support unicode emoji) const SLACK_VERDICT = { '⚠️': ':warning:', @@ -135,7 +124,6 @@ function buildSuccessBlocks({ jobUrl, repo, samplyUrls, - tracingChromeUrls, }) { const { emoji, label } = verdict(summary.changes); const headerEmoji = SLACK_VERDICT[emoji] || emoji; @@ -151,18 +139,18 @@ function buildSuccessBlocks({ if (prNumber) metaParts.push(`*<${prUrl}|PR #${prNumber}>*`); metaParts.push(`triggered by ${actorSlackId ? `<@${actorSlackId}>` : `@${actor}`}`); + if (process.env.BENCH_TRACING_CHROME === 'true' && process.env.BENCH_RESULTS_URL) { + metaParts.push(`<${process.env.BENCH_RESULTS_URL}|Download Chrome traces> (extract and open in Perfetto)`); + } + // Baseline/feature lines with samply profile links let baselineLine = `*Baseline:* ${baselineLink}`; const baselineProfiles = profileLinks(samplyUrls, 'baseline'); if (baselineProfiles.length) baselineLine += ` | ${baselineProfiles.join(' | ')}`; - const baselineTraces = chromeTraceLinks(tracingChromeUrls, 'baseline'); - if (baselineTraces.length) baselineLine += ` | ${baselineTraces.join(' | ')}`; let featureLine = `*Feature:* ${featureLink}`; const featureProfiles = profileLinks(samplyUrls, 'feature'); if (featureProfiles.length) featureLine += ` | ${featureProfiles.join(' | ')}`; - const featureTraces = chromeTraceLinks(tracingChromeUrls, 'feature'); - if (featureTraces.length) featureLine += ` | ${featureTraces.join(' | ')}`; const countsLine = blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | '); const configLine = benchConfigLine(); @@ -288,7 +276,6 @@ async function success({ core, context }) { `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR); - const tracingChromeUrls = loadTracingChromeUrls(process.env.BENCH_WORK_DIR); const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.'); const actorSlackId = slackUsers[actor]; @@ -301,7 +288,6 @@ async function success({ core, context }) { jobUrl, repo, samplyUrls, - tracingChromeUrls, }); const text = `Bench: ${summary.baseline.name} vs ${summary.feature.name}`; diff --git a/.github/scripts/bench-state-test.sh b/.github/scripts/bench-state-test.sh new file mode 100644 index 00000000000..f72e998c8cc --- /dev/null +++ b/.github/scripts/bench-state-test.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -euo pipefail +scripts="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +export STATE_TEST_DIR="$scratch" +export GITHUB_OUTPUT="$scratch/output" +export GITHUB_REPOSITORY=owner/repo +export GITHUB_REF_NAME=main +export STATE_TEST_VALUE="" +export STATE_TEST_FAILURE="" +export GH_TOKEN=workflow-test-token +feature=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +previous=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +export STATE_TEST_DATE +STATE_TEST_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +gh() { + case "$1" in + api) + if [[ "$2" == *'/git/'* ]]; then + if [[ "$*" == *'.object.type'* ]]; then printf 'commit\n'; else printf '%s\n' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb; fi + return + fi + [[ "$GH_TOKEN" == workflow-test-token ]] || return 1 + if [[ "$*" == *'--paginate --slurp'* ]]; then + [[ "$STATE_TEST_FAILURE" != list ]] || return 1 + if [[ -n "${STATE_TEST_ARTIFACTS:-}" ]]; then + printf '%s\n' "$STATE_TEST_ARTIFACTS" + elif [[ -z "$STATE_TEST_VALUE" ]]; then + printf '[{"artifacts":[]}]\n' + else + local name="${!#}" + name="${name#*name=}" + name="${name%%&*}" + jq -nc --arg name "$name" '[{artifacts:[{id:10,name:$name,expired:false, + workflow_run:{id:100,head_branch:"main",repository_id:1,head_repository_id:1}}]}]' + fi + elif [[ "$2" == */runs/* ]]; then + [[ "$STATE_TEST_FAILURE" != run ]] || return 1 + jq -nc --arg conclusion "${STATE_TEST_CONCLUSION:-success}" \ + --arg path "${STATE_TEST_PATH:-.github/workflows/bench-scheduled.yml}" \ + --arg event "${STATE_TEST_EVENT:-schedule}" --arg status "${STATE_TEST_STATUS:-completed}" \ + '{path:$path,head_branch:"main",status:$status,conclusion:$conclusion,event:$event}' + elif [[ "$2" == */zip ]]; then + [[ "$STATE_TEST_FAILURE" != download ]] || return 1 + printf '%s\n' "$2" >> "$STATE_TEST_DIR/downloads" + if [[ "$STATE_TEST_FAILURE" == archive ]]; then printf 'invalid zip'; return; fi + printf '%s\n' "$STATE_TEST_VALUE" > "$STATE_TEST_DIR/feature-ref" + (cd "$STATE_TEST_DIR" && zip -q - feature-ref) + else + return 1 + fi + ;; + release) printf '{"tagName":"v1.0.0"}\n' ;; + run) + if [[ "$*" == *'--status=in_progress'* ]]; then + printf '0\n' + else + printf '[{"headSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","createdAt":"%s","conclusion":"success"}]\n' "$STATE_TEST_DATE" + fi + ;; + *) return 1 ;; + esac +} +git() { + case "$*" in + fetch*) ;; + 'rev-parse origin/main') printf '%s\n' aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;; + 'rev-parse origin/main~1') printf '%s\n' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb ;; + 'log -1 --format=%cI '* ) printf '%s\n' "$STATE_TEST_DATE" ;; + *) return 1 ;; + esac +} +export -f gh git + +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +export STATE_TEST_VALUE="$previous" +[[ "$(bash "$scripts/bench-state.sh" owner/repo hourly)" == "$previous" ]] +export STATE_TEST_VALUE=invalid +if bash "$scripts/bench-state.sh" owner/repo hourly 2>/dev/null; then exit 1; fi +if bash "$scripts/bench-state.sh" owner/repo invalid 2>/dev/null; then exit 1; fi +export STATE_TEST_VALUE="$previous" +for failure in list run download archive; do + export STATE_TEST_FAILURE="$failure" + if bash "$scripts/bench-state.sh" owner/repo hourly 2>/dev/null; then exit 1; fi +done +export STATE_TEST_FAILURE="" +echo 'PASS: missing state, real ZIP restore, SHA validation, and API/archive failures' + +# Newer unusable artifacts must not hide the newest usable artifact, even when +# it is on another page. The workflow SHA is deliberately not the feature SHA. +export STATE_TEST_ARTIFACTS +STATE_TEST_ARTIFACTS=$(jq -nc ' + def artifact($id): {id:$id,name:"bench-state-hourly",expired:false, + workflow_run:{id:100,head_branch:"main",head_sha:"cccccccccccccccccccccccccccccccccccccccc", + repository_id:1,head_repository_id:1}}; + [{artifacts:[ + (artifact(90) | .expired=true), + (artifact(80) | .workflow_run.head_branch="test"), + (artifact(70) | .workflow_run.head_repository_id=2), + (artifact(60) | .name="bench-state-nightly")]}, + {artifacts:[artifact(10),artifact(20)]}]') +: > "$scratch/downloads" +[[ "$(bash "$scripts/bench-state.sh" owner/repo hourly)" == "$previous" ]] +[[ "$(cat "$scratch/downloads")" == repos/owner/repo/actions/artifacts/20/zip ]] +export GITHUB_REF_NAME=other-branch +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +export GITHUB_REF_NAME=main +export STATE_TEST_ARTIFACTS='[{"artifacts":[]}]' +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +unset STATE_TEST_ARTIFACTS +for conclusion in failure cancelled skipped; do + export STATE_TEST_CONCLUSION="$conclusion" + [[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +done +unset STATE_TEST_CONCLUSION +export STATE_TEST_STATUS=in_progress +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +unset STATE_TEST_STATUS +export STATE_TEST_PATH=.github/workflows/other.yml +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +unset STATE_TEST_PATH +export STATE_TEST_EVENT=pull_request +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +unset STATE_TEST_EVENT +echo 'PASS: pagination, newest state, expiry, branch/mode/fork/workflow/event isolation, and run outcomes' + +for mode in hourly nightly release; do + for value in "" "$feature" "$previous"; do + export STATE_TEST_VALUE="$value" + : > "$GITHUB_OUTPUT" + bash "$scripts/bench-scheduled-refs.sh" false "$mode" > "$scratch/log" 2>&1 + if [[ "$value" == "$feature" ]]; then + grep -q '^should-skip=true$' "$GITHUB_OUTPUT" + else + grep -q '^should-skip=false$' "$GITHUB_OUTPUT" + fi + if [[ "$value" == "$previous" ]]; then + grep -q "^baseline-ref=$previous$" "$GITHUB_OUTPUT" + fi + done + export STATE_TEST_VALUE="$feature" + : > "$GITHUB_OUTPUT" + bash "$scripts/bench-scheduled-refs.sh" true "$mode" > "$scratch/log" 2>&1 + grep -q '^should-skip=false$' "$GITHUB_OUTPUT" + export STATE_TEST_FAILURE=list + if bash "$scripts/bench-scheduled-refs.sh" false "$mode" > "$scratch/log" 2>&1; then exit 1; fi + export STATE_TEST_FAILURE="" + echo "PASS: $mode first run, unchanged/changed commit, force, and API failure" +done diff --git a/.github/scripts/bench-state.sh b/.github/scripts/bench-state.sh new file mode 100644 index 00000000000..a8f39977095 --- /dev/null +++ b/.github/scripts/bench-state.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Restore the last successful benchmark commit from an Actions artifact. +# Usage: bench-state.sh owner/repo hourly|nightly|release +# Requires GH_TOKEN with actions:read. State is scoped to GITHUB_REF_NAME (main +# outside Actions). Missing/expired state bootstraps; API/corruption errors fail. +set -euo pipefail + +repo=${1:?expected owner/repo} +mode=${2:?expected benchmark mode} +case "$mode" in hourly|nightly|release) ;; *) echo "Invalid benchmark mode: $mode" >&2; exit 1 ;; esac +name="bench-state-$mode" +branch=${GITHUB_REF_NAME:-main} + +# Filter before downloading: dispatches on test branches must not change main's +# baseline. Pagination matters when hourly state accumulates over its retention. +artifacts=$(gh api --paginate --slurp "repos/$repo/actions/artifacts?name=$name&per_page=100") +candidates=$(jq -r --arg name "$name" --arg branch "$branch" ' + [.[].artifacts[] | select(.name == $name and .expired == false + and .workflow_run.head_branch == $branch + and .workflow_run.head_repository_id == .workflow_run.repository_id)] + | sort_by(.id) | reverse | .[] | [.id, .workflow_run.id] | @tsv +' <<< "$artifacts") + +while IFS=$'\t' read -r artifact_id run_id; do + [[ -n "$artifact_id" ]] || continue + run=$(gh api "repos/$repo/actions/runs/$run_id") + if ! jq -e --arg branch "$branch" ' + .path == ".github/workflows/bench-scheduled.yml" and .head_branch == $branch + and .status == "completed" and .conclusion == "success" + and (.event == "schedule" or .event == "workflow_dispatch") + ' <<< "$run" >/dev/null; then + continue + fi + scratch=$(mktemp -d) + trap 'rm -rf "$scratch"' EXIT + gh api "repos/$repo/actions/artifacts/$artifact_id/zip" > "$scratch/state.zip" + value=$(unzip -p "$scratch/state.zip" feature-ref) + if [[ ! "$value" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid commit SHA in $name artifact $artifact_id" >&2 + exit 1 + fi + echo "Restored $name from run $run_id" >&2 + printf '%s\n' "$value" + exit 0 +done <<< "$candidates" diff --git a/.github/scripts/bench-txgen-build.sh b/.github/scripts/bench-txgen-build.sh index 4c040558064..8803641548c 100755 --- a/.github/scripts/bench-txgen-build.sh +++ b/.github/scripts/bench-txgen-build.sh @@ -8,6 +8,9 @@ # payload format. set -euxo pipefail +# Use the runner's Git authentication configuration when fetching dependencies. +export CARGO_NET_GIT_FETCH_WITH_CLI=true + MODE="$1" SOURCE_DIR="$2" COMMIT="$3" @@ -22,10 +25,8 @@ else fi EXTRA_FEATURES="" -EXTRA_RUSTFLAGS="" if [ "${BENCH_TRACY:-off}" != "off" ]; then EXTRA_FEATURES="tracy,tracy-client/ondemand" - EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes" fi build_node_binary() { @@ -39,7 +40,7 @@ build_node_binary() { fi # shellcheck disable=SC2086 - RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \ + RUSTFLAGS="-C target-cpu=native -C force-frame-pointers=yes" \ cargo build --locked --profile profiling $NODE_PKG $workspace_arg $features_arg } diff --git a/.github/scripts/bench-txgen-extract.sh b/.github/scripts/bench-txgen-extract.sh index a513dcfd1d4..8d1c9953cbc 100755 --- a/.github/scripts/bench-txgen-extract.sh +++ b/.github/scripts/bench-txgen-extract.sh @@ -15,8 +15,28 @@ # /benchmark-blocks.ndjson # In RPC mode, the corresponding names end in `-transactions.ndjson`. # +# In call mode it instead writes a request corpus replayed by `bench call`: +# /corpus.jsonl[.gz] one JSON-RPC request per line +# /corpus.meta.json record counts and the tip it was built for +# +# Corpus sources +# -------------- +# `corpus=static` generates the corpus from the chain around the snapshot tip +# for the requested class. `corpus=` replays a corpus staged on the runner +# under BENCH_CORPUS_DIR (default /reth-bench/corpora), which holds: +# +# .jsonl.gz the corpus itself, gzip optional +# .sha256 optional checksum, verified before the corpus is used +# tracers/.js JavaScript tracers selectable as tracer=js: +# +# Provisioning that directory is manual (copy the file onto the runner). Names +# must not contain path separators. Corpus contents are never logged and never +# uploaded as build artifacts; only the record counts in corpus.meta.json are. +# # Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS -# Optional env: BENCH_EXECUTION_MODE, BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL +# Optional env: BENCH_EXECUTION_MODE, BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL, +# BENCH_CORPUS, BENCH_CORPUS_DIR, BENCH_CALL_CLASS, BENCH_CALL_METHODS, BENCH_CALL_TOP_GAS, +# BENCH_CALL_TRACER, BENCH_CALL_TRACER_CONFIG, BENCH_CALL_TRACE_OPTIONS, BENCH_CALL_NAMESPACE set -euxo pipefail BINARY="$1" @@ -69,6 +89,11 @@ RETH_ARGS=( --no-persist-peers ) +# Call mode probes the snapshot's tracing range before the node is stopped. +if [ "$EXECUTION_MODE" = "call" ]; then + RETH_ARGS+=(--http.api "eth,net,web3,debug,trace") +fi + if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then RETH_ARGS+=(--debug.startup-sync-state-idle) fi @@ -104,12 +129,304 @@ HEAD_HEX=$(jq -r '.result' <<< "$HEAD_JSON") HEAD_DEC=$((16#${HEAD_HEX#0x})) echo "Snapshot chain tip: ${HEAD_DEC}" -# Stop the throwaway node and recover snapshot before extraction. -sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true -sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true -kill "${TAIL_PID:-}" 2>/dev/null || true -TAIL_PID= -sudo schelk recover -y --kill || true +HEAD_HASH=$(curl -sf http://127.0.0.1:8545 -X POST \ + -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}' \ + | jq -r '.result.hash') + +# Returns non-zero when the snapshot can no longer trace the given block, which +# happens once the block falls out of the retained account and storage history. +# The archive node serves the blocks after the snapshot tip for the call +# classes; probe it once so an outage falls back instead of failing the run. +archive_rpc_reachable() { + local response + response=$(curl -sf -m 15 "$BENCH_RPC_URL" -X POST -H 'Content-Type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}' 2>/dev/null) || return 1 + [[ "$response" == *'"result"'* ]] +} + +block_is_traceable() { + local block="$1" + local block_hex tx_hash response + block_hex=$(printf '0x%x' "$block") + tx_hash=$(curl -sf http://127.0.0.1:8545 -X POST \ + -H 'Content-Type: application/json' \ + -d "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"${block_hex}\",false],\"id\":1}" \ + | jq -r '.result.transactions[0] // empty') + if [ -z "$tx_hash" ]; then + echo "Block ${block} has no transactions, skipping traceability probe" + return 0 + fi + # The block class never resolves hashes, so it probes the block-addressed + # method it will replay; the transaction class probes by hash, which also + # detects a pruned transaction lookup. + local probe + if [ "$CALL_CLASS" = "traceblock" ]; then + probe="{\"jsonrpc\":\"2.0\",\"method\":\"debug_traceBlockByNumber\",\"params\":[\"${block_hex}\",{\"tracer\":\"callTracer\",\"tracerConfig\":{\"onlyTopCall\":true}}],\"id\":1}" + else + probe="{\"jsonrpc\":\"2.0\",\"method\":\"debug_traceTransaction\",\"params\":[\"${tx_hash}\",{\"tracer\":\"callTracer\"}],\"id\":1}" + fi + if ! response=$(curl -sf http://127.0.0.1:8545 -X POST \ + -H 'Content-Type: application/json' \ + -d "$probe"); then + echo "traceability probe request for block ${block} failed" + return 1 + fi + if jq -e 'has("error")' <<< "$response" > /dev/null 2>&1; then + local message + message="$(jq -r '.error.message' <<< "$response")" + # A transaction the node just listed in the block but cannot find by hash + # means the transaction lookup index is pruned, so no block in the range + # will do any better; report that instead of shrinking the range. + if [[ "$message" == *"transaction not found"* ]]; then + echo "::error::Snapshot has no transaction hash index (transaction lookup is pruned); hash-addressed classes cannot run on it. Use class=tracecall or class=traceblock, or a snapshot that keeps the transaction lookup." + exit 1 + fi + echo "Block ${block} is not traceable: ${message}" + return 1 + fi + return 0 +} + +CALL_BLOCKS="${BENCH_BLOCKS:-20}" +CALL_CLASS="${BENCH_CALL_CLASS:-call}" +CORPUS_SOURCE="${BENCH_CORPUS:-static}" +if [ "$EXECUTION_MODE" = "call" ] && [ "$CORPUS_SOURCE" = "static" ]; then + case "$CALL_CLASS" in + tracetx|traceblock) + # The snapshot is a pruned full node whose persisted state can lag its + # header tip, so the newest blocks may trace against stale state ("nonce + # too high"). Walk down from the tip to the newest block that actually + # traces, then window below it; also shrink the window if its oldest + # block has fallen past the retained history floor. + CORPUS_TOP="$HEAD_DEC" + probe_budget="${BENCH_TRACE_PROBE_DEPTH:-1024}" + while ! block_is_traceable "$CORPUS_TOP"; do + probe_budget=$(( probe_budget - 1 )) + if [ "$probe_budget" -le 0 ] || [ "$CORPUS_TOP" -le 1 ]; then + echo "::error::Snapshot has no traceable block near its tip ${HEAD_DEC} (searched down to ${CORPUS_TOP}); its persisted state lags too far behind the header for class ${CALL_CLASS}." + exit 1 + fi + CORPUS_TOP=$(( CORPUS_TOP - 1 )) + done + if [ "$CORPUS_TOP" -ne "$HEAD_DEC" ]; then + echo "Newest traceable block is ${CORPUS_TOP}, $(( HEAD_DEC - CORPUS_TOP )) below the tip" + fi + while [ "$CALL_BLOCKS" -gt 1 ] && ! block_is_traceable $(( CORPUS_TOP - CALL_BLOCKS + 1 )); do + CALL_BLOCKS=$(( CALL_BLOCKS / 2 )) + echo "Retrying traceability probe with ${CALL_BLOCKS} source blocks" + done + echo "Snapshot traces blocks $(( CORPUS_TOP - CALL_BLOCKS + 1 ))..${CORPUS_TOP}" + ;; + esac +fi + +# Stop the throwaway node and recover the snapshot before extraction. Call mode +# keeps it up: the trace classes read their blocks from it, and the exit trap +# stops it once the corpus is staged. +if [ "$EXECUTION_MODE" != "call" ]; then + sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true + sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true + kill "${TAIL_PID:-}" 2>/dev/null || true + TAIL_PID= + sudo schelk recover -y --kill || true +fi + +# --- Stage the call corpus --- + +if [ "$EXECUTION_MODE" = "call" ]; then + CALL_METHODS="${BENCH_CALL_METHODS:-}" + CORPUS_FILE="$OUTPUT_DIR/corpus.jsonl" + + read_corpus() { + case "$1" in + *.gz) gzip -dc "$1" ;; + *) cat "$1" ;; + esac + } + + if [ "$CORPUS_SOURCE" = "static" ]; then + TXGEN_ETHEREUM="$(which txgen-ethereum)" + if [ "$CALL_BLOCKS" -le 0 ]; then + echo "::error::BENCH_BLOCKS must be greater than 0" + exit 1 + fi + case "$CALL_CLASS" in + call|tracecall) + # Transactions of the blocks after the tip are exactly the calls a + # client would have served against the tip state; only the archive + # node has those blocks. When it is unreachable, the blocks just below + # the tip on the snapshot stand in: their calls replay against the tip + # state too, only with more reverts. + CORPUS_FORMAT=calls + if archive_rpc_reachable; then + CORPUS_FROM=$(( HEAD_DEC + 1 )) + CORPUS_TO=$(( HEAD_DEC + CALL_BLOCKS )) + CORPUS_RPC="$BENCH_RPC_URL" + CORPUS_BLOCKS_SOURCE=archive + else + echo "::warning::Archive node ${BENCH_RPC_URL} is unreachable; building the ${CALL_CLASS} corpus from snapshot blocks below the tip" + CORPUS_FROM=$(( HEAD_DEC - CALL_BLOCKS + 1 )) + CORPUS_TO="$HEAD_DEC" + CORPUS_RPC="http://127.0.0.1:8545" + CORPUS_BLOCKS_SOURCE=snapshot + # These transactions already executed, so replay each against the + # state before its block instead of the tip, where its preconditions + # no longer hold. + CORPUS_BLOCK_PARAM=parent + fi + ;; + tracetx|traceblock) + # These blocks are on the snapshot, so the throwaway node serves them + # and the archive node is not needed. CORPUS_TOP is the newest block the + # probe found traceable, which may sit below the header tip. + CORPUS_FROM=$(( CORPUS_TOP - CALL_BLOCKS + 1 )) + CORPUS_TO="$CORPUS_TOP" + CORPUS_FORMAT=traces + CORPUS_RPC="http://127.0.0.1:8545" + CORPUS_BLOCKS_SOURCE=snapshot + ;; + *) + echo "::error::Unknown call class: ${CALL_CLASS}" + exit 1 + ;; + esac + CORPUS_METHOD_ARGS=() + if [ -n "$CALL_METHODS" ]; then + CORPUS_METHOD_ARGS+=(--methods "$CALL_METHODS") + fi + if [ -n "${BENCH_CALL_TOP_GAS:-}" ]; then + CORPUS_METHOD_ARGS+=(--top-gas "$BENCH_CALL_TOP_GAS") + fi + if [ -n "${BENCH_CALL_TRACER:-}" ]; then + IFS=',' read -r -a tracer_specs <<< "$BENCH_CALL_TRACER" + for spec in "${tracer_specs[@]}"; do + spec="${spec// /}" + [ -z "$spec" ] && continue + case "$spec" in + js:*) + tracer_name="${spec#js:}" + case "$tracer_name" in + */*|.*|"") + echo "::error::Invalid tracer name: ${spec}" + exit 1 + ;; + esac + tracer_file="${BENCH_CORPUS_DIR:-/reth-bench/corpora}/tracers/${tracer_name}.js" + if [ ! -f "$tracer_file" ]; then + echo "::error::JS tracer ${tracer_name} not found at ${tracer_file}" + exit 1 + fi + CORPUS_METHOD_ARGS+=(--tracer "js:${tracer_file}") + ;; + *) + CORPUS_METHOD_ARGS+=(--tracer "$spec") + ;; + esac + done + fi + if [ -n "${BENCH_CALL_TRACER_CONFIG:-}" ]; then + CORPUS_METHOD_ARGS+=(--tracer-config "$BENCH_CALL_TRACER_CONFIG") + fi + if [ -n "${CORPUS_BLOCK_PARAM:-}" ]; then + CORPUS_METHOD_ARGS+=(--block-param "$CORPUS_BLOCK_PARAM") + fi + if [ -n "${BENCH_CALL_TRACE_OPTIONS:-}" ]; then + CORPUS_METHOD_ARGS+=(--trace-options "$BENCH_CALL_TRACE_OPTIONS") + fi + echo "Generating ${CALL_CLASS} corpus from blocks ${CORPUS_FROM}..${CORPUS_TO} (methods: ${CALL_METHODS:-default})" + "$TXGEN_ETHEREUM" extract \ + --rpc "$CORPUS_RPC" \ + --from "$CORPUS_FROM" \ + --to "$CORPUS_TO" \ + --format "$CORPUS_FORMAT" \ + "${CORPUS_METHOD_ARGS[@]}" \ + -o "$CORPUS_FILE" + else + case "$CORPUS_SOURCE" in + */*|.*) + echo "::error::Invalid corpus name: ${CORPUS_SOURCE}" + exit 1 + ;; + esac + CORPUS_DIR="${BENCH_CORPUS_DIR:-/reth-bench/corpora}" + CORPUS_SRC="" + for candidate in "$CORPUS_DIR/$CORPUS_SOURCE.jsonl.gz" "$CORPUS_DIR/$CORPUS_SOURCE.jsonl"; do + if [ -f "$candidate" ]; then + CORPUS_SRC="$candidate" + break + fi + done + if [ -z "$CORPUS_SRC" ]; then + echo "::error::Corpus ${CORPUS_SOURCE} not found in ${CORPUS_DIR} (expected ${CORPUS_SOURCE}.jsonl.gz or ${CORPUS_SOURCE}.jsonl)" + exit 1 + fi + CORPUS_SHA256="$CORPUS_DIR/$CORPUS_SOURCE.sha256" + if [ -f "$CORPUS_SHA256" ]; then + EXPECTED_SHA="$(awk '{print $1; exit}' "$CORPUS_SHA256")" + ACTUAL_SHA="$(sha256sum "$CORPUS_SRC" | awk '{print $1}')" + if [ "$EXPECTED_SHA" != "$ACTUAL_SHA" ]; then + echo "::error::Checksum mismatch for corpus ${CORPUS_SOURCE}" + exit 1 + fi + echo "Corpus ${CORPUS_SOURCE} checksum verified" + fi + case "$CORPUS_SRC" in + *.gz) CORPUS_FILE="$OUTPUT_DIR/corpus.jsonl.gz" ;; + esac + cp "$CORPUS_SRC" "$CORPUS_FILE" + fi + + CORPUS_RECORDS=$(read_corpus "$CORPUS_FILE" | wc -l | tr -d ' ') + if [ "$CORPUS_RECORDS" -le 0 ]; then + echo "::error::Corpus ${CORPUS_SOURCE} is empty" + exit 1 + fi + CORPUS_RECORDS_PER_METHOD=$(read_corpus "$CORPUS_FILE" | jq -r '.method' | sort | uniq -c \ + | jq -R -s 'split("\n") + | map(select(length > 0) | capture("^\\s*(?\\d+)\\s+(?.+)$") | {(.method): (.count | tonumber)}) + | add // {}') + jq -n \ + --arg source "$CORPUS_SOURCE" \ + --arg name "$CORPUS_SOURCE" \ + --arg class "$CALL_CLASS" \ + --arg methods "$CALL_METHODS" \ + --argjson records "$CORPUS_RECORDS" \ + --argjson records_per_method "$CORPUS_RECORDS_PER_METHOD" \ + --argjson tip "$HEAD_DEC" \ + --arg tip_hash "$HEAD_HASH" \ + --arg blocks_source "${CORPUS_BLOCKS_SOURCE:-}" \ + --arg block_param "${CORPUS_BLOCK_PARAM:-}" \ + --argjson corpus_from "${CORPUS_FROM:-null}" \ + --argjson corpus_to "${CORPUS_TO:-null}" \ + --argjson top_gas "${BENCH_CALL_TOP_GAS:-null}" \ + --arg tracer "${BENCH_CALL_TRACER:-}" \ + --arg namespace "${BENCH_CALL_NAMESPACE:-}" \ + --argjson tracer_config "${BENCH_CALL_TRACER_CONFIG:-null}" \ + --argjson trace_options "${BENCH_CALL_TRACE_OPTIONS:-null}" \ + '{ + source: (if $source == "static" then "static" else "custom" end), + name: $name, + class: $class, + methods: (if $methods == "" then [] else ($methods | split(",")) end), + records: $records, + records_per_method: $records_per_method, + tip: $tip, + tip_hash: $tip_hash, + blocks_source: (if $blocks_source == "" then null else $blocks_source end), + block_param: (if $block_param == "" then null else $block_param end), + corpus_from: $corpus_from, + corpus_to: $corpus_to, + top_gas: $top_gas, + tracer: (if $tracer == "" then null else ($tracer | split(",")) end), + namespace: (if $namespace == "" then null else $namespace end), + tracer_config: $tracer_config, + trace_options: $trace_options, + }' > "$OUTPUT_DIR/corpus.meta.json" + + echo "Corpus staged: ${CORPUS_RECORDS} records in ${CORPUS_FILE}" + exit 0 +fi # --- Extract payloads from the remote RPC --- diff --git a/.github/scripts/bench-txgen-install.sh b/.github/scripts/bench-txgen-install.sh index 6ac1d31897c..366a01d01eb 100755 --- a/.github/scripts/bench-txgen-install.sh +++ b/.github/scripts/bench-txgen-install.sh @@ -4,6 +4,7 @@ # # Optional env: # TXGEN_REPO – txgen repository URL (default: https://github.com/tempoxyz/txgen) +# TXGEN_REF – branch or commit to install instead of the default branch set -euxo pipefail TXGEN_REPO="${TXGEN_REPO:-https://github.com/tempoxyz/txgen}" @@ -31,4 +32,14 @@ elif [ -n "${TXGEN_TOKEN:-${GH_PROJECT_TOKEN:-${DEREK_PAT:-${DEREK_TOKEN:-}}}}" fi export CARGO_NET_GIT_FETCH_WITH_CLI=true -cargo install --git "$TXGEN_REPO" --locked txgen-ethereum bench-cli +INSTALL_ARGS=() +if [ -n "${TXGEN_REF:-}" ]; then + if [[ "$TXGEN_REF" =~ ^[0-9a-f]{7,40}$ ]]; then + INSTALL_ARGS+=(--rev "$TXGEN_REF") + else + INSTALL_ARGS+=(--branch "$TXGEN_REF") + fi + # A pinned ref must replace whatever build of the same version is installed. + INSTALL_ARGS+=(--force) +fi +cargo install --git "$TXGEN_REPO" ${INSTALL_ARGS[@]+"${INSTALL_ARGS[@]}"} --locked txgen-ethereum bench-cli diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh index b709cf7c544..902f4ebb174 100755 --- a/.github/scripts/bench-txgen-run.sh +++ b/.github/scripts/bench-txgen-run.sh @@ -4,10 +4,16 @@ # mount snapshot → start node → extract source blocks → warmup → replay → # convert txgen JSON report into legacy benchmark CSVs. # +# In call mode it replays the request corpus staged by bench-txgen-extract.sh +# instead, and writes responses.ndjson, record_timings.csv, requests.csv, +# report.json and cpu.json for bench-call-summary.py. +# # Usage: bench-txgen-run.sh