feat: add intelligent contract benchmark runner

docs/benchmarking.md

@@ -0,0 +1,63 @@
+# Benchmarking Intelligent Contracts
+
+`gltest bench` measures repeated contract-method execution without replacing the
+existing pytest-based `gltest` command.
+
+## Direct mode
+
+Use direct mode for fast local execution:
+
+```bash
+gltest bench contracts/Storage.py \
+  --method get_storage \
+  --mode direct \
+  --iterations 100 \
+  --warmup 5
+```
+
+Constructor and method arguments are JSON arrays:
+
+```bash
+gltest bench contracts/Counter.py \
+  --method increment \
+  --constructor-args '[0]' \
+  --args '[2]'
+```
+
+## Studio mode
+
+Use studio mode to measure the configured Studio/RPC execution path:
+
+```bash
+gltest bench contracts/Oracle.py \
+  --method resolve \
+  --mode studio \
+  --iterations 20
+```
+
+The contract is deployed once before measurement. A read-only method is executed
+with `.call()`; a state-changing method is executed with `.transact()`. Studio
+latency therefore includes the configured RPC and transaction wait path. Direct
+mode measures native local execution.
+
+## Metrics
+
+The report contains:
+
+- mean, median, p95, minimum, and maximum wall-clock latency;
+- total measured time and throughput in operations per second;
+- process CPU time consumed during the measured loop;
+- peak process RSS sampled after each measured execution.
+
+Warmup executions and contract deployment are excluded from measured latency.
+State-changing methods are not reset between iterations, so benchmarks that need
+identical state should use an idempotent method or prepare a contract specifically
+for benchmarking.
+
+For machine-readable output:
+
+```bash
+gltest bench contracts/Storage.py --method get_storage --json
+```
+
+Add `--include-samples` to include each latency sample in the JSON result.

gltest/bench/__init__.py

@@ -0,0 +1,3 @@
+from .runner import BenchmarkResult, BenchmarkRunner
+
+__all__ = ["BenchmarkResult", "BenchmarkRunner"]

gltest/bench/runner.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import math
+import statistics
+import time
+from dataclasses import dataclass
+from typing import Any, Callable
+
+import psutil
+
+
+@dataclass(frozen=True)
+class BenchmarkResult:
+    """Summary of a measured benchmark run."""
+
+    mode: str
+    iterations: int
+    warmup_iterations: int
+    total_ms: float
+    mean_ms: float
+    median_ms: float
+    p95_ms: float
+    min_ms: float
+    max_ms: float
+    throughput_ops_s: float | None
+    cpu_time_ms: float
+    process_rss_peak_mb: float
+    samples_ms: tuple[float, ...]
+
+    def as_dict(self, *, include_samples: bool = False) -> dict[str, Any]:
+        result: dict[str, Any] = {
+            "mode": self.mode,
+            "iterations": self.iterations,
+            "warmup_iterations": self.warmup_iterations,
+            "total_ms": self.total_ms,
+            "mean_ms": self.mean_ms,
+            "median_ms": self.median_ms,
+            "p95_ms": self.p95_ms,
+            "min_ms": self.min_ms,
+            "max_ms": self.max_ms,
+            "throughput_ops_s": self.throughput_ops_s,
+            "cpu_time_ms": self.cpu_time_ms,
+            "process_rss_peak_mb": self.process_rss_peak_mb,
+        }
+        if include_samples:
+            result["samples_ms"] = list(self.samples_ms)
+        return result
+
+
+class BenchmarkRunner:
+    """Measure repeated execution of a prepared Intelligent Contract operation.
+
+    Deployment/setup is intentionally outside the measured operation so benchmark
+    samples represent the contract method itself. For studio mode this still
+    includes the configured RPC/wait/consensus path used by the supplied
+    operation.
+    """
+
+    def __init__(self, operation: Callable[[], Any], *, mode: str) -> None:
+        if mode not in {"direct", "studio"}:
+            raise ValueError("mode must be 'direct' or 'studio'")
+        self._operation = operation
+        self._mode = mode
+
+    def run(
+        self,
+        *,
+        iterations: int = 100,
+        warmup_iterations: int = 3,
+    ) -> BenchmarkResult:
+        if iterations <= 0:
+            raise ValueError("iterations must be greater than zero")
+        if warmup_iterations < 0:
+            raise ValueError("warmup_iterations cannot be negative")
+
+        for _ in range(warmup_iterations):
+            self._operation()
+
+        process = psutil.Process()
+        samples_ms: list[float] = []
+        rss_peak = process.memory_info().rss
+        cpu_before = process.cpu_times()
+
+        for _ in range(iterations):
+            started_ns = time.perf_counter_ns()
+            self._operation()
+            finished_ns = time.perf_counter_ns()
+            samples_ms.append((finished_ns - started_ns) / 1_000_000)
+            rss_peak = max(rss_peak, process.memory_info().rss)
+
+        cpu_after = process.cpu_times()
+
+        total_ms = sum(samples_ms)
+        total_seconds = total_ms / 1_000
+        sorted_samples = sorted(samples_ms)
+        p95_index = max(0, math.ceil(len(sorted_samples) * 0.95) - 1)
+        cpu_time_ms = (
+            (cpu_after.user - cpu_before.user)
+            + (cpu_after.system - cpu_before.system)
+        ) * 1_000
+
+        return BenchmarkResult(
+            mode=self._mode,
+            iterations=iterations,
+            warmup_iterations=warmup_iterations,
+            total_ms=total_ms,
+            mean_ms=statistics.mean(samples_ms),
+            median_ms=statistics.median(samples_ms),
+            p95_ms=sorted_samples[p95_index],
+            min_ms=min(samples_ms),
+            max_ms=max(samples_ms),
+            throughput_ops_s=(iterations / total_seconds) if total_seconds > 0 else None,
+            cpu_time_ms=cpu_time_ms,
+            process_rss_peak_mb=rss_peak / (1024 * 1024),
+            samples_ms=tuple(samples_ms),
+        )

gltest_cli/bench.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+from pathlib import Path
+import sys
+from typing import Any, Sequence
+
+from gltest.bench import BenchmarkResult, BenchmarkRunner
+
+
+def _json_list(value: str, *, option: str) -> list[Any]:
+    try:
+        parsed = json.loads(value)
+    except json.JSONDecodeError as exc:
+        raise argparse.ArgumentTypeError(f"{option} must be valid JSON") from exc
+    if not isinstance(parsed, list):
+        raise argparse.ArgumentTypeError(f"{option} must be a JSON array")
+    return parsed
+
+
+def _run_direct(
+    contract_path: Path,
+    method_name: str,
+    constructor_args: list[Any],
+    method_args: list[Any],
+    iterations: int,
+    warmup_iterations: int,
+) -> BenchmarkResult:
+    from gltest.direct import VMContext, deploy_contract
+
+    vm = VMContext()
+    with vm.activate():
+        contract = deploy_contract(contract_path, vm, *constructor_args)
+        try:
+            method = getattr(contract, method_name)
+        except AttributeError as exc:
+            raise ValueError(f"Contract has no method '{method_name}'") from exc
+
+        operation = lambda: method(*method_args)
+        return BenchmarkRunner(operation, mode="direct").run(
+            iterations=iterations,
+            warmup_iterations=warmup_iterations,
+        )
+
+
+def _run_studio(
+    contract_path: Path,
+    method_name: str,
+    constructor_args: list[Any],
+    method_args: list[Any],
+    iterations: int,
+    warmup_iterations: int,
+) -> BenchmarkResult:
+    from gltest import get_contract_factory
+    from gltest.assertions import tx_execution_succeeded
+
+    factory = get_contract_factory(contract_file_path=contract_path)
+    contract = factory.deploy(args=constructor_args)
+
+    try:
+        method_factory = getattr(contract, method_name)
+    except AttributeError as exc:
+        raise ValueError(f"Contract has no method '{method_name}'") from exc
+
+    contract_function = method_factory(args=method_args)
+
+    if contract_function.read_only:
+        operation = contract_function.call
+    else:
+        def operation():
+            receipt = contract_function.transact()
+            if not tx_execution_succeeded(receipt):
+                raise RuntimeError(
+                    f"Studio benchmark transaction failed for '{method_name}'"
+                )
+            return receipt
+
+    return BenchmarkRunner(operation, mode="studio").run(
+        iterations=iterations,
+        warmup_iterations=warmup_iterations,
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="gltest bench",
+        description=(
+            "Benchmark an Intelligent Contract method. Deployment is excluded "
+            "from measured samples."
+        ),
+    )
+    parser.add_argument("contract", type=Path, help="Path to the contract Python file")
+    parser.add_argument("--method", required=True, help="Contract method to benchmark")
+    parser.add_argument(
+        "--mode",
+        choices=("direct", "studio"),
+        default="direct",
+        help="Execution path to measure (default: direct)",
+    )
+    parser.add_argument(
+        "--iterations",
+        type=int,
+        default=100,
+        help="Measured executions (default: 100)",
+    )
+    parser.add_argument(
+        "--warmup",
+        type=int,
+        default=3,
+        help="Unmeasured warmup executions (default: 3)",
+    )
+    parser.add_argument(
+        "--constructor-args",
+        default="[]",
+        metavar="JSON",
+        help='Constructor arguments as a JSON array (default: "[]")',
+    )
+    parser.add_argument(
+        "--args",
+        default="[]",
+        metavar="JSON",
+        help='Method arguments as a JSON array (default: "[]")',
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Print machine-readable JSON",
+    )
+    parser.add_argument(
+        "--include-samples",
+        action="store_true",
+        help="Include per-iteration latency samples in JSON output",
+    )
+    return parser
+
+
+def _format_human(result: BenchmarkResult) -> str:
+    throughput = (
+        f"{result.throughput_ops_s:.2f} ops/s"
+        if result.throughput_ops_s is not None
+        else "n/a"
+    )
+    return "\n".join(
+        [
+            f"Mode: {result.mode}",
+            f"Iterations: {result.iterations} (+{result.warmup_iterations} warmup)",
+            f"Mean latency: {result.mean_ms:.3f} ms",
+            f"Median latency: {result.median_ms:.3f} ms",
+            f"P95 latency: {result.p95_ms:.3f} ms",
+            f"Min / max: {result.min_ms:.3f} / {result.max_ms:.3f} ms",
+            f"Throughput: {throughput}",
+            f"CPU time: {result.cpu_time_ms:.3f} ms",
+            f"Peak process RSS: {result.process_rss_peak_mb:.2f} MiB",
+        ]
+    )
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    if args.iterations <= 0:
+        parser.error("--iterations must be greater than zero")
+    if args.warmup < 0:
+        parser.error("--warmup cannot be negative")
+    if not args.contract.is_file():
+        parser.error(f"contract file not found: {args.contract}")
+
+    try:
+        constructor_args = _json_list(
+            args.constructor_args,
+            option="--constructor-args",
+        )
+        method_args = _json_list(args.args, option="--args")
+    except argparse.ArgumentTypeError as exc:
+        parser.error(str(exc))
+
+    runner = _run_direct if args.mode == "direct" else _run_studio
+
+    if args.json:
+        with contextlib.redirect_stdout(sys.stderr):
+            result = runner(
+                args.contract,
+                args.method,
+                constructor_args,
+                method_args,
+                args.iterations,
+                args.warmup,
+            )
+    else:
+        result = runner(
+            args.contract,
+            args.method,
+            constructor_args,
+            method_args,
+            args.iterations,
+            args.warmup,
+        )
+
+    if args.json:
+        print(
+            json.dumps(
+                result.as_dict(include_samples=args.include_samples),
+                sort_keys=True,
+            )
+        )
+    else:
+        print(_format_human(result))
+    return 0