diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 0000000..e90e49b --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,63 @@ +# Benchmarking Intelligent Contracts + +`gltest bench` measures repeated contract-method execution without replacing the +existing pytest-based `gltest` command. + +## Direct mode + +Use direct mode for fast local execution: + +```bash +gltest bench contracts/Storage.py \ + --method get_storage \ + --mode direct \ + --iterations 100 \ + --warmup 5 +``` + +Constructor and method arguments are JSON arrays: + +```bash +gltest bench contracts/Counter.py \ + --method increment \ + --constructor-args '[0]' \ + --args '[2]' +``` + +## Studio mode + +Use studio mode to measure the configured Studio/RPC execution path: + +```bash +gltest bench contracts/Oracle.py \ + --method resolve \ + --mode studio \ + --iterations 20 +``` + +The contract is deployed once before measurement. A read-only method is executed +with `.call()`; a state-changing method is executed with `.transact()`. Studio +latency therefore includes the configured RPC and transaction wait path. Direct +mode measures native local execution. + +## Metrics + +The report contains: + +- mean, median, p95, minimum, and maximum wall-clock latency; +- total measured time and throughput in operations per second; +- process CPU time consumed during the measured loop; +- peak process RSS sampled after each measured execution. + +Warmup executions and contract deployment are excluded from measured latency. +State-changing methods are not reset between iterations, so benchmarks that need +identical state should use an idempotent method or prepare a contract specifically +for benchmarking. + +For machine-readable output: + +```bash +gltest bench contracts/Storage.py --method get_storage --json +``` + +Add `--include-samples` to include each latency sample in the JSON result. diff --git a/gltest/bench/__init__.py b/gltest/bench/__init__.py new file mode 100644 index 0000000..ce2aa9b --- /dev/null +++ b/gltest/bench/__init__.py @@ -0,0 +1,3 @@ +from .runner import BenchmarkResult, BenchmarkRunner + +__all__ = ["BenchmarkResult", "BenchmarkRunner"] diff --git a/gltest/bench/runner.py b/gltest/bench/runner.py new file mode 100644 index 0000000..2f25075 --- /dev/null +++ b/gltest/bench/runner.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import math +import statistics +import time +from dataclasses import dataclass +from typing import Any, Callable + +import psutil + + +@dataclass(frozen=True) +class BenchmarkResult: + """Summary of a measured benchmark run.""" + + mode: str + iterations: int + warmup_iterations: int + total_ms: float + mean_ms: float + median_ms: float + p95_ms: float + min_ms: float + max_ms: float + throughput_ops_s: float | None + cpu_time_ms: float + process_rss_peak_mb: float + samples_ms: tuple[float, ...] + + def as_dict(self, *, include_samples: bool = False) -> dict[str, Any]: + result: dict[str, Any] = { + "mode": self.mode, + "iterations": self.iterations, + "warmup_iterations": self.warmup_iterations, + "total_ms": self.total_ms, + "mean_ms": self.mean_ms, + "median_ms": self.median_ms, + "p95_ms": self.p95_ms, + "min_ms": self.min_ms, + "max_ms": self.max_ms, + "throughput_ops_s": self.throughput_ops_s, + "cpu_time_ms": self.cpu_time_ms, + "process_rss_peak_mb": self.process_rss_peak_mb, + } + if include_samples: + result["samples_ms"] = list(self.samples_ms) + return result + + +class BenchmarkRunner: + """Measure repeated execution of a prepared Intelligent Contract operation. + + Deployment/setup is intentionally outside the measured operation so benchmark + samples represent the contract method itself. For studio mode this still + includes the configured RPC/wait/consensus path used by the supplied + operation. + """ + + def __init__(self, operation: Callable[[], Any], *, mode: str) -> None: + if mode not in {"direct", "studio"}: + raise ValueError("mode must be 'direct' or 'studio'") + self._operation = operation + self._mode = mode + + def run( + self, + *, + iterations: int = 100, + warmup_iterations: int = 3, + ) -> BenchmarkResult: + if iterations <= 0: + raise ValueError("iterations must be greater than zero") + if warmup_iterations < 0: + raise ValueError("warmup_iterations cannot be negative") + + for _ in range(warmup_iterations): + self._operation() + + process = psutil.Process() + samples_ms: list[float] = [] + rss_peak = process.memory_info().rss + cpu_before = process.cpu_times() + + for _ in range(iterations): + started_ns = time.perf_counter_ns() + self._operation() + finished_ns = time.perf_counter_ns() + samples_ms.append((finished_ns - started_ns) / 1_000_000) + rss_peak = max(rss_peak, process.memory_info().rss) + + cpu_after = process.cpu_times() + + total_ms = sum(samples_ms) + total_seconds = total_ms / 1_000 + sorted_samples = sorted(samples_ms) + p95_index = max(0, math.ceil(len(sorted_samples) * 0.95) - 1) + cpu_time_ms = ( + (cpu_after.user - cpu_before.user) + + (cpu_after.system - cpu_before.system) + ) * 1_000 + + return BenchmarkResult( + mode=self._mode, + iterations=iterations, + warmup_iterations=warmup_iterations, + total_ms=total_ms, + mean_ms=statistics.mean(samples_ms), + median_ms=statistics.median(samples_ms), + p95_ms=sorted_samples[p95_index], + min_ms=min(samples_ms), + max_ms=max(samples_ms), + throughput_ops_s=(iterations / total_seconds) if total_seconds > 0 else None, + cpu_time_ms=cpu_time_ms, + process_rss_peak_mb=rss_peak / (1024 * 1024), + samples_ms=tuple(samples_ms), + ) diff --git a/gltest_cli/bench.py b/gltest_cli/bench.py new file mode 100644 index 0000000..6b69acf --- /dev/null +++ b/gltest_cli/bench.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import argparse +import contextlib +import json +from pathlib import Path +import sys +from typing import Any, Sequence + +from gltest.bench import BenchmarkResult, BenchmarkRunner + + +def _json_list(value: str, *, option: str) -> list[Any]: + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise argparse.ArgumentTypeError(f"{option} must be valid JSON") from exc + if not isinstance(parsed, list): + raise argparse.ArgumentTypeError(f"{option} must be a JSON array") + return parsed + + +def _run_direct( + contract_path: Path, + method_name: str, + constructor_args: list[Any], + method_args: list[Any], + iterations: int, + warmup_iterations: int, +) -> BenchmarkResult: + from gltest.direct import VMContext, deploy_contract + + vm = VMContext() + with vm.activate(): + contract = deploy_contract(contract_path, vm, *constructor_args) + try: + method = getattr(contract, method_name) + except AttributeError as exc: + raise ValueError(f"Contract has no method '{method_name}'") from exc + + operation = lambda: method(*method_args) + return BenchmarkRunner(operation, mode="direct").run( + iterations=iterations, + warmup_iterations=warmup_iterations, + ) + + +def _run_studio( + contract_path: Path, + method_name: str, + constructor_args: list[Any], + method_args: list[Any], + iterations: int, + warmup_iterations: int, +) -> BenchmarkResult: + from gltest import get_contract_factory + from gltest.assertions import tx_execution_succeeded + + factory = get_contract_factory(contract_file_path=contract_path) + contract = factory.deploy(args=constructor_args) + + try: + method_factory = getattr(contract, method_name) + except AttributeError as exc: + raise ValueError(f"Contract has no method '{method_name}'") from exc + + contract_function = method_factory(args=method_args) + + if contract_function.read_only: + operation = contract_function.call + else: + def operation(): + receipt = contract_function.transact() + if not tx_execution_succeeded(receipt): + raise RuntimeError( + f"Studio benchmark transaction failed for '{method_name}'" + ) + return receipt + + return BenchmarkRunner(operation, mode="studio").run( + iterations=iterations, + warmup_iterations=warmup_iterations, + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="gltest bench", + description=( + "Benchmark an Intelligent Contract method. Deployment is excluded " + "from measured samples." + ), + ) + parser.add_argument("contract", type=Path, help="Path to the contract Python file") + parser.add_argument("--method", required=True, help="Contract method to benchmark") + parser.add_argument( + "--mode", + choices=("direct", "studio"), + default="direct", + help="Execution path to measure (default: direct)", + ) + parser.add_argument( + "--iterations", + type=int, + default=100, + help="Measured executions (default: 100)", + ) + parser.add_argument( + "--warmup", + type=int, + default=3, + help="Unmeasured warmup executions (default: 3)", + ) + parser.add_argument( + "--constructor-args", + default="[]", + metavar="JSON", + help='Constructor arguments as a JSON array (default: "[]")', + ) + parser.add_argument( + "--args", + default="[]", + metavar="JSON", + help='Method arguments as a JSON array (default: "[]")', + ) + parser.add_argument( + "--json", + action="store_true", + help="Print machine-readable JSON", + ) + parser.add_argument( + "--include-samples", + action="store_true", + help="Include per-iteration latency samples in JSON output", + ) + return parser + + +def _format_human(result: BenchmarkResult) -> str: + throughput = ( + f"{result.throughput_ops_s:.2f} ops/s" + if result.throughput_ops_s is not None + else "n/a" + ) + return "\n".join( + [ + f"Mode: {result.mode}", + f"Iterations: {result.iterations} (+{result.warmup_iterations} warmup)", + f"Mean latency: {result.mean_ms:.3f} ms", + f"Median latency: {result.median_ms:.3f} ms", + f"P95 latency: {result.p95_ms:.3f} ms", + f"Min / max: {result.min_ms:.3f} / {result.max_ms:.3f} ms", + f"Throughput: {throughput}", + f"CPU time: {result.cpu_time_ms:.3f} ms", + f"Peak process RSS: {result.process_rss_peak_mb:.2f} MiB", + ] + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + if args.iterations <= 0: + parser.error("--iterations must be greater than zero") + if args.warmup < 0: + parser.error("--warmup cannot be negative") + if not args.contract.is_file(): + parser.error(f"contract file not found: {args.contract}") + + try: + constructor_args = _json_list( + args.constructor_args, + option="--constructor-args", + ) + method_args = _json_list(args.args, option="--args") + except argparse.ArgumentTypeError as exc: + parser.error(str(exc)) + + runner = _run_direct if args.mode == "direct" else _run_studio + + if args.json: + with contextlib.redirect_stdout(sys.stderr): + result = runner( + args.contract, + args.method, + constructor_args, + method_args, + args.iterations, + args.warmup, + ) + else: + result = runner( + args.contract, + args.method, + constructor_args, + method_args, + args.iterations, + args.warmup, + ) + + if args.json: + print( + json.dumps( + result.as_dict(include_samples=args.include_samples), + sort_keys=True, + ) + ) + else: + print(_format_human(result)) + return 0 diff --git a/gltest_cli/main.py b/gltest_cli/main.py index a77e435..fa5e39b 100644 --- a/gltest_cli/main.py +++ b/gltest_cli/main.py @@ -1,5 +1,11 @@ +import sys + import pytest def main(): + if len(sys.argv) > 1 and sys.argv[1] == "bench": + from gltest_cli.bench import main as bench_main + + return bench_main(sys.argv[2:]) return pytest.main() diff --git a/pyproject.toml b/pyproject.toml index 82931b6..5f8e4f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,8 @@ dependencies = [ "genlayer-py>=0.19.0rc2,<0.20.0", "colorama>=0.4.6", "pyyaml", - "python-dotenv" + "python-dotenv", + "psutil>=5.9", ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/tests/gltest/bench/test_runner.py b/tests/gltest/bench/test_runner.py new file mode 100644 index 0000000..6b24351 --- /dev/null +++ b/tests/gltest/bench/test_runner.py @@ -0,0 +1,131 @@ +from types import SimpleNamespace + +import pytest + +from gltest.bench import BenchmarkRunner +import gltest.bench.runner as runner_module + + +class _FakeProcess: + def __init__(self): + self._rss = iter( + [ + 10 * 1024 * 1024, + 20 * 1024 * 1024, + 30 * 1024 * 1024, + 25 * 1024 * 1024, + ] + ) + self._cpu = iter( + [ + SimpleNamespace(user=1.0, system=2.0), + SimpleNamespace(user=1.004, system=2.002), + ] + ) + + def memory_info(self): + return SimpleNamespace(rss=next(self._rss)) + + def cpu_times(self): + return next(self._cpu) + + +def test_benchmark_runner_reports_latency_throughput_and_resources(monkeypatch): + timestamps = iter( + [ + 0, + 1_000_000, + 1_000_000, + 3_000_000, + 3_000_000, + 6_000_000, + ] + ) + calls = [] + + monkeypatch.setattr(runner_module.time, "perf_counter_ns", lambda: next(timestamps)) + monkeypatch.setattr(runner_module.psutil, "Process", _FakeProcess) + + result = BenchmarkRunner(lambda: calls.append("call"), mode="direct").run( + iterations=3, + warmup_iterations=0, + ) + + assert calls == ["call", "call", "call"] + assert result.samples_ms == (1.0, 2.0, 3.0) + assert result.total_ms == 6.0 + assert result.mean_ms == 2.0 + assert result.median_ms == 2.0 + assert result.p95_ms == 3.0 + assert result.min_ms == 1.0 + assert result.max_ms == 3.0 + assert result.throughput_ops_s == pytest.approx(500.0) + assert result.cpu_time_ms == pytest.approx(6.0) + assert result.process_rss_peak_mb == 30.0 + + +def test_benchmark_runner_warms_up_before_measurement(monkeypatch): + timestamps = iter([0, 1_000_000]) + calls = [] + + fake_process = _FakeProcess() + monkeypatch.setattr(runner_module.time, "perf_counter_ns", lambda: next(timestamps)) + monkeypatch.setattr(runner_module.psutil, "Process", lambda: fake_process) + + BenchmarkRunner(lambda: calls.append("call"), mode="studio").run( + iterations=1, + warmup_iterations=2, + ) + + assert calls == ["call", "call", "call"] + + +@pytest.mark.parametrize( + ("iterations", "warmup_iterations", "message"), + [ + (0, 0, "iterations"), + (-1, 0, "iterations"), + (1, -1, "warmup_iterations"), + ], +) +def test_benchmark_runner_rejects_invalid_counts( + iterations, + warmup_iterations, + message, +): + with pytest.raises(ValueError, match=message): + BenchmarkRunner(lambda: None, mode="direct").run( + iterations=iterations, + warmup_iterations=warmup_iterations, + ) + + +def test_result_dict_samples_are_opt_in(monkeypatch): + timestamps = iter([0, 1_000_000]) + fake_process = _FakeProcess() + monkeypatch.setattr(runner_module.time, "perf_counter_ns", lambda: next(timestamps)) + monkeypatch.setattr(runner_module.psutil, "Process", lambda: fake_process) + + result = BenchmarkRunner(lambda: None, mode="direct").run( + iterations=1, + warmup_iterations=0, + ) + + assert "samples_ms" not in result.as_dict() + assert result.as_dict(include_samples=True)["samples_ms"] == [1.0] + + +def test_zero_duration_uses_json_safe_null_throughput(monkeypatch): + timestamps = iter([0, 0]) + fake_process = _FakeProcess() + monkeypatch.setattr(runner_module.time, "perf_counter_ns", lambda: next(timestamps)) + monkeypatch.setattr(runner_module.psutil, "Process", lambda: fake_process) + + result = BenchmarkRunner(lambda: None, mode="direct").run( + iterations=1, + warmup_iterations=0, + ) + + assert result.total_ms == 0.0 + assert result.throughput_ops_s is None + assert result.as_dict()["throughput_ops_s"] is None diff --git a/tests/gltest_cli/test_bench.py b/tests/gltest_cli/test_bench.py new file mode 100644 index 0000000..746a135 --- /dev/null +++ b/tests/gltest_cli/test_bench.py @@ -0,0 +1,247 @@ +import json +import sys + +import pytest + +from gltest.bench import BenchmarkResult +import gltest_cli.bench as bench_cli +import gltest_cli.main as cli_main + + +def _result(mode="direct", throughput=500.0): + return BenchmarkResult( + mode=mode, + iterations=2, + warmup_iterations=1, + total_ms=4.0, + mean_ms=2.0, + median_ms=2.0, + p95_ms=3.0, + min_ms=1.0, + max_ms=3.0, + throughput_ops_s=throughput, + cpu_time_ms=1.0, + process_rss_peak_mb=42.0, + samples_ms=(1.0, 3.0), + ) + + +def test_gltest_keeps_forwarding_non_bench_args_to_pytest(monkeypatch): + monkeypatch.setattr(sys, "argv", ["gltest", "-q", "tests/gltest/"]) + seen = {} + + def fake_pytest_main(): + seen["called"] = True + return 7 + + monkeypatch.setattr(cli_main.pytest, "main", fake_pytest_main) + + assert cli_main.main() == 7 + assert seen == {"called": True} + + +def test_gltest_dispatches_bench_subcommand(monkeypatch): + monkeypatch.setattr( + sys, + "argv", + ["gltest", "bench", "contract.py", "--method", "ping"], + ) + seen = {} + + def fake_bench_main(argv): + seen["argv"] = argv + return 0 + + monkeypatch.setattr(bench_cli, "main", fake_bench_main) + + assert cli_main.main() == 0 + assert seen["argv"] == ["contract.py", "--method", "ping"] + + +def test_bench_json_output_and_arguments(monkeypatch, tmp_path, capsys): + contract = tmp_path / "Contract.py" + contract.write_text("# contract") + seen = {} + + def fake_run( + contract_path, + method_name, + constructor_args, + method_args, + iterations, + warmup_iterations, + ): + seen.update( + contract_path=contract_path, + method_name=method_name, + constructor_args=constructor_args, + method_args=method_args, + iterations=iterations, + warmup_iterations=warmup_iterations, + ) + return _result() + + monkeypatch.setattr(bench_cli, "_run_direct", fake_run) + + rc = bench_cli.main( + [ + str(contract), + "--method", + "ping", + "--iterations", + "2", + "--warmup", + "1", + "--constructor-args", + "[1]", + "--args", + "[\"hello\"]", + "--json", + "--include-samples", + ] + ) + + assert rc == 0 + assert seen == { + "contract_path": contract, + "method_name": "ping", + "constructor_args": [1], + "method_args": ["hello"], + "iterations": 2, + "warmup_iterations": 1, + } + captured = capsys.readouterr() + payload = json.loads(captured.out) + assert payload["p95_ms"] == 3.0 + assert payload["samples_ms"] == [1.0, 3.0] + + +def test_json_mode_redirects_contract_stdout_to_stderr(monkeypatch, tmp_path, capsys): + contract = tmp_path / "Contract.py" + contract.write_text("# contract") + + def fake_run(*_args): + print("contract noise") + return _result() + + monkeypatch.setattr(bench_cli, "_run_direct", fake_run) + + assert ( + bench_cli.main( + [ + str(contract), + "--method", + "ping", + "--json", + ] + ) + == 0 + ) + + captured = capsys.readouterr() + json.loads(captured.out) + assert "contract noise" not in captured.out + assert "contract noise" in captured.err + + +@pytest.mark.parametrize( + ("option", "value", "message"), + [ + ("--args", "{", "--args must be valid JSON"), + ("--constructor-args", "{}", "--constructor-args must be a JSON array"), + ], +) +def test_invalid_json_args_report_cli_usage_error( + monkeypatch, + tmp_path, + capsys, + option, + value, + message, +): + contract = tmp_path / "Contract.py" + contract.write_text("# contract") + monkeypatch.setattr(bench_cli, "_run_direct", lambda *_args: _result()) + + with pytest.raises(SystemExit) as exc: + bench_cli.main( + [ + str(contract), + "--method", + "ping", + option, + value, + ] + ) + + assert exc.value.code == 2 + assert message in capsys.readouterr().err + + +def test_bench_selects_studio_adapter(monkeypatch, tmp_path): + contract = tmp_path / "Contract.py" + contract.write_text("# contract") + seen = {} + + def fake_run(*args): + seen["args"] = args + return _result(mode="studio") + + monkeypatch.setattr(bench_cli, "_run_studio", fake_run) + + assert ( + bench_cli.main( + [ + str(contract), + "--method", + "ping", + "--mode", + "studio", + "--iterations", + "2", + ] + ) + == 0 + ) + assert seen["args"][1] == "ping" + + +def test_studio_write_benchmark_rejects_failed_receipts(monkeypatch): + import gltest + import gltest.assertions as assertions + + class FakeContractFunction: + read_only = False + + def transact(self): + return {"status": "failed"} + + class FakeContract: + def ping(self, args=None): + return FakeContractFunction() + + class FakeFactory: + def deploy(self, args=None): + return FakeContract() + + monkeypatch.setattr( + gltest, + "get_contract_factory", + lambda contract_file_path: FakeFactory(), + ) + monkeypatch.setattr(assertions, "tx_execution_succeeded", lambda _receipt: False) + + with pytest.raises(RuntimeError, match="Studio benchmark transaction failed"): + bench_cli._run_studio( + contract_path=None, + method_name="ping", + constructor_args=[], + method_args=[], + iterations=1, + warmup_iterations=0, + ) + + +def test_human_output_handles_unavailable_throughput(): + output = bench_cli._format_human(_result(throughput=None)) + assert "Throughput: n/a" in output