diff --git a/docs/CLI.md b/docs/CLI.md new file mode 100644 index 0000000..32e078f --- /dev/null +++ b/docs/CLI.md @@ -0,0 +1,149 @@ +# JevBench command line + +The CLI lives in `jevbench/cli.py`. After `pip install -e .` it is available as +`jevbench`; without installing, `python -m jevbench.cli` is equivalent. + +``` +jevbench {run,summarize} ... +jevbench --help | jevbench run --help | jevbench summarize --help +``` + +There are two subcommands: + +| subcommand | does | +|-------------|------| +| `run` | runs one adapter over task files, one request per task, serially and with no retries; writes per-item results, raw evidence and budget ledger entries | +| `summarize` | aggregates a results file into overall, per-family and per-split metrics and prints them as JSON | + +A local run on the public split is a smoke test, not a published result. + +## Where files go + +The runner refuses to write per-item results or raw responses inside the +repository (`ValueError`), and it never overwrites: the results file and each +raw response file are opened in exclusive-create mode. Use a fresh directory +outside the repo for every run, e.g. `/tmp/jb-/`. + +## `jevbench run` + +``` +jevbench run --tasks --adapter --results [options] +``` + +| flag | default | meaning | +|------|---------|---------| +| `--tasks` | required | task JSONL file(s), comma-separated. Public split: `datasets/public/easy.jsonl`, `hard.jsonl`, `original.jsonl` | +| `--adapter` | required | one of `typesafe`, `systemone_list`, `gradio_space`, `local_openjev`, `openai_compat`, `needle_local`, `semif_direct`, `so1_decider`, `remote_inproc`, `sg_system_one`, `djev`, `laya_local`, `gliner2_local`, `verdict_local`, `paw_local`, `classifier_dev`, `certo_local`, `qwen_flash_linear` | +| `--endpoint` | none | base URL of the system under test; for local-checkpoint adapters (e.g. `local_openjev`, `laya_local`) the local snapshot path. Required except for `typesafe`, `djev`, `needle_local`, `semif_direct`, `so1_decider`, `sg_system_one`, `classifier_dev` (exit status 2 if missing) | +| `--model` | none | model name to request; `openai_compat` sends `""` if unset | +| `--key-env` | `TYPESAFE_API_KEY` | **name** of the environment variable holding the API key. The key is never printed or logged | +| `--results` | required | per-item results JSONL to create (must not exist; outside the repo) | +| `--ledger` | `/../private/ledger.jsonl` | shared budget ledger JSONL, appended to | +| `--raw-dir` | `/../private/raw_responses` | raw request/response evidence, one JSON file per task (outside the repo) | +| `--cap-usd` | `15.0` | budget cap in USD; if the ledger already records a lower cap, the lower one applies. A request that would exceed it is not sent and the run stops | +| `--reserve-usd` | `0.02` | minimum USD reserved per request before it is sent; an adapter's own estimate is used when it is larger | +| `--price-in-per-m` | none | USD per million input tokens | +| `--price-out-per-m` | none | USD per million output tokens. With both prices set, `cost_usd` is derived from reported usage; with both `0`, `cost_usd` is `0`; otherwise it stays `null` (unmetered is not free) | +| `--limit` | none | only run the first N tasks | +| `--delay-s` | `0.0` | pause between requests, for someone else's free public demo | +| `--request-options` | none | JSON object of extra request body fields, e.g. `'{"reasoning_effort": "low"}'`; recorded in the manifest | +| `--revision` | none | pinned revision of a local checkpoint (local-checkpoint adapters only) | +| `--run-label` | `--model`, else `--adapter` | label stored in the manifest | +| `--cost-basis` | none | why this route's per-decision cost is what it is, e.g. `no_billable_account_public_demo` | +| `--manifest` | none | write the run's exact settings as JSON to this path | + +Environment: `JEVBENCH_WARM_LOAD=1` makes adapters that have a `load()` +method load their weights before the clock starts; the load time is printed. + +The run stops early on a budget stop, on HTTP 401/403/429, or after 3 +consecutive infrastructure failures (a 422 counts as a wrong answer, not a +failure). Remaining tasks are left unattempted. + +**Exit status:** `0` every task attempted, `3` run stopped early, `2` usage +error (bad flags or missing `--endpoint`). + +**Output files:** + +* `--results`: one JSON object per attempted task (`task_id`, `family`, + `split`, `status`, `valid`, `correct`, `predicted`, `probs`, + `probs_as_returned`, `probs_source`, `model`, `latency_s`, `usage`, + `cost_usd`, `cost_basis`, `charged_usd`, `raw_sha256`, ...). +* `--raw-dir`: `.json` holding the request body, the raw + response and the HTTP status; its hash is the record's `raw_sha256`. +* `--ledger`: `cap`, `reserve` and `settle` events. +* `--manifest` (optional): adapter, endpoint, requested and resolved models, + request options, prices, cost basis, `dataset_hash`, planned/attempted + counts, charged USD, start and finish times. + +## `jevbench summarize` + +``` +jevbench summarize --tasks --results [options] +``` + +| flag | default | meaning | +|------|---------|---------| +| `--tasks` | required | the same task JSONL file(s) the run used, comma-separated | +| `--results` | required | per-item results JSONL written by `jevbench run` | +| `--ledger` | none | if given and the file exists, its total is reported as `ledger_charged_usd` (else `0.0`) | +| `--public-export` | none | also write an allowlisted aggregate JSON (no item text or labels, `ledger_charged_usd` null, plus `dataset_hash`) | +| `--include-excluded` | off | passed to `summarize()` as `headline_only=False`, which the function currently does not use: the output is identical with or without it | + +The summary JSON goes to stdout. Items with an `exclude_reason` or no +expected answer are never scored; they still count as attempted. It fails +with `ValueError` on duplicate records or records for tasks not in `--tasks`. + +## End-to-end example on the public split, offline + +None of the built-in adapters is a dummy, and the CLI has no plug-in hook for +custom adapters (for that, see the Python example in +[ADAPTERS.md](ADAPTERS.md#6-minimal-worked-example)). To try the full CLI +path offline, run `openai_compat` against a tiny local stand-in server that +answers every request with a uniform distribution. Save this **outside** the +repo, e.g. as `/tmp/stub_server.py`: + +```python +import json +from http.server import BaseHTTPRequestHandler, HTTPServer + +class H(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + keys = body["response_format"]["json_schema"]["schema"]["properties"]["probabilities"]["required"] + content = json.dumps({"probabilities": {k: 1 / len(keys) for k in keys}}) + out = json.dumps({"model": "uniform-stub", "choices": [{"message": {"content": content}}], + "usage": {"prompt_tokens": 0, "completion_tokens": 0}}).encode() + self.send_response(200); self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(out))); self.end_headers(); self.wfile.write(out) + def log_message(self, *a): pass + +HTTPServer(("127.0.0.1", 8765), H).serve_forever() +``` + +```sh +python /tmp/stub_server.py & +STUB_KEY=offline jevbench run --tasks datasets/public/easy.jsonl --adapter openai_compat \ + --endpoint http://127.0.0.1:8765/v1 --model uniform-stub --key-env STUB_KEY \ + --price-in-per-m 0 --price-out-per-m 0 --cost-basis local_stub_no_tariff \ + --results /tmp/jb/results.jsonl --ledger /tmp/jb/ledger.jsonl \ + --raw-dir /tmp/jb/raw --manifest /tmp/jb/manifest.json +jevbench summarize --tasks datasets/public/easy.jsonl --results /tmp/jb/results.jsonl \ + --ledger /tmp/jb/ledger.jsonl --public-export /tmp/jb/public.json +kill %1 +``` + +Real output (summary abridged): + +``` +[jevbench] adapter=openai_compat endpoint=http://127.0.0.1:8765/v1 tasks=48 cap=$15.0 ledger=/tmp/jb/ledger.jsonl +10/48 completed +... +[jevbench] done: 48/48 attempted, 0 failed; charged=$0.0000 +[jevbench] public export -> /tmp/jb/public.json +{'n_attempted': 48, 'accuracy': 0.2708..., 'schema_validity': 1.0, 'brier_mean': 0.7156..., + 'price_per_1000_decisions_usd': 0.0, 'cost_basis': ['local_stub_no_tariff'], + 'probability_sources': ['verbalized'], 'ledger_charged_usd': 0.0, 'complete': True} +``` + +The uniform baseline gives the same accuracy and Brier score as the Python +example in ADAPTERS.md. Delete `/tmp/jb` before running again. diff --git a/jevbench/cli.py b/jevbench/cli.py index da38bea..6206fed 100644 --- a/jevbench/cli.py +++ b/jevbench/cli.py @@ -1,15 +1,17 @@ """JevBench CLI. - python -m jevbench.cli run --tasks \ - --adapter typesafe|systemone_list|gradio_space|local_openjev|openai_compat \ - --results [--endpoint URL] [--model NAME] [--key-env ENV] \ - [--cap-usd N] [--ledger PATH] [--raw-dir DIR] [--reserve-usd N] \ - [--price-in-per-m X] [--price-out-per-m X] [--limit N] - python -m jevbench.cli summarize --tasks --results \ + jevbench run --tasks --adapter --results \ + [--endpoint URL] [--model NAME] [--key-env ENV] [--cap-usd N] [--ledger PATH] \ + [--raw-dir DIR] [--reserve-usd N] [--price-in-per-m X] [--price-out-per-m X] \ + [--limit N] [--delay-s S] [--request-options JSON] [--revision REV] \ + [--run-label LABEL] [--cost-basis TEXT] [--manifest PATH] + jevbench summarize --tasks --results \ [--ledger PATH] [--public-export PATH] [--include-excluded] +`python -m jevbench.cli ...` is equivalent. See docs/CLI.md for every flag. + Never prints or logs secrets. Raw responses are written into --raw-dir, -which should live OUTSIDE the repo (defaults to ../private/raw_responses). +which must live OUTSIDE the repo (defaults to ../private/raw_responses). """ from __future__ import annotations @@ -151,38 +153,110 @@ def cmd_summarize(args) -> int: return 0 +_NO_ENDPOINT_ADAPTERS = ("typesafe, djev, needle_local, semif_direct, so1_decider, " + "sg_system_one, classifier_dev") + +_MAIN_EPILOG = """\ +examples: + jevbench run --tasks datasets/public/easy.jsonl --adapter openai_compat \\ + --endpoint http://127.0.0.1:8000/v1 --model my-model --key-env MY_KEY \\ + --results /tmp/jb/results.jsonl --ledger /tmp/jb/ledger.jsonl --raw-dir /tmp/jb/raw + jevbench summarize --tasks datasets/public/easy.jsonl \\ + --results /tmp/jb/results.jsonl --ledger /tmp/jb/ledger.jsonl + +Results, ledger and raw responses must be written outside the repository. +Full reference: docs/CLI.md +""" + +_RUN_EPILOG = f"""\ +--endpoint is required except for: {_NO_ENDPOINT_ADAPTERS}. +--results and --raw-dir must be outside the repository; an existing results +file or raw response file is never overwritten, so use a fresh directory. +Exit status: 0 all tasks attempted, 3 run stopped early (budget, auth/rate +limit or 3 consecutive infrastructure errors), 2 usage error. + +example: + jevbench run --tasks datasets/public/easy.jsonl,datasets/public/hard.jsonl \\ + --adapter openai_compat --endpoint http://127.0.0.1:8000/v1 --model my-model \\ + --key-env MY_KEY --results /tmp/jb/results.jsonl --ledger /tmp/jb/ledger.jsonl \\ + --raw-dir /tmp/jb/raw --manifest /tmp/jb/manifest.json --limit 10 +""" + +_SUM_EPILOG = """\ +Prints the summary JSON to stdout. Items with an exclude_reason or no +expected answer are never scored (they still count as attempted). + +example: + jevbench summarize --tasks datasets/public/easy.jsonl \\ + --results /tmp/jb/results.jsonl --ledger /tmp/jb/ledger.jsonl \\ + --public-export /tmp/jb/public.json +""" + + def main(argv=None) -> int: - ap = argparse.ArgumentParser(prog="jevbench") - sub = ap.add_subparsers(dest="cmd", required=True) + fmt = argparse.RawDescriptionHelpFormatter + ap = argparse.ArgumentParser( + prog="jevbench", formatter_class=fmt, epilog=_MAIN_EPILOG, + description="JevBench: run a decision model adapter over JevBench task " + "files and summarize the per-item results.") + sub = ap.add_subparsers(dest="cmd", required=True, metavar="{run,summarize}") def ds_default(name: str) -> str: return os.path.join(REPO_ROOT, "..", "private", name) - p_run = sub.add_parser("run", help="run the benchmark over tasks") - p_run.add_argument("--tasks", required=True) + p_run = sub.add_parser( + "run", help="run one adapter over task files and write per-item results", + description="Run one adapter serially (no retries) over the given tasks. " + "Writes one JSON line per attempted task to --results, raw " + "request/response evidence to --raw-dir, and reservations " + "to the shared budget --ledger.", + formatter_class=fmt, epilog=_RUN_EPILOG) + p_run.add_argument("--tasks", required=True, + help="task JSONL file(s), comma-separated, " + "e.g. datasets/public/easy.jsonl") p_run.add_argument("--adapter", required=True, choices=["typesafe", "systemone_list", "gradio_space", "local_openjev", "openai_compat", "needle_local", "semif_direct", "so1_decider", "remote_inproc", "sg_system_one", "djev", "laya_local", "gliner2_local", "verdict_local", "paw_local", "classifier_dev", "certo_local", - "qwen_flash_linear"]) - p_run.add_argument("--endpoint", default=None) - p_run.add_argument("--model", default=None) + "qwen_flash_linear"], + help="adapter to run: %(choices)s") + p_run.add_argument("--endpoint", default=None, + help="base URL (or path, for local adapters) of the system " + "under test; required for most adapters (see below)") + p_run.add_argument("--model", default=None, + help="model name to request (openai_compat sends \"\" if unset)") p_run.add_argument("--key-env", dest="key_env", - default="TYPESAFE_API_KEY") - p_run.add_argument("--results", required=True) - p_run.add_argument("--ledger", default=ds_default("ledger.jsonl")) + default="TYPESAFE_API_KEY", + help="NAME of the environment variable holding the API key; " + "the key itself is never printed (default: %(default)s)") + p_run.add_argument("--results", required=True, + help="per-item results JSONL to create; must not exist " + "and must be outside the repository") + p_run.add_argument("--ledger", default=ds_default("ledger.jsonl"), + help="shared budget ledger JSONL, appended to " + "(default: /../private/ledger.jsonl)") p_run.add_argument("--raw-dir", dest="raw_dir", - default=ds_default("raw_responses")) - p_run.add_argument("--cap-usd", dest="cap_usd", type=float, default=15.0) + default=ds_default("raw_responses"), + help="directory for raw request/response evidence, outside " + "the repository (default: /../private/raw_responses)") + p_run.add_argument("--cap-usd", dest="cap_usd", type=float, default=15.0, + help="budget cap in USD; if the ledger already records a lower " + "cap, the lower one applies (default: %(default)s)") p_run.add_argument("--reserve-usd", dest="reserve_usd", type=float, - default=DEFAULT_RESERVE_USD) + default=DEFAULT_RESERVE_USD, + help="minimum USD reserved per request before it is sent " + "(default: %(default)s)") p_run.add_argument("--price-in-per-m", dest="price_in_per_m", type=float, - default=None) + default=None, + help="USD per million input tokens; with --price-out-per-m " + "this turns reported usage into cost_usd") p_run.add_argument("--price-out-per-m", dest="price_out_per_m", type=float, - default=None) - p_run.add_argument("--limit", type=int, default=None) + default=None, + help="USD per million output tokens") + p_run.add_argument("--limit", type=int, default=None, + help="only run the first N tasks (smoke tests)") p_run.add_argument("--delay-s", dest="delay_s", type=float, default=0.0, help="pause between requests; be a polite guest on " "someone else's free public demo") @@ -191,22 +265,35 @@ def ds_default(name: str) -> str: "'{\"reasoning_effort\": \"low\"}'") p_run.add_argument("--revision", default=None, help="pinned revision of a local checkpoint") - p_run.add_argument("--run-label", dest="run_label", default=None) + p_run.add_argument("--run-label", dest="run_label", default=None, + help="label stored in the manifest (default: --model, " + "else --adapter)") p_run.add_argument("--cost-basis", dest="cost_basis", default=None, help="why this route's per-decision cost is what it is, " "e.g. no_billable_account_public_demo") p_run.add_argument("--manifest", default=None, - help="write the run's exact settings here") + help="write the run's exact settings (JSON) to this path") p_run.set_defaults(fn=cmd_run) - p_sum = sub.add_parser("summarize", help="aggregate results") - p_sum.add_argument("--tasks", required=True) - p_sum.add_argument("--results", required=True) - p_sum.add_argument("--ledger", default=None) - p_sum.add_argument("--public-export", dest="public_export", default=None) + p_sum = sub.add_parser( + "summarize", help="aggregate a results file into summary metrics", + description="Aggregate per-item results from `jevbench run` into " + "overall, per-family and per-split metrics.", + formatter_class=fmt, epilog=_SUM_EPILOG) + p_sum.add_argument("--tasks", required=True, + help="the same task JSONL file(s) the run used, comma-separated") + p_sum.add_argument("--results", required=True, + help="per-item results JSONL written by `jevbench run`") + p_sum.add_argument("--ledger", default=None, + help="ledger JSONL; if given and present, its total is " + "reported as ledger_charged_usd") + p_sum.add_argument("--public-export", dest="public_export", default=None, + help="also write an allowlisted aggregate JSON (no item " + "text or labels, no ledger spend) to this path") p_sum.add_argument("--include-excluded", dest="include_excluded", action="store_true", - help="include non-headline (excluded/unmeasured) items") + help="include non-headline (excluded/unmeasured) items; currently " + "has no effect on the output (see docs/CLI.md)") p_sum.set_defaults(fn=cmd_summarize) args = ap.parse_args(argv) diff --git a/tests/test_cli_help.py b/tests/test_cli_help.py new file mode 100644 index 0000000..9e9097e --- /dev/null +++ b/tests/test_cli_help.py @@ -0,0 +1,74 @@ +"""`--help` works for the main command and every subcommand, and documents every flag.""" + +import argparse +import os +import subprocess +import sys + +import pytest + +from jevbench import cli + +SUBCOMMANDS = { + "run": ["--tasks", "--adapter", "--endpoint", "--model", "--key-env", "--results", + "--ledger", "--raw-dir", "--cap-usd", "--reserve-usd", "--price-in-per-m", + "--price-out-per-m", "--limit", "--delay-s", "--request-options", "--revision", + "--run-label", "--cost-basis", "--manifest"], + "summarize": ["--tasks", "--results", "--ledger", "--public-export", "--include-excluded"], +} + + +def _help(capsys, argv): + with pytest.raises(SystemExit) as exc: + cli.main(argv) + assert exc.value.code == 0 + return capsys.readouterr().out + + +def test_main_help(capsys): + out = _help(capsys, ["--help"]) + assert out.startswith("usage: jevbench") + for name in SUBCOMMANDS: + assert name in out + assert "examples:" in out + + +@pytest.mark.parametrize("name", sorted(SUBCOMMANDS)) +def test_subcommand_help(capsys, name): + out = _help(capsys, [name, "--help"]) + assert out.startswith(f"usage: jevbench {name}") + assert "example:" in out + for flag in SUBCOMMANDS[name]: + assert flag in out, flag + + +def test_every_option_has_help_text(monkeypatch): + parser_actions = [] + + def capture(self, args=None, namespace=None): + for action in self._actions: + if isinstance(action, argparse._SubParsersAction): + for sub in action.choices.values(): + parser_actions.extend((sub.prog, a) for a in sub._actions) + raise SystemExit(0) + + monkeypatch.setattr(argparse.ArgumentParser, "parse_args", capture) + with pytest.raises(SystemExit): + cli.main([]) + assert parser_actions + missing = [f"{prog} {a.option_strings}" for prog, a in parser_actions if not a.help] + assert not missing, missing + + +def test_module_help_subprocess(): + proc = subprocess.run([sys.executable, "-m", "jevbench.cli", "--help"], + capture_output=True, text=True, timeout=60, + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + assert proc.returncode == 0, proc.stderr + assert "usage: jevbench" in proc.stdout + + +def test_missing_subcommand_is_usage_error(capsys): + with pytest.raises(SystemExit) as exc: + cli.main([]) + assert exc.value.code == 2