From f11f11d8a842eb99c4898863eaef19c2968606cf Mon Sep 17 00:00:00 2001 From: barckcode Date: Tue, 8 Sep 2026 19:29:06 +0100 Subject: [PATCH] Distinguish a truncated reasoning block from a wrong answer A reasoning model served behind a reasoning parser puts its thinking in a separate field and leaves `content` empty until that block closes. When the token budget runs out first, the endpoint returns an empty reply with `finish_reason: "length"`. The harness scored that as "no files parsed from the reply", which is indistinguishable from a model that wrote prose instead of files, so a budget that was too small was published as a capability gap. `complete` now reads the reasoning field under both names servers use for it (`reasoning` on vLLM, `reasoning_content` elsewhere) and reports its length. `answer` flags the case where no content arrived and the budget was the reason, with a message that names how much reasoning was spent getting nowhere. The failure is still a failure: the model did not deliver inside the budget it was given, and it stays in the pass rate. What changes is that it is now its own category. `analyse.py` and `compare.py` classify it as a harness failure rather than infrastructure the model got wrong, `analyse.py` prints how often it happened and how much reasoning preceded it, and the published table gains a `Truncated, no answer` column. A non-zero count is a signal to raise --max-tokens and run again, not a number to interpret. Found while standing up the Ornith-1.5 baseline: at 120 tokens the model returned an empty reply having spent all 120 inside ``. Co-Authored-By: Claude Opus 5 --- README.md | 13 +++ analyse.py | 16 +++- compare.py | 22 ++++- evaluate.py | 26 +++++- tests/test_evaluate.py | 178 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 246 insertions(+), 9 deletions(-) create mode 100644 tests/test_evaluate.py diff --git a/README.md b/README.md index 834d03c..680071f 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,19 @@ A second, opt-in **network tier** adds validators that download or execute. Its Terraform half is working and verified; its `docker build` half is specified and deliberately refuses to run until an isolating runner exists. See below. +## Reasoning models need their own token budget + +`--max-tokens` defaults to 16384. A reasoning model served behind a reasoning parser +returns an **empty** `content` while it is still inside its thinking block, so a budget +that is comfortable for a non-reasoning model shows up as a reply with nothing in it, +not as a truncated one. Scored naively that is a wrong answer, and the run reports a +capability gap where the cause is the budget. + +The harness records this as its own category: `truncated_before_answer` per row, +a dedicated line in `analyse.py`, and a `Truncated, no answer` column in the published +table. It still counts as a failure. But if that count is not zero, raise +`--max-tokens` and run again before reading anything into the pass rate. + ## Reference results, and why one run is not a measurement Full tables, per class and per task type, in **[RESULTS.md](RESULTS.md)**. One pass diff --git a/analyse.py b/analyse.py index f15659c..254844e 100644 --- a/analyse.py +++ b/analyse.py @@ -14,7 +14,11 @@ import sys from collections import Counter, defaultdict -HARNESS = ("no files parsed", "model error") +# Failures the harness or the run configuration owns, not the model's code. +# "truncated before any answer" is a reply that spent its whole budget inside a +# reasoning block: a real failure, but one fixed by raising --max-tokens, so it +# must not be read as infrastructure the model got wrong. +HARNESS = ("no files parsed", "model error", "truncated before any answer") def rate(rows: list[dict]) -> tuple[int, int, float]: @@ -71,6 +75,16 @@ def main() -> int: if truncated: print(f"\nreplies cut off by max_tokens: {len(truncated)} " f"({100*len(truncated)/len(rows):.1f}%)") + # Reported separately from the line above: these are the ones that produced no + # answer at all, which is the shape a reasoning model takes when the budget is + # too small for its thinking block. + starved = [r for r in rows if r.get("truncated_before_answer")] + if starved: + spent = sum(r.get("reasoning_chars", 0) for r in starved) / len(starved) + print(f" of those, produced NO answer: {len(starved)} " + f"({100*len(starved)/len(rows):.1f}%), " + f"averaging {spent:,.0f} chars of reasoning first") + print(" -> raise --max-tokens and re-run before reading this as capability") print("\n--- three failures the model owns ---") shown = 0 diff --git a/compare.py b/compare.py index d3e7a5f..08b36f4 100644 --- a/compare.py +++ b/compare.py @@ -22,7 +22,11 @@ CLASSES = ["dockerfile", "helm_chart", "workflow", "manifest_set", "compose", "terraform_module", "ansible_role"] TYPES = ["repair", "completion", "generation"] -HARNESS = ("no files parsed", "model error") +# Failures the harness or the run configuration owns, not the model's code. +# "truncated before any answer" is a reply that spent its whole budget inside a +# reasoning block: a real failure, but one fixed by raising --max-tokens, so it +# must not be read as infrastructure the model got wrong. +HARNESS = ("no files parsed", "model error", "truncated before any answer") def rate(rows: list[dict]) -> tuple[int, int, float]: @@ -83,15 +87,17 @@ def main() -> int: "real deployments. One pass per model over the same 674 tasks, temperature " "0, no retries, no tool access, no iteration, offline tier.", "", "Generated by `compare.py` from the run files; do not edit by hand.", "", - "| Model | Pass rate | Unreadable replies | Output tokens | Run |", - "|---|---|---|---|---|"] + "| Model | Pass rate | Unreadable replies | Truncated, no answer | " + "Output tokens | Run |", + "|---|---|---|---|---|---|"] for model, (path, rows) in ordered.items(): passed, total, r = rate(rows) unreadable = sum(1 for x in rows if not x["passed"] and any(x["reason"].startswith(h) for h in HARNESS)) + starved = sum(1 for x in rows if x.get("truncated_before_answer")) tokens = sum(x.get("usage", {}).get("completion_tokens", 0) for x in rows) out.append(f"| **{model}** | **{passed}/{total} = {r:.1f}%** | " - f"{unreadable} | {tokens:,} | {when(path)} |") + f"{unreadable} | {starved} | {tokens:,} | {when(path)} |") out += ["", "### By task type", ""] + table(models, "task_type", TYPES) out += ["", "### By class", ""] + table(models, "unit_type", CLASSES) @@ -112,6 +118,14 @@ def main() -> int: "rate is that rather than the model's infrastructure code. `analyse.py` breaks " "it down per run.", "", + "**A truncated reply is not a wrong answer.** The `Truncated, no answer` " + "column counts replies that hit `max_tokens` while still inside a " + "reasoning block and so produced no content at all. They are scored as " + "failures, because the model did not deliver inside the budget it was " + "given, but a non-zero count means the run measured the budget rather " + "than the model. Reasoning models need a larger `--max-tokens` than " + "non-reasoning ones; set it per model and re-run before comparing.", + "", "**These are not equal-footing comparisons of raw capability.** Every model " "was prompted identically and given no retries, no tool access and no " "iteration, which is not how any of them is used in practice. What the numbers " diff --git a/evaluate.py b/evaluate.py index 71c12c0..a2682f1 100644 --- a/evaluate.py +++ b/evaluate.py @@ -220,9 +220,16 @@ def complete(messages: list[dict], model: str, base: str, key: str, with urllib.request.urlopen(request, timeout=timeout) as response: body = json.load(response) choice = body["choices"][0] - text = choice["message"].get("content") or "" + message = choice["message"] + text = message.get("content") or "" + # A server with a reasoning parser puts the thinking block in its own + # field and leaves `content` empty until the block closes. vLLM calls it + # `reasoning`, others `reasoning_content`. Reading neither makes a reply + # that ran out of budget mid-thought look like a model with nothing to say. + reasoning = message.get("reasoning") or message.get("reasoning_content") or "" return text, {"finish_reason": choice.get("finish_reason"), - "usage": body.get("usage", {})} + "usage": body.get("usage", {}), + "reasoning_chars": len(reasoning)} except urllib.error.HTTPError as error: detail = error.read().decode(errors="replace")[:200] last = f"HTTP {error.code}: {detail}" @@ -251,13 +258,24 @@ def answer(task: Task, options) -> dict: "elapsed": round(time.time() - started, 1)} return record + reasoning_chars = meta.get("reasoning_chars", 0) + # An empty reply that hit the token ceiling never got to the answer at all. It is + # still a failure - the model did not deliver inside the budget it was given - but + # it is a different failure from writing something that does not parse, and + # conflating them reports a capability gap where the cause is the budget. + truncated_before_answer = (not text.strip() + and meta["finish_reason"] == "length") record |= {"finish_reason": meta["finish_reason"], "usage": meta["usage"], - "reply_chars": len(text)} + "reply_chars": len(text), "reasoning_chars": reasoning_chars, + "truncated_before_answer": truncated_before_answer} parsed = parse_files(text, task) record["returned_paths"] = sorted(parsed) if not parsed: - record |= {"passed": False, "reason": "no files parsed from the reply", + reason = ("no files parsed from the reply" if not truncated_before_answer else + f"truncated before any answer: hit max_tokens after " + f"{reasoning_chars} chars of reasoning and no content") + record |= {"passed": False, "reason": reason, "raw": text[:2000], "elapsed": round(time.time() - started, 1)} return record diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py new file mode 100644 index 0000000..b95b14f --- /dev/null +++ b/tests/test_evaluate.py @@ -0,0 +1,178 @@ +"""Tests for the model-facing half of the harness. + +The cases here all come from one real failure: a reasoning model served behind vLLM +spends its whole token budget inside a thinking block, and the endpoint returns an +empty `content` with `finish_reason: "length"`. Scored naively that is a wrong answer, +so the run reports a capability gap where the truth is a budget that was too small. +""" + +import json +import types + +import pytest + +import evaluate +from devopsbench.tasks import Task + + +def make_task(**overrides) -> Task: + base = dict( + task_id="drop-instruction-000001", + unit_type="dockerfile", + mutation="drop-instruction", + kind="repair", + task_type="repair", + instruction="fix it", + broken_files={"Dockerfile": "FROM alpine:3.20\n"}, + reference_files={"Dockerfile": "FROM alpine:3.20\nUSER nobody\n"}, + repo_path="acme/app", + commit_id="abc", + original_verdict="hadolint: ok", + mutant_verdict="hadolint: failed (1)", + ) + return Task(**{**base, **overrides}) + + +def make_options(**overrides): + base = dict(model="m", base="http://x/v1", key="k", max_tokens=64, + request_timeout=1, score_timeout=1) + return types.SimpleNamespace(**{**base, **overrides}) + + +def fake_endpoint(monkeypatch, payload: dict): + """Stub urlopen so `complete` sees exactly the body a real server would send.""" + + class Response: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return json.dumps(payload).encode() + + monkeypatch.setattr(evaluate.urllib.request, "urlopen", + lambda *a, **k: Response()) + + +def chat_body(content, *, finish_reason="stop", **message_extra): + message = {"role": "assistant", "content": content, **message_extra} + return {"choices": [{"message": message, "finish_reason": finish_reason}], + "usage": {"completion_tokens": 64}} + + +# --- the reasoning field ---------------------------------------------------------- + +@pytest.mark.parametrize("field", ["reasoning", "reasoning_content"]) +def test_complete_captures_reasoning_under_either_field_name(monkeypatch, field): + """vLLM calls it `reasoning`; other servers call it `reasoning_content`. + + Reading only one of the two silently loses the evidence that the model was still + thinking when the budget ran out. + """ + fake_endpoint(monkeypatch, chat_body("", finish_reason="length", + **{field: "step one, step two"})) + + text, meta = evaluate.complete([], "m", "http://x/v1", "k", 64, 1) + + assert text == "" + assert meta["reasoning_chars"] == len("step one, step two") + + +def test_complete_reports_no_reasoning_when_the_model_does_not_emit_any(monkeypatch): + fake_endpoint(monkeypatch, chat_body("### path: Dockerfile\n```\nFROM x\n```")) + + _, meta = evaluate.complete([], "m", "http://x/v1", "k", 64, 1) + + assert meta["reasoning_chars"] == 0 + + +# --- truncation before the answer ------------------------------------------------- + +def test_truncated_inside_reasoning_is_reported_as_truncation_not_as_a_bad_answer( + monkeypatch): + """The whole point: an empty reply cut off by the budget is its own category.""" + monkeypatch.setattr(evaluate, "complete", + lambda *a, **k: ("", {"finish_reason": "length", + "usage": {}, "reasoning_chars": 4096})) + + record = evaluate.answer(make_task(), make_options()) + + assert record["passed"] is False, "a model that never answered did not pass" + assert record["truncated_before_answer"] is True + assert "truncated" in record["reason"] + assert "4096" in record["reason"], "the reason names the reasoning it spent" + + +def test_unparseable_but_complete_reply_keeps_the_original_reason(monkeypatch): + """Prose instead of files is a real failure and must not be excused as truncation.""" + monkeypatch.setattr(evaluate, "complete", + lambda *a, **k: ("I would suggest adding a USER line.", + {"finish_reason": "stop", "usage": {}, + "reasoning_chars": 0})) + + record = evaluate.answer(make_task(), make_options()) + + assert record["passed"] is False + assert record["truncated_before_answer"] is False + assert record["reason"] == "no files parsed from the reply" + + +def test_truncated_after_some_output_is_not_counted_as_truncated_before_answer( + monkeypatch): + """Content that exists but does not parse is a parse failure, budget or not.""" + monkeypatch.setattr(evaluate, "complete", + lambda *a, **k: ("### path: Dockerfile\n```\nFROM alp", + {"finish_reason": "length", "usage": {}, + "reasoning_chars": 0})) + + record = evaluate.answer(make_task(), make_options()) + + assert record["truncated_before_answer"] is False + + +def test_every_record_carries_the_flag_so_analysis_never_sees_a_missing_key(monkeypatch): + monkeypatch.setattr(evaluate, "complete", + lambda *a, **k: ("", {"finish_reason": "stop", "usage": {}, + "reasoning_chars": 0})) + + record = evaluate.answer(make_task(), make_options()) + + assert "truncated_before_answer" in record + assert "reasoning_chars" in record + + +# --- regression: inline thinking blocks ------------------------------------------- + +def test_inline_think_block_is_still_stripped_before_parsing(): + """Endpoints without a reasoning parser leave in `content`.""" + reply = ("maybe I should write foo.yaml\n" + "### path: Dockerfile\n```\nFROM alpine:3.20\nUSER nobody\n```\n") + + files = evaluate.parse_files(reply, make_task()) + + assert set(files) == {"Dockerfile"} + assert "maybe I should" not in files["Dockerfile"] + + +# --- how the reporting scripts classify a truncated reply ------------------------- + +def test_analyse_counts_truncation_as_a_harness_failure_not_a_model_failure(): + """Otherwise a budget that was too small is printed as code the model got wrong.""" + import analyse + + record = {"reason": "truncated before any answer: hit max_tokens after " + "4096 chars of reasoning and no content", "passed": False} + + assert analyse.bucket(record) == "harness" + + +def test_compare_does_not_count_truncation_as_a_readable_answer(): + """`compare.py` builds the published table; its notion of unreadable must match.""" + import compare + + reason = ("truncated before any answer: hit max_tokens after 4096 chars " + "of reasoning and no content") + + assert any(reason.startswith(h) for h in compare.HARNESS)