From d1817a55df727f273c428ca02ad707d1591edac5 Mon Sep 17 00:00:00 2001 From: Yunyue Li Date: Wed, 5 Aug 2026 10:31:39 +0800 Subject: [PATCH] fix(evals): three of my own evals were measuring themselves, not the graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First run against a real provider (DeepSeek). All three model-backed evals had a scoring bug of the same shape as the Pass-7 tautology this series set out to fix — the eval graded its own output, or graded a window that excluded the evidence it was asking about. 1. entailment reported 35.8%. It sent the judge `chunk_text[:1500]`, but Alice's chunks run to ~4,800 characters, so the evidence span usually fell past the cut and the judge correctly answered "that quote is not in the passage you gave me". The same claim was judged both ways in a single run depending on where its span landed. Centre the window on the span: 0/120 samples now fall outside it, and the real rate is 85.0%. It also vindicates the stratification: both multi_step claims fail (0/2), one_step strata sit at 0.5–0.75, explicit at 0.8+. That is exactly why Pass-7 verifies every multi_step claim instead of sampling it. 2. contamination reported the graph at 12/12. `ground_truth` was a restatement of `graph_answer`, so the judge was asked whether an answer matched itself. Grade both arms against the source passage instead: closed-book 0/12, graph 5/12. 3. perturbation scored plainly correct answers as reversions — "Pellwyn is a curious young girl who follows a White Rabbit down a rabbit-hole" was marked as having reverted to the published text. Cause: an instruction fed into the answer-scoring judge's ground-truth slot. Entailment and reversion now have their own prompts rather than borrowing one built for a different question. Rescored: memory follows the altered source 2/6, reading it follows 4/6. The raw answers say more than the ratio. Asked whether the text states X, the memory arm answers yes and quotes the published wording — wording absent from the excerpt it was given. And both failures of the reading arm quote the altered clause and then contradict it: it cites "the Queen tried the little golden key" and answers "Alice has the key"; it cites "she had *not* peeped into the book" and answers "yes, she did". Putting the source in the context window does not make the answer faithful to it. `LOREGRAPH_COVE_SUPPORTED_FLOOR` 0.85 -> 0.80. The 0.85 was a guess and the measurement landed exactly on it, so it would have aborted every run. 0.80 is alice's measured rate less a 5-point margin; the comment records the date, the judge model, and that one book is not a distribution. `model_arm.SPEND` accumulates across every arm and judge, so a run's cost comes out of the harness. The whole session, including the three wasted runs: 0.65 CNY. --- src/loregraph/cli/main.py | 2 +- src/loregraph/config.py | 12 ++- src/loregraph/evals/contamination.py | 42 ++++++--- src/loregraph/evals/entailment.py | 34 ++++--- src/loregraph/evals/model_arm.py | 110 +++++++++++++++++++++++ src/loregraph/evals/perturbation.py | 127 +++++++++++++++++++++++++++ 6 files changed, 296 insertions(+), 31 deletions(-) diff --git a/src/loregraph/cli/main.py b/src/loregraph/cli/main.py index 2945356..107bc32 100644 --- a/src/loregraph/cli/main.py +++ b/src/loregraph/cli/main.py @@ -243,7 +243,7 @@ def eval_( # These fall back to a dry preview when no provider is configured, # printing exactly what would be sent rather than failing or, worse, # silently scoring a subset. - "perturbation": perturbation.dry_run, + "perturbation": lambda b: asyncio.run(perturbation.run(b, per_kind=2)), "contamination": lambda b: asyncio.run(contamination.run(b, limit=probes)), "entailment": lambda b: asyncio.run(entailment.run(b, budget=budget)), } diff --git a/src/loregraph/config.py b/src/loregraph/config.py index 3287f45..d8e1b3c 100644 --- a/src/loregraph/config.py +++ b/src/loregraph/config.py @@ -98,10 +98,14 @@ class Settings(BaseSettings): # Floor on the fraction of sampled claims the judge finds *supported* by # their evidence span. This is the gate that actually measures extraction # quality — the literal-match rate is an upstream invariant and cannot - # fail. PROVISIONAL: no calibrated distribution exists yet; run - # `loregraph eval entailment` to measure a book before trusting a number. - # 0 disables the gate (records the rate without enforcing it). - cove_supported_floor: float = Field(0.85, alias="LOREGRAPH_COVE_SUPPORTED_FLOOR") + # fail. 0 disables the gate (records the rate without enforcing it). + # + # Measured, not guessed: 0.80 is alice's rate (102/120 sampled claims, + # deepseek-chat as judge, 2026-08-03) less a 5-point margin. The earlier + # provisional 0.85 sat exactly at the measured rate and would have aborted + # every run. ONE book is not a distribution — re-measure with + # `loregraph eval entailment` before relying on this on a new corpus. + cove_supported_floor: float = Field(0.80, alias="LOREGRAPH_COVE_SUPPORTED_FLOOR") # ── Provider lookup helpers ───────────────────────────────────── def resolved_api_key(self, provider: str) -> str | None: diff --git a/src/loregraph/evals/contamination.py b/src/loregraph/evals/contamination.py index d76c3eb..e3a60f3 100644 --- a/src/loregraph/evals/contamination.py +++ b/src/loregraph/evals/contamination.py @@ -19,7 +19,7 @@ from __future__ import annotations import asyncio -from collections.abc import Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass from typing import TypeVar @@ -39,6 +39,15 @@ class Probe: question: str ground_truth: str + """The **source passage**, not a restatement of the graph's claim. + + The first version of this eval set ground_truth to a paraphrase of + graph_answer, so the judge was asked whether an answer matched itself and + the graph scored 12/12 by construction — the same tautology this project's + literal-match gate had. Grade both arms against the text, and the graph can + lose. + """ + graph_answer: str """What the graph alone says — the pipeline's arm, assembled without a model.""" source: str @@ -56,6 +65,8 @@ def build(book: BookUnderTest, *, limit: int = 24) -> list[Probe]: """ probes: list[Probe] = [] + passage = book.chunk_text + edges = _confident(book.edges) # Spread across chapters so the battery is not all opening-scene trivia, # which is the part of a famous book a model remembers best. @@ -66,10 +77,7 @@ def build(book: BookUnderTest, *, limit: int = 24) -> list[Probe]: probes.append( Probe( question=f"In this work, what is the relationship between {src} and {dst}?", - ground_truth=( - f"{src} {edge.predicate or edge.relation} {dst}. " - f"The text reads: {edge.evidence_span.strip()!r}" - ), + ground_truth=passage.get(edge.atom_id, edge.evidence_span)[:1800], graph_answer=( f"{src} —{edge.predicate or edge.relation}→ {dst} " f"({edge.evidence_span.strip()!r}, {edge.atom_id})" @@ -89,7 +97,7 @@ def build(book: BookUnderTest, *, limit: int = 24) -> list[Probe]: f"In this work, what does the text establish about {who} " f"regarding {fact.dimension}?" ), - ground_truth=(f"{fact.statement} The text reads: {fact.evidence_span.strip()!r}"), + ground_truth=passage.get(fact.atom_id, fact.evidence_span)[:1800], graph_answer=f"{fact.statement} ({fact.evidence_span.strip()!r}, {fact.atom_id})", source=fact.atom_id, ) @@ -137,14 +145,22 @@ async def run(book: BookUnderTest, *, limit: int = 24) -> EvalResult: question_with_work = [f"Work: {book.title} by {book.author}.\n\n{p.question}" for p in probes] closed_answers = await closed.answer_all(question_with_work) + def graded(answer_of: Callable[[int, Probe], str]) -> list[tuple[str, str, str]]: + return [ + ( + p.question, + answer_of(i, p), + "The source passage below is the only authority. An answer is " + "correct if the passage states or directly implies it, and " + "incorrect if the passage contradicts it or is silent on it.\n\n" + f"Passage:\n{p.ground_truth}", + ) + for i, p in enumerate(probes) + ] + closed_scores, graph_scores = await asyncio.gather( - judge.score_all( - [ - (p.question, a.text, p.ground_truth) - for p, a in zip(probes, closed_answers, strict=True) - ] - ), - judge.score_all([(p.question, p.graph_answer, p.ground_truth) for p in probes]), + judge.score_all(graded(lambda i, p: closed_answers[i].text)), + judge.score_all(graded(lambda i, p: p.graph_answer)), ) closed_right = sum(1 for s in closed_scores if s.correct) diff --git a/src/loregraph/evals/entailment.py b/src/loregraph/evals/entailment.py index aa28732..d3dd43f 100644 --- a/src/loregraph/evals/entailment.py +++ b/src/loregraph/evals/entailment.py @@ -73,6 +73,25 @@ def sample(book: BookUnderTest, *, budget: int = 150, seed: int = 7) -> list[Cla ) +# How much of the chunk the judge sees. A blind head-truncation is a trap: the +# evidence span often sits past it, the judge correctly reports "that quote is +# not in the passage you gave me", and the eval scores the truncation instead +# of the extraction. Centre the window on the span so it is always included. +_PASSAGE_CHARS = 2400 + + +def _passage_around(claim: Claim) -> str: + text = claim.chunk_text + if len(text) <= _PASSAGE_CHARS: + return text + at = text.find(claim.evidence_span) + if at < 0: # should not happen — spans are literal by construction + return text[:_PASSAGE_CHARS] + half = (_PASSAGE_CHARS - len(claim.evidence_span)) // 2 + lo = max(0, at - half) + return text[lo : lo + _PASSAGE_CHARS] + + def preview(book: BookUnderTest, *, budget: int = 150) -> EvalResult: picked = sample(book, budget=budget) strata: dict[str, int] = {} @@ -116,19 +135,8 @@ async def run(book: BookUnderTest, *, budget: int = 150) -> EvalResult: ) judge = Judge() - verdicts = await judge.score_all( - [ - ( - f"Does this passage support the claim?\n\nPassage: {c.chunk_text[:1500]}", - f"Claim: {c.statement}\nCited span: {c.evidence_span}", - ( - "The claim is supported only if the cited span, read in the " - "passage, states or directly implies it. A span that is real " - "but about something else is NOT support." - ), - ) - for c in picked - ] + verdicts = await judge.entails_all( + [(c.statement, c.evidence_span, _passage_around(c)) for c in picked] ) by_stratum: dict[str, list[int]] = {} diff --git a/src/loregraph/evals/model_arm.py b/src/loregraph/evals/model_arm.py index c599a7b..2d9e0a7 100644 --- a/src/loregraph/evals/model_arm.py +++ b/src/loregraph/evals/model_arm.py @@ -25,6 +25,31 @@ _CONCURRENCY = 8 +# Every Arm and Judge merges into this, so a run's cost is one number at the +# end rather than something you reconstruct from provider dashboards later. +SPEND = LLMUsage() + + +def spend_report() -> dict[str, float | int]: + """Tokens used so far this process, and what they cost. + + Prices come from settings so they track the configured provider; the + defaults are DeepSeek's. `usd` is an estimate — it ignores the provider's + cache-hit discount, so it reads high rather than low. + """ + settings = get_settings() + usd = ( + SPEND.input_tokens / 1_000_000 * settings.price_per_mtok_input + + SPEND.output_tokens / 1_000_000 * settings.price_per_mtok_output + ) + return { + "requests": SPEND.requests, + "input_tokens": SPEND.input_tokens, + "output_tokens": SPEND.output_tokens, + "usd": round(usd, 4), + "cny": round(usd * 7.15, 3), + } + def available() -> tuple[bool, str]: """(usable, why-not). Never raises — callers report, they do not crash.""" @@ -70,6 +95,7 @@ async def answer(self, question: str, context: str = "") -> Answer: user = f"{context}\n\n{question}".strip() if context else question message = await self.llm.complete(system=self.system, user=user) self.usage.merge(message) + SPEND.merge(message) return Answer(question=question, text=self.llm.extract_text(message).strip(), arm=self.name) async def answer_all(self, questions: list[str], context: str = "") -> list[Answer]: @@ -110,6 +136,32 @@ async def one(q: str) -> Answer: ) +ENTAILMENT_SYSTEM = ( + "You decide whether a quoted span of a passage supports a claim extracted " + "from it. Return JSON only: " + '{"correct": bool, "follows_source": true, "confidence": 0.0-1.0, "reason": "..."}. ' + "`correct` is true only if the cited span, read in its surrounding passage, " + "states the claim or directly implies it. A span that is genuinely in the " + "text but is about something else does NOT support the claim. Judge the " + "claim against the span and passage alone; do not use anything you know " + "about the work from elsewhere." +) + + +REVERSION_SYSTEM = ( + "A passage of a well-known work has been deliberately altered. You are " + "given the alteration, what the published work says, and an answer. Decide " + "whether the answer reflects the ALTERED text or reverts to the published " + "version. Return JSON only: " + '{"correct": bool, "follows_source": bool, "confidence": 0.0-1.0, "reason": "..."}. ' + "Set both true when the answer matches the altered text. Set both false " + "when it gives the published version instead, or asserts the altered " + "detail does not exist. An answer that quotes the altered clause but then " + "states the published fact has reverted: the quote is not the answer. " + "Ignore style, length and hedging; judge only the substance." +) + + class Judge: def __init__(self, llm: LLMClient | None = None) -> None: self.llm = llm or make_llm_client() @@ -119,6 +171,7 @@ async def score(self, *, question: str, answer: str, ground_truth: str) -> Judge user = f"Question: {question}\n\nGround truth: {ground_truth}\n\nAnswer to score: {answer}" message = await self.llm.complete(system=JUDGE_SYSTEM, user=user) self.usage.merge(message) + SPEND.merge(message) try: return parse_into(Judgement, self.llm.extract_text(message)) except LLMOutputError: @@ -127,6 +180,63 @@ async def score(self, *, question: str, answer: str, ground_truth: str) -> Judge correct=False, follows_source=False, confidence=0.0, reason="unparsable judgement" ) + async def entails(self, *, claim: str, span: str, passage: str) -> Judgement: + """Does `span`, read inside `passage`, support `claim`?""" + user = ( + f"Passage:\n{passage}\n\n" + f"Cited span:\n{span}\n\n" + f"Claim extracted from that span:\n{claim}" + ) + message = await self.llm.complete(system=ENTAILMENT_SYSTEM, user=user) + self.usage.merge(message) + SPEND.merge(message) + try: + return parse_into(Judgement, self.llm.extract_text(message)) + except LLMOutputError: + log.warning("entailment judge returned malformed JSON; scoring as unsupported") + return Judgement(correct=False, confidence=0.0, reason="unparsable judgement") + + async def entails_all(self, items: list[tuple[str, str, str]]) -> list[Judgement]: + gate = asyncio.Semaphore(_CONCURRENCY) + + async def one(triple: tuple[str, str, str]) -> Judgement: + claim, span, passage = triple + async with gate: + return await self.entails(claim=claim, span=span, passage=passage) + + return list(await asyncio.gather(*(one(t) for t in items))) + + async def reverted(self, *, alteration: str, published: str, answer: str) -> Judgement: + message = await self.llm.complete( + system=REVERSION_SYSTEM, + user=( + f"Alteration made to the text:\n{alteration}\n\n" + f"What the published work says:\n{published}\n\n" + f"Answer to judge:\n{answer}" + ), + ) + self.usage.merge(message) + SPEND.merge(message) + try: + return parse_into(Judgement, self.llm.extract_text(message)) + except LLMOutputError: + log.warning("reversion judge returned malformed JSON; scoring as reverted") + return Judgement( + correct=False, follows_source=False, confidence=0.0, reason="unparsable" + ) + + async def reverted_all(self, items: list[tuple[str, str, str]]) -> list[Judgement]: + gate = asyncio.Semaphore(_CONCURRENCY) + + async def one(triple: tuple[str, str, str]) -> Judgement: + alteration, published, answer = triple + async with gate: + return await self.reverted( + alteration=alteration, published=published, answer=answer + ) + + return list(await asyncio.gather(*(one(t) for t in items))) + async def score_all(self, items: list[tuple[str, str, str]]) -> list[Judgement]: gate = asyncio.Semaphore(_CONCURRENCY) diff --git a/src/loregraph/evals/perturbation.py b/src/loregraph/evals/perturbation.py index 509ec53..b033c6c 100644 --- a/src/loregraph/evals/perturbation.py +++ b/src/loregraph/evals/perturbation.py @@ -23,11 +23,20 @@ from __future__ import annotations +import asyncio import hashlib import re from dataclasses import dataclass, field from loregraph.evals.corpus import BookUnderTest +from loregraph.evals.model_arm import ( + CLOSED_BOOK_SYSTEM, + OPEN_BOOK_SYSTEM, + Answer, + Arm, + Judge, + available, +) from loregraph.evals.report import EvalResult # Names chosen to be pronounceable, era-neutral, and absent from any published @@ -394,3 +403,121 @@ def dry_run(book: BookUnderTest, *, per_kind: int = 2) -> EvalResult: *ineffective, ], ) + + +# How much text the open-book arm is shown. The whole novel would be the +# purest test and costs ~45k tokens a question; a window around the edit is +# both affordable and a fairer analogue of what a retrieval system would +# actually put in front of a model. +_WINDOW_CHUNKS = 3 + + +def _window(book: BookUnderTest, item: Perturbation, altered: str) -> str: + """The altered passage plus a few chunks either side, in reading order.""" + needles = [new for _, new in item.span_edits] + [new for _, new in item.replacements] + chunks = [c.text for c in book.chunks] + # Re-cut the altered full text along the original chunk boundaries by + # length, which is close enough: edits change length by a few characters. + hit = -1 + for i, chunk in enumerate(chunks): + probe = item.apply(chunk) + if any(n in probe for n in needles): + hit = i + break + if hit < 0: + return altered[:12_000] + lo = max(0, hit - _WINDOW_CHUNKS // 2) + hi = min(len(chunks), lo + _WINDOW_CHUNKS) + return "\n\n".join(item.apply(c) for c in chunks[lo:hi]) + + +async def run(book: BookUnderTest, *, per_kind: int = 2) -> EvalResult: + """Ask the same question of memory and of the altered text. + + Two arms, and neither is the pipeline: without a database the extraction + cannot be re-run on the altered source, so what this measures is the + *ceiling* the pipeline aims at — whether having the text in hand changes + the answer at all on a book the model knows by heart. If the open-book arm + also reverts to the published version, evidence-grounding is a genuinely + hard problem and the pipeline has a real target. If it follows the text + trivially, the argument for the pipeline has to rest on cost, latency and + auditability instead, not on accuracy. + """ + usable, _ = available() + if not usable or not book.has_text: + return dry_run(book, per_kind=per_kind) + + plan = [p for p in build(book, per_kind=per_kind) if p.replacements or p.span_edits] + if not plan: + return dry_run(book, per_kind=per_kind) + altered = perturbed_text(book, plan) + + closed = Arm("memory", CLOSED_BOOK_SYSTEM) + open_book = Arm("altered-text", OPEN_BOOK_SYSTEM) + judge = Judge() + + closed_answers = await closed.answer_all( + [f"Work: {book.title} by {book.author}.\n\n{p.question}" for p in plan] + ) + open_answers = await asyncio.gather( + *(open_book.answer(p.question, f"Excerpt:\n\n{_window(book, p, altered)}") for p in plan) + ) + + # A dedicated reversion judge, not the answer-scoring one. Feeding an + # instruction into that judge's "ground truth" slot scored two plainly + # correct answers as reverted — the same slot-abuse that made the first + # entailment run report 35.8% when the real figure was 85%. + def cases(answers: list[Answer]) -> list[tuple[str, str, str]]: + return [ + (p.description, f"{p.original_answer}", a.text) + for p, a in zip(plan, answers, strict=True) + ] + + closed_scores, open_scores = await asyncio.gather( + judge.reverted_all(cases(closed_answers)), + judge.reverted_all(cases(open_answers)), + ) + + findings = [] + for item, mem, txt, ms, ts in zip( + plan, closed_answers, open_answers, closed_scores, open_scores, strict=True + ): + findings.append( + { + "kind": item.kind, + "edit": item.description, + "question": item.question, + "from_memory": mem.text[:200], + "memory_followed_source": ms.correct, + "from_altered_text": txt.text[:200], + "text_followed_source": ts.correct, + } + ) + + mem_right = sum(1 for f in findings if f["memory_followed_source"]) + txt_right = sum(1 for f in findings if f["text_followed_source"]) + n = len(findings) + return EvalResult( + name="perturbation", + book_id=book.book_id, + headline=( + f"On {n} altered passages: answering from memory follows the altered " + f"source {mem_right}/{n} times, answering from the altered text does " + f"{txt_right}/{n}. Gap = {(txt_right - mem_right) / n:+.0%}." + ), + metrics={ + "perturbations": n, + "memory_follows_source": mem_right, + "text_follows_source": txt_right, + "reading_advantage": round((txt_right - mem_right) / n, 4), + }, + findings=findings, + skipped=[ + "neither arm is the LoreGraph pipeline: re-extracting from the altered " + "source needs the database, which is not available here. This measures " + "the ceiling the pipeline aims at, not the pipeline.", + f"the open-book arm sees a {_WINDOW_CHUNKS}-chunk window around each " + "edit, not the whole book — cheaper, and closer to what a retrieval " + "system would actually supply.", + ], + )