diff --git a/src/loregraph/cli/main.py b/src/loregraph/cli/main.py index 2945356..107bc32 100644 --- a/src/loregraph/cli/main.py +++ b/src/loregraph/cli/main.py @@ -243,7 +243,7 @@ def eval_( # These fall back to a dry preview when no provider is configured, # printing exactly what would be sent rather than failing or, worse, # silently scoring a subset. - "perturbation": perturbation.dry_run, + "perturbation": lambda b: asyncio.run(perturbation.run(b, per_kind=2)), "contamination": lambda b: asyncio.run(contamination.run(b, limit=probes)), "entailment": lambda b: asyncio.run(entailment.run(b, budget=budget)), } diff --git a/src/loregraph/config.py b/src/loregraph/config.py index 3287f45..d8e1b3c 100644 --- a/src/loregraph/config.py +++ b/src/loregraph/config.py @@ -98,10 +98,14 @@ class Settings(BaseSettings): # Floor on the fraction of sampled claims the judge finds *supported* by # their evidence span. This is the gate that actually measures extraction # quality — the literal-match rate is an upstream invariant and cannot - # fail. PROVISIONAL: no calibrated distribution exists yet; run - # `loregraph eval entailment` to measure a book before trusting a number. - # 0 disables the gate (records the rate without enforcing it). - cove_supported_floor: float = Field(0.85, alias="LOREGRAPH_COVE_SUPPORTED_FLOOR") + # fail. 0 disables the gate (records the rate without enforcing it). + # + # Measured, not guessed: 0.80 is alice's rate (102/120 sampled claims, + # deepseek-chat as judge, 2026-08-03) less a 5-point margin. The earlier + # provisional 0.85 sat exactly at the measured rate and would have aborted + # every run. ONE book is not a distribution — re-measure with + # `loregraph eval entailment` before relying on this on a new corpus. + cove_supported_floor: float = Field(0.80, alias="LOREGRAPH_COVE_SUPPORTED_FLOOR") # ── Provider lookup helpers ───────────────────────────────────── def resolved_api_key(self, provider: str) -> str | None: diff --git a/src/loregraph/evals/contamination.py b/src/loregraph/evals/contamination.py index d76c3eb..e3a60f3 100644 --- a/src/loregraph/evals/contamination.py +++ b/src/loregraph/evals/contamination.py @@ -19,7 +19,7 @@ from __future__ import annotations import asyncio -from collections.abc import Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass from typing import TypeVar @@ -39,6 +39,15 @@ class Probe: question: str ground_truth: str + """The **source passage**, not a restatement of the graph's claim. + + The first version of this eval set ground_truth to a paraphrase of + graph_answer, so the judge was asked whether an answer matched itself and + the graph scored 12/12 by construction — the same tautology this project's + literal-match gate had. Grade both arms against the text, and the graph can + lose. + """ + graph_answer: str """What the graph alone says — the pipeline's arm, assembled without a model.""" source: str @@ -56,6 +65,8 @@ def build(book: BookUnderTest, *, limit: int = 24) -> list[Probe]: """ probes: list[Probe] = [] + passage = book.chunk_text + edges = _confident(book.edges) # Spread across chapters so the battery is not all opening-scene trivia, # which is the part of a famous book a model remembers best. @@ -66,10 +77,7 @@ def build(book: BookUnderTest, *, limit: int = 24) -> list[Probe]: probes.append( Probe( question=f"In this work, what is the relationship between {src} and {dst}?", - ground_truth=( - f"{src} {edge.predicate or edge.relation} {dst}. " - f"The text reads: {edge.evidence_span.strip()!r}" - ), + ground_truth=passage.get(edge.atom_id, edge.evidence_span)[:1800], graph_answer=( f"{src} —{edge.predicate or edge.relation}→ {dst} " f"({edge.evidence_span.strip()!r}, {edge.atom_id})" @@ -89,7 +97,7 @@ def build(book: BookUnderTest, *, limit: int = 24) -> list[Probe]: f"In this work, what does the text establish about {who} " f"regarding {fact.dimension}?" ), - ground_truth=(f"{fact.statement} The text reads: {fact.evidence_span.strip()!r}"), + ground_truth=passage.get(fact.atom_id, fact.evidence_span)[:1800], graph_answer=f"{fact.statement} ({fact.evidence_span.strip()!r}, {fact.atom_id})", source=fact.atom_id, ) @@ -137,14 +145,22 @@ async def run(book: BookUnderTest, *, limit: int = 24) -> EvalResult: question_with_work = [f"Work: {book.title} by {book.author}.\n\n{p.question}" for p in probes] closed_answers = await closed.answer_all(question_with_work) + def graded(answer_of: Callable[[int, Probe], str]) -> list[tuple[str, str, str]]: + return [ + ( + p.question, + answer_of(i, p), + "The source passage below is the only authority. An answer is " + "correct if the passage states or directly implies it, and " + "incorrect if the passage contradicts it or is silent on it.\n\n" + f"Passage:\n{p.ground_truth}", + ) + for i, p in enumerate(probes) + ] + closed_scores, graph_scores = await asyncio.gather( - judge.score_all( - [ - (p.question, a.text, p.ground_truth) - for p, a in zip(probes, closed_answers, strict=True) - ] - ), - judge.score_all([(p.question, p.graph_answer, p.ground_truth) for p in probes]), + judge.score_all(graded(lambda i, p: closed_answers[i].text)), + judge.score_all(graded(lambda i, p: p.graph_answer)), ) closed_right = sum(1 for s in closed_scores if s.correct) diff --git a/src/loregraph/evals/entailment.py b/src/loregraph/evals/entailment.py index aa28732..d3dd43f 100644 --- a/src/loregraph/evals/entailment.py +++ b/src/loregraph/evals/entailment.py @@ -73,6 +73,25 @@ def sample(book: BookUnderTest, *, budget: int = 150, seed: int = 7) -> list[Cla ) +# How much of the chunk the judge sees. A blind head-truncation is a trap: the +# evidence span often sits past it, the judge correctly reports "that quote is +# not in the passage you gave me", and the eval scores the truncation instead +# of the extraction. Centre the window on the span so it is always included. +_PASSAGE_CHARS = 2400 + + +def _passage_around(claim: Claim) -> str: + text = claim.chunk_text + if len(text) <= _PASSAGE_CHARS: + return text + at = text.find(claim.evidence_span) + if at < 0: # should not happen — spans are literal by construction + return text[:_PASSAGE_CHARS] + half = (_PASSAGE_CHARS - len(claim.evidence_span)) // 2 + lo = max(0, at - half) + return text[lo : lo + _PASSAGE_CHARS] + + def preview(book: BookUnderTest, *, budget: int = 150) -> EvalResult: picked = sample(book, budget=budget) strata: dict[str, int] = {} @@ -116,19 +135,8 @@ async def run(book: BookUnderTest, *, budget: int = 150) -> EvalResult: ) judge = Judge() - verdicts = await judge.score_all( - [ - ( - f"Does this passage support the claim?\n\nPassage: {c.chunk_text[:1500]}", - f"Claim: {c.statement}\nCited span: {c.evidence_span}", - ( - "The claim is supported only if the cited span, read in the " - "passage, states or directly implies it. A span that is real " - "but about something else is NOT support." - ), - ) - for c in picked - ] + verdicts = await judge.entails_all( + [(c.statement, c.evidence_span, _passage_around(c)) for c in picked] ) by_stratum: dict[str, list[int]] = {} diff --git a/src/loregraph/evals/model_arm.py b/src/loregraph/evals/model_arm.py index c599a7b..2d9e0a7 100644 --- a/src/loregraph/evals/model_arm.py +++ b/src/loregraph/evals/model_arm.py @@ -25,6 +25,31 @@ _CONCURRENCY = 8 +# Every Arm and Judge merges into this, so a run's cost is one number at the +# end rather than something you reconstruct from provider dashboards later. +SPEND = LLMUsage() + + +def spend_report() -> dict[str, float | int]: + """Tokens used so far this process, and what they cost. + + Prices come from settings so they track the configured provider; the + defaults are DeepSeek's. `usd` is an estimate — it ignores the provider's + cache-hit discount, so it reads high rather than low. + """ + settings = get_settings() + usd = ( + SPEND.input_tokens / 1_000_000 * settings.price_per_mtok_input + + SPEND.output_tokens / 1_000_000 * settings.price_per_mtok_output + ) + return { + "requests": SPEND.requests, + "input_tokens": SPEND.input_tokens, + "output_tokens": SPEND.output_tokens, + "usd": round(usd, 4), + "cny": round(usd * 7.15, 3), + } + def available() -> tuple[bool, str]: """(usable, why-not). Never raises — callers report, they do not crash.""" @@ -70,6 +95,7 @@ async def answer(self, question: str, context: str = "") -> Answer: user = f"{context}\n\n{question}".strip() if context else question message = await self.llm.complete(system=self.system, user=user) self.usage.merge(message) + SPEND.merge(message) return Answer(question=question, text=self.llm.extract_text(message).strip(), arm=self.name) async def answer_all(self, questions: list[str], context: str = "") -> list[Answer]: @@ -110,6 +136,32 @@ async def one(q: str) -> Answer: ) +ENTAILMENT_SYSTEM = ( + "You decide whether a quoted span of a passage supports a claim extracted " + "from it. Return JSON only: " + '{"correct": bool, "follows_source": true, "confidence": 0.0-1.0, "reason": "..."}. ' + "`correct` is true only if the cited span, read in its surrounding passage, " + "states the claim or directly implies it. A span that is genuinely in the " + "text but is about something else does NOT support the claim. Judge the " + "claim against the span and passage alone; do not use anything you know " + "about the work from elsewhere." +) + + +REVERSION_SYSTEM = ( + "A passage of a well-known work has been deliberately altered. You are " + "given the alteration, what the published work says, and an answer. Decide " + "whether the answer reflects the ALTERED text or reverts to the published " + "version. Return JSON only: " + '{"correct": bool, "follows_source": bool, "confidence": 0.0-1.0, "reason": "..."}. ' + "Set both true when the answer matches the altered text. Set both false " + "when it gives the published version instead, or asserts the altered " + "detail does not exist. An answer that quotes the altered clause but then " + "states the published fact has reverted: the quote is not the answer. " + "Ignore style, length and hedging; judge only the substance." +) + + class Judge: def __init__(self, llm: LLMClient | None = None) -> None: self.llm = llm or make_llm_client() @@ -119,6 +171,7 @@ async def score(self, *, question: str, answer: str, ground_truth: str) -> Judge user = f"Question: {question}\n\nGround truth: {ground_truth}\n\nAnswer to score: {answer}" message = await self.llm.complete(system=JUDGE_SYSTEM, user=user) self.usage.merge(message) + SPEND.merge(message) try: return parse_into(Judgement, self.llm.extract_text(message)) except LLMOutputError: @@ -127,6 +180,63 @@ async def score(self, *, question: str, answer: str, ground_truth: str) -> Judge correct=False, follows_source=False, confidence=0.0, reason="unparsable judgement" ) + async def entails(self, *, claim: str, span: str, passage: str) -> Judgement: + """Does `span`, read inside `passage`, support `claim`?""" + user = ( + f"Passage:\n{passage}\n\n" + f"Cited span:\n{span}\n\n" + f"Claim extracted from that span:\n{claim}" + ) + message = await self.llm.complete(system=ENTAILMENT_SYSTEM, user=user) + self.usage.merge(message) + SPEND.merge(message) + try: + return parse_into(Judgement, self.llm.extract_text(message)) + except LLMOutputError: + log.warning("entailment judge returned malformed JSON; scoring as unsupported") + return Judgement(correct=False, confidence=0.0, reason="unparsable judgement") + + async def entails_all(self, items: list[tuple[str, str, str]]) -> list[Judgement]: + gate = asyncio.Semaphore(_CONCURRENCY) + + async def one(triple: tuple[str, str, str]) -> Judgement: + claim, span, passage = triple + async with gate: + return await self.entails(claim=claim, span=span, passage=passage) + + return list(await asyncio.gather(*(one(t) for t in items))) + + async def reverted(self, *, alteration: str, published: str, answer: str) -> Judgement: + message = await self.llm.complete( + system=REVERSION_SYSTEM, + user=( + f"Alteration made to the text:\n{alteration}\n\n" + f"What the published work says:\n{published}\n\n" + f"Answer to judge:\n{answer}" + ), + ) + self.usage.merge(message) + SPEND.merge(message) + try: + return parse_into(Judgement, self.llm.extract_text(message)) + except LLMOutputError: + log.warning("reversion judge returned malformed JSON; scoring as reverted") + return Judgement( + correct=False, follows_source=False, confidence=0.0, reason="unparsable" + ) + + async def reverted_all(self, items: list[tuple[str, str, str]]) -> list[Judgement]: + gate = asyncio.Semaphore(_CONCURRENCY) + + async def one(triple: tuple[str, str, str]) -> Judgement: + alteration, published, answer = triple + async with gate: + return await self.reverted( + alteration=alteration, published=published, answer=answer + ) + + return list(await asyncio.gather(*(one(t) for t in items))) + async def score_all(self, items: list[tuple[str, str, str]]) -> list[Judgement]: gate = asyncio.Semaphore(_CONCURRENCY) diff --git a/src/loregraph/evals/perturbation.py b/src/loregraph/evals/perturbation.py index 509ec53..b033c6c 100644 --- a/src/loregraph/evals/perturbation.py +++ b/src/loregraph/evals/perturbation.py @@ -23,11 +23,20 @@ from __future__ import annotations +import asyncio import hashlib import re from dataclasses import dataclass, field from loregraph.evals.corpus import BookUnderTest +from loregraph.evals.model_arm import ( + CLOSED_BOOK_SYSTEM, + OPEN_BOOK_SYSTEM, + Answer, + Arm, + Judge, + available, +) from loregraph.evals.report import EvalResult # Names chosen to be pronounceable, era-neutral, and absent from any published @@ -394,3 +403,121 @@ def dry_run(book: BookUnderTest, *, per_kind: int = 2) -> EvalResult: *ineffective, ], ) + + +# How much text the open-book arm is shown. The whole novel would be the +# purest test and costs ~45k tokens a question; a window around the edit is +# both affordable and a fairer analogue of what a retrieval system would +# actually put in front of a model. +_WINDOW_CHUNKS = 3 + + +def _window(book: BookUnderTest, item: Perturbation, altered: str) -> str: + """The altered passage plus a few chunks either side, in reading order.""" + needles = [new for _, new in item.span_edits] + [new for _, new in item.replacements] + chunks = [c.text for c in book.chunks] + # Re-cut the altered full text along the original chunk boundaries by + # length, which is close enough: edits change length by a few characters. + hit = -1 + for i, chunk in enumerate(chunks): + probe = item.apply(chunk) + if any(n in probe for n in needles): + hit = i + break + if hit < 0: + return altered[:12_000] + lo = max(0, hit - _WINDOW_CHUNKS // 2) + hi = min(len(chunks), lo + _WINDOW_CHUNKS) + return "\n\n".join(item.apply(c) for c in chunks[lo:hi]) + + +async def run(book: BookUnderTest, *, per_kind: int = 2) -> EvalResult: + """Ask the same question of memory and of the altered text. + + Two arms, and neither is the pipeline: without a database the extraction + cannot be re-run on the altered source, so what this measures is the + *ceiling* the pipeline aims at — whether having the text in hand changes + the answer at all on a book the model knows by heart. If the open-book arm + also reverts to the published version, evidence-grounding is a genuinely + hard problem and the pipeline has a real target. If it follows the text + trivially, the argument for the pipeline has to rest on cost, latency and + auditability instead, not on accuracy. + """ + usable, _ = available() + if not usable or not book.has_text: + return dry_run(book, per_kind=per_kind) + + plan = [p for p in build(book, per_kind=per_kind) if p.replacements or p.span_edits] + if not plan: + return dry_run(book, per_kind=per_kind) + altered = perturbed_text(book, plan) + + closed = Arm("memory", CLOSED_BOOK_SYSTEM) + open_book = Arm("altered-text", OPEN_BOOK_SYSTEM) + judge = Judge() + + closed_answers = await closed.answer_all( + [f"Work: {book.title} by {book.author}.\n\n{p.question}" for p in plan] + ) + open_answers = await asyncio.gather( + *(open_book.answer(p.question, f"Excerpt:\n\n{_window(book, p, altered)}") for p in plan) + ) + + # A dedicated reversion judge, not the answer-scoring one. Feeding an + # instruction into that judge's "ground truth" slot scored two plainly + # correct answers as reverted — the same slot-abuse that made the first + # entailment run report 35.8% when the real figure was 85%. + def cases(answers: list[Answer]) -> list[tuple[str, str, str]]: + return [ + (p.description, f"{p.original_answer}", a.text) + for p, a in zip(plan, answers, strict=True) + ] + + closed_scores, open_scores = await asyncio.gather( + judge.reverted_all(cases(closed_answers)), + judge.reverted_all(cases(open_answers)), + ) + + findings = [] + for item, mem, txt, ms, ts in zip( + plan, closed_answers, open_answers, closed_scores, open_scores, strict=True + ): + findings.append( + { + "kind": item.kind, + "edit": item.description, + "question": item.question, + "from_memory": mem.text[:200], + "memory_followed_source": ms.correct, + "from_altered_text": txt.text[:200], + "text_followed_source": ts.correct, + } + ) + + mem_right = sum(1 for f in findings if f["memory_followed_source"]) + txt_right = sum(1 for f in findings if f["text_followed_source"]) + n = len(findings) + return EvalResult( + name="perturbation", + book_id=book.book_id, + headline=( + f"On {n} altered passages: answering from memory follows the altered " + f"source {mem_right}/{n} times, answering from the altered text does " + f"{txt_right}/{n}. Gap = {(txt_right - mem_right) / n:+.0%}." + ), + metrics={ + "perturbations": n, + "memory_follows_source": mem_right, + "text_follows_source": txt_right, + "reading_advantage": round((txt_right - mem_right) / n, 4), + }, + findings=findings, + skipped=[ + "neither arm is the LoreGraph pipeline: re-extracting from the altered " + "source needs the database, which is not available here. This measures " + "the ceiling the pipeline aims at, not the pipeline.", + f"the open-book arm sees a {_WINDOW_CHUNKS}-chunk window around each " + "edit, not the whole book — cheaper, and closer to what a retrieval " + "system would actually supply.", + ], + )