Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions evals/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,38 @@ means that law. Keyword statistics are kept across every loaded law, so a run on
holding other laws can differ slightly from one holding that law alone. The published scores
are measured with every pack loaded, and so is CI's gate.

## Judging a change

A change to search is judged on the tuning sets, by a rule fixed before the change is
measured. Each question scores by the place of the first clause that answers it among the ten
returned: 1 for first place, a half for second, down to a tenth, and nothing when none answers.
A count of hits at five would not see a clause climb from ninth to sixth, or fall from first to
fifth.

Two runs are compared question by question. A topic asked more than once, in another language
or the way people type in chat, is one topic: its questions move together, so their changes are
counted together. The change is kept when both of these hold:

- **The gain is more than chance gives.** Each topic's change is given a random sign, ten
thousand times, and the measured gain has to be larger than 95% of those.
- **No set is worse.** A set is worse when its score falls by more than chance explains, by the
same test on that set alone, with the 5% shared between the sets.

The held-out sets take no part in it. They are measured for the published score.

```bash
uv run dpolens evals run --pack gdpr --outcomes before/gdpr.json
uv run dpolens evals run --pack uae-pdpl --outcomes before/uae-pdpl.json
# with the change in place, and whatever it needs rebuilt:
uv run dpolens evals run --pack gdpr --outcomes after/gdpr.json
uv run dpolens evals run --pack uae-pdpl --outcomes after/uae-pdpl.json
uv run dpolens evals compare before after
```

`--outcomes` writes, for every question of the configuration being scored, the clauses returned
and the place of the first that answers. `compare` takes two such files, or two folders with a
file for each law, and prints each set before and after, then whether the change is kept.

## Running them

```bash
Expand Down
117 changes: 117 additions & 0 deletions packages/dpolens/src/dpolens/cli/evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,18 @@
from dpolens.engine.embedding import DEFAULT_MODEL, Embedder, get_model
from dpolens.engine.evals import (
EvalSetError,
Judgement,
Placed,
Run,
Unpaired,
check_configuration,
judge,
placed_in,
read_questions,
run_set,
run_together,
sets_named,
topic_of,
)
from dpolens.engine.session import session_scope
from dpolens.settings import load_settings
Expand Down Expand Up @@ -82,6 +88,15 @@ def run(
typer.Option(help="Also score the sets as one, a topic asked in each counted once"),
] = False,
write: Annotated[bool, typer.Option(help="Write evals/results/<pack>.json")] = False,
outcomes: Annotated[
Path | None,
typer.Option(
help=(
"Write where each question's right clause came to this file, for "
"`dpolens evals compare`"
)
),
] = None,
as_json: Annotated[
bool,
typer.Option("--json", help="Print one run as JSON: the sets together, or the first"),
Expand Down Expand Up @@ -128,6 +143,8 @@ def run(
if together
else []
)
if outcomes is not None:
_keep(pack, [result for result in runs if result.configuration == configuration], outcomes)
runs += combined

if as_json:
Expand All @@ -152,6 +169,106 @@ def run(
_write(pack, runs)


@app.command()
def compare(
before: Annotated[
Path,
typer.Argument(help="An outcomes file from before a change, or a folder of them"),
],
after: Annotated[Path, typer.Argument(help="The same, from after it")],
) -> None:
"""Judge a change to search from two runs over the same questions."""
try:
judgement = judge(_placed(before), _placed(after))
except Unpaired as problem:
raise typer.BadParameter(str(problem)) from problem

for change in judgement.sets:
typer.echo(
f"{change.pack} {change.question_set}: {change.questions} questions "
f"hits at five {change.hits_before} to {change.hits_after} "
f"place score {change.score_before:.3f} to {change.score_after:.3f}"
)
typer.echo(
f"\nAll: {judgement.questions} questions on {judgement.topics} topics "
f"hits at five {judgement.hits_before} to {judgement.hits_after} "
f"place score {judgement.score_before:.3f} to {judgement.score_after:.3f}"
)
typer.echo(f"By topic at five: {judgement.topics_found} found, {judgement.topics_lost} lost\n")
typer.echo(_verdict(judgement))


def _verdict(judgement: Judgement) -> str:
said = []
if judgement.score_after <= judgement.score_before:
said.append("No gain: the right clauses sit no higher than before.")
else:
said.append(
f"Chance alone gives a gain this large {_share(judgement.chance)} of the time, "
"and a change is kept under 5%."
)
for change in judgement.sets:
if change.worse:
said.append(
f"{change.pack} {change.question_set} is worse: chance alone gives it a loss "
f"this large {_share(change.chance_of_loss)} of the time."
)
said.append("Kept." if judgement.kept else "Not kept.")
return "\n".join(said)


def _share(chance: float) -> str:
return f"{100 * chance:.2f}%"


def _placed(path: Path) -> list[Placed]:
"""Every question in an outcomes file, or in each file of a folder."""
files = sorted(path.glob("*.json")) if path.is_dir() else [path]
if not files:
raise typer.BadParameter(f"{path} holds no outcomes file")
placed: list[Placed] = []
for file in files:
try:
placed += placed_in(json.loads(file.read_text(encoding="utf-8")))
except (OSError, ValueError, KeyError, TypeError) as problem:
raise typer.BadParameter(
f"{file} is not an outcomes file, as `dpolens evals run --outcomes` writes one"
) from problem
return placed


def _keep(pack: str, runs: list[Run], path: Path) -> None:
"""The configuration scored, with where each question's right clause came."""
payload = {
"pack": pack,
"measured_at": datetime.now(UTC).date().isoformat(),
"commit": _commit(),
"configuration": runs[0].configuration,
"model": runs[0].model,
"context_recipe": runs[0].recipe,
"sets": [
{
"question_set": result.question_set,
"questions": [
{
"id": outcome.question_id,
"topic": topic_of(outcome.question_id),
"expected": list(outcome.expected),
"returned": list(outcome.returned),
"place": outcome.first_hit,
}
for outcome in result.outcomes
],
}
for result in runs
],
}
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
# Not to standard output, which `--json` keeps to the one record a script reads.
typer.echo(f"Wrote {path}", err=True)


def _as_record(result: Run) -> dict[str, Any]:
return {
"question_set": result.question_set,
Expand Down
16 changes: 16 additions & 0 deletions packages/dpolens/src/dpolens/engine/evals/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
"""Measuring whether retrieval finds the right clause."""

from dpolens.engine.evals.gate import Verdict, compare
from dpolens.engine.evals.judge import (
Judgement,
Placed,
SetChange,
Unpaired,
judge,
placed_in,
topic_of,
)
from dpolens.engine.evals.run import (
EvalSetError,
Question,
Expand All @@ -23,18 +32,25 @@
__all__ = [
"EvalSetError",
"Interval",
"Judgement",
"Outcome",
"Placed",
"Question",
"Run",
"Score",
"SetChange",
"Unpaired",
"Verdict",
"bootstrap_interval",
"check_configuration",
"compare",
"grouped_interval",
"judge",
"placed_in",
"read_questions",
"recall_at",
"run_set",
"run_together",
"sets_named",
"topic_of",
]
Loading
Loading