Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions evals/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,12 @@ means that law. Keyword statistics are kept across every loaded law, so a run on
holding other laws can differ slightly from one holding that law alone. The published scores
are measured with every pack loaded, and so is CI's gate.

An organisation that selects several laws searches them together. `--every-law` measures a
law that way: the run searches every law loaded, the pack's questions still expect the pack's
own clauses, and the set is named with the laws beside it, such as `tuning.ar beside uae-pdpl`.
The laws then share the ten results, so `recall@10` is the figure to read. Such a run is never
written to `evals/results`: the published scores are of a law searched alone.

## Judging a change

A change to search is judged on the sets used while improving retrieval, tuning and
Expand Down Expand Up @@ -166,6 +172,10 @@ uv run dpolens evals run --pack uae-pdpl --outcomes after/uae-pdpl.json
uv run dpolens evals compare before after
```

A change to how several laws share the results is judged with the same runs made again with
`--every-law`, kept in the same folders under another file name. A law alone and a law beside
the others are two sets, and a topic asked in both is still one topic.

`--outcomes` writes, for every question of the configuration being scored, the clauses returned
and the place of the first that answers. `compare` takes two such files, or two folders with a
file for each law, and prints each set before and after, then whether the change is kept.
Expand All @@ -176,6 +186,7 @@ file for each law, and prints each set before and after, then whether the change
uv run dpolens evals run --pack gdpr --set tuning
uv run dpolens evals run --pack gdpr --set tuning.ar
uv run dpolens evals run --pack gdpr --set assistant
uv run dpolens evals run --pack gdpr --set tuning --every-law
uv run dpolens evals run --pack gdpr --set held_out --baselines --write
uv run dpolens evals run --pack uae-pdpl --set held_out --together --baselines --write
```
Expand Down
31 changes: 27 additions & 4 deletions packages/dpolens/src/dpolens/cli/evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,16 @@ def run(
bool,
typer.Option(help="Also score the sets as one, a topic asked in each counted once"),
] = False,
every_law: Annotated[
bool,
typer.Option(
"--every-law",
help=(
"Search every law loaded here, not the pack's alone, as an organisation "
"that selected them all does"
),
),
] = False,
write: Annotated[bool, typer.Option(help="Write evals/results/<pack>.json")] = False,
outcomes: Annotated[
Path | None,
Expand Down Expand Up @@ -125,14 +135,27 @@ def run(
"so a number measured on a set used for tuning does not belong in it.",
param_hint="--set",
)
if write and every_law:
raise typer.BadParameter(
"--write is for a law searched alone, which is what the published scores measure",
param_hint="--every-law",
)
questions = {name: read_questions(EVALS / pack / f"{name}.yaml") for name in sets}

with (
Embedder(get_model(model)) as embedder,
session_scope(load_settings()) as session,
):
runs = [
run_set(session, embedder, questions[name], config, pack=pack, question_set=name)
run_set(
session,
embedder,
questions[name],
config,
pack=pack,
question_set=name,
every_law=every_law,
)
for name in sets
for config in configurations
]
Expand All @@ -155,7 +178,7 @@ def run(

for name in dict.fromkeys(result.question_set for result in runs):
block = [result for result in runs if result.question_set == name]
typer.echo(f"{pack} {name}: {block[0].score.questions} questions, model {model}\n")
typer.echo(f"{pack} {block[0].name}: {block[0].score.questions} questions, model {model}\n")
for result in block:
typer.echo(
f" {result.configuration:12} recall@5 {result.score.recall_at_5:.2f} "
Expand Down Expand Up @@ -257,7 +280,7 @@ def _keep(pack: str, runs: list[Run], path: Path) -> None:
"context_recipe": runs[0].recipe,
"sets": [
{
"question_set": result.question_set,
"question_set": result.name,
"questions": [
{
"id": outcome.question_id,
Expand All @@ -280,7 +303,7 @@ def _keep(pack: str, runs: list[Run], path: Path) -> None:

def _as_record(result: Run) -> dict[str, Any]:
return {
"question_set": result.question_set,
"question_set": result.name,
"configuration": result.configuration,
"model": result.model,
"context_recipe": result.recipe,
Expand Down
2 changes: 2 additions & 0 deletions packages/dpolens/src/dpolens/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
EmptySelection,
NothingSelected,
NotSelected,
OnlyLaw,
UnknownLaw,
)
from dpolens.engine.policies.blocks import UnreadablePolicy
Expand Down Expand Up @@ -108,6 +109,7 @@
UnknownToken,
SurfaceOwnerIsAPerson,
UnknownLaw,
OnlyLaw,
NotSelected,
NothingSelected,
WeakPassword,
Expand Down
34 changes: 30 additions & 4 deletions packages/dpolens/src/dpolens/engine/evals/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from dpolens.engine.embedding.encode import Embedder
from dpolens.engine.embedding.recipe import RECIPE_VERSION
from dpolens.engine.evals.score import Outcome, Score, score
from dpolens.engine.packs.selection import Scope, law_alone
from dpolens.engine.packs.selection import Scope, law_alone, law_beside_the_others
from dpolens.engine.search.engine import CANDIDATE_DEPTH, search
from dpolens.engine.search.fuse import Fused, Fusion, fuse
from dpolens.engine.search.keyword import ALTERNATIVES, Candidate, KeywordIndex, keyword_search
Expand Down Expand Up @@ -56,6 +56,15 @@ class Run:
recipe: str
score: Score
outcomes: tuple[Outcome, ...]
beside: tuple[str, ...] = ()
"""The other laws searched with the pack's own, in a run that measures it beside them."""

@property
def name(self) -> str:
"""The set as a reader names it: tuning.ar, or tuning.ar beside uae-pdpl."""
if not self.beside:
return self.question_set
return f"{self.question_set} beside {', '.join(self.beside)}"


def read_questions(path: Path) -> list[Question]:
Expand Down Expand Up @@ -112,6 +121,7 @@ def run_set(
pack: str = "",
question_set: str = "",
reranker: Reranker | None = None,
every_law: bool = False,
) -> Run:
"""Score one configuration over one question set.

Expand All @@ -125,9 +135,19 @@ def run_set(
A run for a pack searches that pack's law alone, whatever else is loaded or
selected, so its score means that law. Without a pack it searches what any
search would.

With `every_law`, a run for a pack searches every loaded law, as an organisation
that selected them all does. Its questions still expect the pack's own clauses,
so the score says how well that law is found beside the others.
"""
check_configuration(configuration, reranker)
scope = law_alone(session, pack) if pack else None
if every_law and not pack:
raise ValueError("a run beside every law is a run for a pack")
if every_law:
scope: Scope | None = law_beside_the_others(session, pack)
else:
scope = law_alone(session, pack) if pack else None
beside = tuple(law for law in scope.laws if law != pack) if scope and every_law else ()
outcomes = []
for question in questions:
returned = _retrieve(session, embedder, question, configuration, reranker, scope)
Expand All @@ -148,6 +168,7 @@ def run_set(
recipe=RECIPE_VERSION,
score=score(outcomes, group_of=_item_of(questions)),
outcomes=tuple(outcomes),
beside=beside,
)


Expand Down Expand Up @@ -182,10 +203,14 @@ def run_together(runs: Sequence[Run]) -> Run:
"""
first = runs[0]
if any(
(run.pack, run.configuration, run.model) != (first.pack, first.configuration, first.model)
(run.pack, run.configuration, run.model, run.beside)
!= (first.pack, first.configuration, first.model, first.beside)
for run in runs
):
raise ValueError("only runs of one pack, configuration and model are scored together")
raise ValueError(
"only runs of one pack, configuration and model, searched beside the same laws, "
"are scored together"
)

outcomes = tuple(
Outcome(
Expand All @@ -204,6 +229,7 @@ def run_together(runs: Sequence[Run]) -> Run:
recipe=first.recipe,
score=score(outcomes, group_of=lambda outcome: outcome.question_id.split("/", 1)[1]),
outcomes=outcomes,
beside=first.beside,
)


Expand Down
22 changes: 22 additions & 0 deletions packages/dpolens/src/dpolens/engine/packs/selection.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,16 @@ class EmptySelection(Exception):
"""A selection names at least one law."""


class OnlyLaw(Exception):
"""A law cannot be searched beside the others where no other is loaded."""

def __init__(self, slug: str) -> None:
self.slug = slug
super().__init__(
f"{slug} is the only law loaded here, so there is no other to search it beside"
)


@dataclass(frozen=True)
class Scope:
"""The laws one search covers, against the laws loaded here."""
Expand Down Expand Up @@ -148,6 +158,18 @@ def law_alone(session: Session, slug: str) -> Scope:
return Scope(laws=(slug,), loaded=loaded)


def law_beside_the_others(session: Session, slug: str) -> Scope:
"""One loaded law with every other loaded law beside it, whatever is selected, which is
what an eval run searches to measure a law as an organisation that chose several finds
it."""
loaded = tuple(pack.slug for pack in loaded_laws(session))
if slug not in loaded:
raise UnknownLaw([slug], loaded)
if len(loaded) == 1:
raise OnlyLaw(slug)
return Scope(laws=loaded, loaded=loaded)


def newly_arrived(session: Session, *, now: datetime | None = None) -> frozenset[str]:
"""The laws loaded after the laws were first chosen, within the last 30 days.

Expand Down
9 changes: 9 additions & 0 deletions tests/integration/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,15 @@ def test_only_a_held_out_set_is_written_and_before_anything_runs(in_the_checkout
assert "--write is for held-out sets, not 'tuning.ar'" in said(result)


def test_a_run_beside_every_law_is_never_written(in_the_checkout: None) -> None:
result = runner.invoke(
app, ["evals", "run", "--pack", "gdpr", "--set", "held_out", "--every-law", "--write"]
)

assert result.exit_code == 2
assert "--write is for a law searched alone" in said(result)


def test_scoring_together_needs_two_sets(in_the_checkout: None) -> None:
result = runner.invoke(
app, ["evals", "run", "--pack", "gdpr", "--set", "held_out.en", "--together"]
Expand Down
13 changes: 13 additions & 0 deletions tests/integration/test_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,3 +227,16 @@ def test_a_set_asked_in_parts_keeps_each_parts_topic_and_counts_whole_queries(
]
assert f"gdpr assistant.en: {len(questions)} questions" in ran.output
assert f"of {queries} queries in the top five" in " ".join(ran.output.split())


def test_a_law_loaded_alone_has_no_other_to_be_searched_beside(
indexed: Session, migrated: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.chdir(REPO)
monkeypatch.setenv("DPOLENS_DATABASE_URL", migrated)

asked_for = ["--pack", "gdpr", "--set", "tuning.en", "--config", "keyword", "--every-law"]
ran = CliRunner().invoke(app, ["evals", "run", *asked_for])

assert ran.exit_code == 1
assert "gdpr is the only law loaded here" in ran.output
107 changes: 107 additions & 0 deletions tests/integration/test_evals_every_law.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
"""A law measured beside the others, as an organisation that selected several searches.

Both real packs are loaded and nothing is embedded: keyword search needs no vectors, and it
is enough to see which laws a run reaches.
"""

from __future__ import annotations

import json
from collections.abc import Callable, Iterator
from pathlib import Path

import pytest
from sqlalchemy import Engine
from sqlalchemy.orm import Session
from typer.testing import CliRunner

from dpolens.cli.main import app
from dpolens.engine.embedding import Embedder, get_model
from dpolens.engine.evals import read_questions, run_set
from dpolens.engine.packs.load import load_pack

pytestmark = pytest.mark.integration

REPO = Path(__file__).parents[2]
TUNING = REPO / "evals" / "uae-pdpl" / "tuning.en.yaml"
KEYWORD = "keyword+normative"


@pytest.fixture(scope="session")
def embedder() -> Iterator[Embedder]:
with Embedder(get_model("e5-small")) as loaded:
yield loaded


@pytest.fixture(scope="module")
def both_laws(engine: Engine, truncate_corpus: Callable[[], None]) -> Iterator[Session]:
with Session(engine) as prepared:
load_pack(prepared, REPO / "packs" / "gdpr")
load_pack(prepared, REPO / "packs" / "uae-pdpl")
prepared.commit()
yield prepared

truncate_corpus()


def laws_reached(returned: tuple[str, ...]) -> set[str]:
return {key.split(":", 1)[0] for key in returned}


def test_a_run_for_a_pack_reaches_its_law_alone(both_laws: Session, embedder: Embedder) -> None:
run = run_set(both_laws, embedder, read_questions(TUNING), KEYWORD, pack="uae-pdpl")

assert run.beside == ()
assert {law for outcome in run.outcomes for law in laws_reached(outcome.returned)} == {
"uae-pdpl"
}


def test_beside_every_law_a_run_reaches_them_all_and_says_which(
both_laws: Session, embedder: Embedder
) -> None:
questions = read_questions(TUNING)

run = run_set(
both_laws,
embedder,
questions,
KEYWORD,
pack="uae-pdpl",
question_set="tuning.en",
every_law=True,
)

assert (run.beside, run.name) == (("gdpr",), "tuning.en beside gdpr")
assert {law for outcome in run.outcomes for law in laws_reached(outcome.returned)} == {
"gdpr",
"uae-pdpl",
}
assert [outcome.expected for outcome in run.outcomes] == [
question.expected_keys for question in questions
]


def test_the_command_keeps_such_a_run_as_a_set_of_its_own(
both_laws: Session, migrated: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Kept beside the run of the law alone, the two are two sets, so one folder holds both
and a change is judged on each."""
monkeypatch.chdir(REPO)
monkeypatch.setenv("DPOLENS_DATABASE_URL", migrated)
runner = CliRunner()
asked_for = ["--pack", "uae-pdpl", "--set", "tuning.en", "--config", KEYWORD]
kept_in = tmp_path / "kept"
alone, beside = kept_in / "alone.json", kept_in / "beside.json"

ran_alone = runner.invoke(app, ["evals", "run", *asked_for, "--outcomes", str(alone)])
ran = runner.invoke(app, ["evals", "run", *asked_for, "--every-law", "--outcomes", str(beside)])
compared = runner.invoke(app, ["evals", "compare", str(kept_in), str(kept_in)])

assert (ran_alone.exit_code, ran.exit_code) == (0, 0), ran.output
assert "uae-pdpl tuning.en beside gdpr: 58 questions" in ran.output
kept = json.loads(beside.read_text(encoding="utf-8"))
assert [asked["question_set"] for asked in kept["sets"]] == ["tuning.en beside gdpr"]
assert compared.exit_code == 0, compared.output
assert "uae-pdpl tuning.en: 58 questions" in compared.output
assert "uae-pdpl tuning.en beside gdpr: 58 questions" in compared.output
Loading
Loading