diff --git a/evals/README.md b/evals/README.md index 274ded5..a75ab82 100644 --- a/evals/README.md +++ b/evals/README.md @@ -138,6 +138,12 @@ means that law. Keyword statistics are kept across every loaded law, so a run on holding other laws can differ slightly from one holding that law alone. The published scores are measured with every pack loaded, and so is CI's gate. +An organisation that selects several laws searches them together. `--every-law` measures a +law that way: the run searches every law loaded, the pack's questions still expect the pack's +own clauses, and the set is named with the laws beside it, such as `tuning.ar beside uae-pdpl`. +The laws then share the ten results, so `recall@10` is the figure to read. Such a run is never +written to `evals/results`: the published scores are of a law searched alone. + ## Judging a change A change to search is judged on the sets used while improving retrieval, tuning and @@ -166,6 +172,10 @@ uv run dpolens evals run --pack uae-pdpl --outcomes after/uae-pdpl.json uv run dpolens evals compare before after ``` +A change to how several laws share the results is judged with the same runs made again with +`--every-law`, kept in the same folders under another file name. A law alone and a law beside +the others are two sets, and a topic asked in both is still one topic. + `--outcomes` writes, for every question of the configuration being scored, the clauses returned and the place of the first that answers. `compare` takes two such files, or two folders with a file for each law, and prints each set before and after, then whether the change is kept. @@ -176,6 +186,7 @@ file for each law, and prints each set before and after, then whether the change uv run dpolens evals run --pack gdpr --set tuning uv run dpolens evals run --pack gdpr --set tuning.ar uv run dpolens evals run --pack gdpr --set assistant +uv run dpolens evals run --pack gdpr --set tuning --every-law uv run dpolens evals run --pack gdpr --set held_out --baselines --write uv run dpolens evals run --pack uae-pdpl --set held_out --together --baselines --write ``` diff --git a/packages/dpolens/src/dpolens/cli/evals.py b/packages/dpolens/src/dpolens/cli/evals.py index b648dd0..dd2785a 100644 --- a/packages/dpolens/src/dpolens/cli/evals.py +++ b/packages/dpolens/src/dpolens/cli/evals.py @@ -88,6 +88,16 @@ def run( bool, typer.Option(help="Also score the sets as one, a topic asked in each counted once"), ] = False, + every_law: Annotated[ + bool, + typer.Option( + "--every-law", + help=( + "Search every law loaded here, not the pack's alone, as an organisation " + "that selected them all does" + ), + ), + ] = False, write: Annotated[bool, typer.Option(help="Write evals/results/.json")] = False, outcomes: Annotated[ Path | None, @@ -125,6 +135,11 @@ def run( "so a number measured on a set used for tuning does not belong in it.", param_hint="--set", ) + if write and every_law: + raise typer.BadParameter( + "--write is for a law searched alone, which is what the published scores measure", + param_hint="--every-law", + ) questions = {name: read_questions(EVALS / pack / f"{name}.yaml") for name in sets} with ( @@ -132,7 +147,15 @@ def run( session_scope(load_settings()) as session, ): runs = [ - run_set(session, embedder, questions[name], config, pack=pack, question_set=name) + run_set( + session, + embedder, + questions[name], + config, + pack=pack, + question_set=name, + every_law=every_law, + ) for name in sets for config in configurations ] @@ -155,7 +178,7 @@ def run( for name in dict.fromkeys(result.question_set for result in runs): block = [result for result in runs if result.question_set == name] - typer.echo(f"{pack} {name}: {block[0].score.questions} questions, model {model}\n") + typer.echo(f"{pack} {block[0].name}: {block[0].score.questions} questions, model {model}\n") for result in block: typer.echo( f" {result.configuration:12} recall@5 {result.score.recall_at_5:.2f} " @@ -257,7 +280,7 @@ def _keep(pack: str, runs: list[Run], path: Path) -> None: "context_recipe": runs[0].recipe, "sets": [ { - "question_set": result.question_set, + "question_set": result.name, "questions": [ { "id": outcome.question_id, @@ -280,7 +303,7 @@ def _keep(pack: str, runs: list[Run], path: Path) -> None: def _as_record(result: Run) -> dict[str, Any]: return { - "question_set": result.question_set, + "question_set": result.name, "configuration": result.configuration, "model": result.model, "context_recipe": result.recipe, diff --git a/packages/dpolens/src/dpolens/cli/main.py b/packages/dpolens/src/dpolens/cli/main.py index 08bd0ef..639a528 100644 --- a/packages/dpolens/src/dpolens/cli/main.py +++ b/packages/dpolens/src/dpolens/cli/main.py @@ -57,6 +57,7 @@ EmptySelection, NothingSelected, NotSelected, + OnlyLaw, UnknownLaw, ) from dpolens.engine.policies.blocks import UnreadablePolicy @@ -108,6 +109,7 @@ UnknownToken, SurfaceOwnerIsAPerson, UnknownLaw, + OnlyLaw, NotSelected, NothingSelected, WeakPassword, diff --git a/packages/dpolens/src/dpolens/engine/evals/run.py b/packages/dpolens/src/dpolens/engine/evals/run.py index 936c4df..000ec55 100644 --- a/packages/dpolens/src/dpolens/engine/evals/run.py +++ b/packages/dpolens/src/dpolens/engine/evals/run.py @@ -20,7 +20,7 @@ from dpolens.engine.embedding.encode import Embedder from dpolens.engine.embedding.recipe import RECIPE_VERSION from dpolens.engine.evals.score import Outcome, Score, score -from dpolens.engine.packs.selection import Scope, law_alone +from dpolens.engine.packs.selection import Scope, law_alone, law_beside_the_others from dpolens.engine.search.engine import CANDIDATE_DEPTH, search from dpolens.engine.search.fuse import Fused, Fusion, fuse from dpolens.engine.search.keyword import ALTERNATIVES, Candidate, KeywordIndex, keyword_search @@ -56,6 +56,15 @@ class Run: recipe: str score: Score outcomes: tuple[Outcome, ...] + beside: tuple[str, ...] = () + """The other laws searched with the pack's own, in a run that measures it beside them.""" + + @property + def name(self) -> str: + """The set as a reader names it: tuning.ar, or tuning.ar beside uae-pdpl.""" + if not self.beside: + return self.question_set + return f"{self.question_set} beside {', '.join(self.beside)}" def read_questions(path: Path) -> list[Question]: @@ -112,6 +121,7 @@ def run_set( pack: str = "", question_set: str = "", reranker: Reranker | None = None, + every_law: bool = False, ) -> Run: """Score one configuration over one question set. @@ -125,9 +135,19 @@ def run_set( A run for a pack searches that pack's law alone, whatever else is loaded or selected, so its score means that law. Without a pack it searches what any search would. + + With `every_law`, a run for a pack searches every loaded law, as an organisation + that selected them all does. Its questions still expect the pack's own clauses, + so the score says how well that law is found beside the others. """ check_configuration(configuration, reranker) - scope = law_alone(session, pack) if pack else None + if every_law and not pack: + raise ValueError("a run beside every law is a run for a pack") + if every_law: + scope: Scope | None = law_beside_the_others(session, pack) + else: + scope = law_alone(session, pack) if pack else None + beside = tuple(law for law in scope.laws if law != pack) if scope and every_law else () outcomes = [] for question in questions: returned = _retrieve(session, embedder, question, configuration, reranker, scope) @@ -148,6 +168,7 @@ def run_set( recipe=RECIPE_VERSION, score=score(outcomes, group_of=_item_of(questions)), outcomes=tuple(outcomes), + beside=beside, ) @@ -182,10 +203,14 @@ def run_together(runs: Sequence[Run]) -> Run: """ first = runs[0] if any( - (run.pack, run.configuration, run.model) != (first.pack, first.configuration, first.model) + (run.pack, run.configuration, run.model, run.beside) + != (first.pack, first.configuration, first.model, first.beside) for run in runs ): - raise ValueError("only runs of one pack, configuration and model are scored together") + raise ValueError( + "only runs of one pack, configuration and model, searched beside the same laws, " + "are scored together" + ) outcomes = tuple( Outcome( @@ -204,6 +229,7 @@ def run_together(runs: Sequence[Run]) -> Run: recipe=first.recipe, score=score(outcomes, group_of=lambda outcome: outcome.question_id.split("/", 1)[1]), outcomes=outcomes, + beside=first.beside, ) diff --git a/packages/dpolens/src/dpolens/engine/packs/selection.py b/packages/dpolens/src/dpolens/engine/packs/selection.py index 63514f3..ed02adc 100644 --- a/packages/dpolens/src/dpolens/engine/packs/selection.py +++ b/packages/dpolens/src/dpolens/engine/packs/selection.py @@ -68,6 +68,16 @@ class EmptySelection(Exception): """A selection names at least one law.""" +class OnlyLaw(Exception): + """A law cannot be searched beside the others where no other is loaded.""" + + def __init__(self, slug: str) -> None: + self.slug = slug + super().__init__( + f"{slug} is the only law loaded here, so there is no other to search it beside" + ) + + @dataclass(frozen=True) class Scope: """The laws one search covers, against the laws loaded here.""" @@ -148,6 +158,18 @@ def law_alone(session: Session, slug: str) -> Scope: return Scope(laws=(slug,), loaded=loaded) +def law_beside_the_others(session: Session, slug: str) -> Scope: + """One loaded law with every other loaded law beside it, whatever is selected, which is + what an eval run searches to measure a law as an organisation that chose several finds + it.""" + loaded = tuple(pack.slug for pack in loaded_laws(session)) + if slug not in loaded: + raise UnknownLaw([slug], loaded) + if len(loaded) == 1: + raise OnlyLaw(slug) + return Scope(laws=loaded, loaded=loaded) + + def newly_arrived(session: Session, *, now: datetime | None = None) -> frozenset[str]: """The laws loaded after the laws were first chosen, within the last 30 days. diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index 6d22247..be19d23 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -239,6 +239,15 @@ def test_only_a_held_out_set_is_written_and_before_anything_runs(in_the_checkout assert "--write is for held-out sets, not 'tuning.ar'" in said(result) +def test_a_run_beside_every_law_is_never_written(in_the_checkout: None) -> None: + result = runner.invoke( + app, ["evals", "run", "--pack", "gdpr", "--set", "held_out", "--every-law", "--write"] + ) + + assert result.exit_code == 2 + assert "--write is for a law searched alone" in said(result) + + def test_scoring_together_needs_two_sets(in_the_checkout: None) -> None: result = runner.invoke( app, ["evals", "run", "--pack", "gdpr", "--set", "held_out.en", "--together"] diff --git a/tests/integration/test_evals.py b/tests/integration/test_evals.py index b98ecd1..855f3bf 100644 --- a/tests/integration/test_evals.py +++ b/tests/integration/test_evals.py @@ -227,3 +227,16 @@ def test_a_set_asked_in_parts_keeps_each_parts_topic_and_counts_whole_queries( ] assert f"gdpr assistant.en: {len(questions)} questions" in ran.output assert f"of {queries} queries in the top five" in " ".join(ran.output.split()) + + +def test_a_law_loaded_alone_has_no_other_to_be_searched_beside( + indexed: Session, migrated: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(REPO) + monkeypatch.setenv("DPOLENS_DATABASE_URL", migrated) + + asked_for = ["--pack", "gdpr", "--set", "tuning.en", "--config", "keyword", "--every-law"] + ran = CliRunner().invoke(app, ["evals", "run", *asked_for]) + + assert ran.exit_code == 1 + assert "gdpr is the only law loaded here" in ran.output diff --git a/tests/integration/test_evals_every_law.py b/tests/integration/test_evals_every_law.py new file mode 100644 index 0000000..51223ac --- /dev/null +++ b/tests/integration/test_evals_every_law.py @@ -0,0 +1,107 @@ +"""A law measured beside the others, as an organisation that selected several searches. + +Both real packs are loaded and nothing is embedded: keyword search needs no vectors, and it +is enough to see which laws a run reaches. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Iterator +from pathlib import Path + +import pytest +from sqlalchemy import Engine +from sqlalchemy.orm import Session +from typer.testing import CliRunner + +from dpolens.cli.main import app +from dpolens.engine.embedding import Embedder, get_model +from dpolens.engine.evals import read_questions, run_set +from dpolens.engine.packs.load import load_pack + +pytestmark = pytest.mark.integration + +REPO = Path(__file__).parents[2] +TUNING = REPO / "evals" / "uae-pdpl" / "tuning.en.yaml" +KEYWORD = "keyword+normative" + + +@pytest.fixture(scope="session") +def embedder() -> Iterator[Embedder]: + with Embedder(get_model("e5-small")) as loaded: + yield loaded + + +@pytest.fixture(scope="module") +def both_laws(engine: Engine, truncate_corpus: Callable[[], None]) -> Iterator[Session]: + with Session(engine) as prepared: + load_pack(prepared, REPO / "packs" / "gdpr") + load_pack(prepared, REPO / "packs" / "uae-pdpl") + prepared.commit() + yield prepared + + truncate_corpus() + + +def laws_reached(returned: tuple[str, ...]) -> set[str]: + return {key.split(":", 1)[0] for key in returned} + + +def test_a_run_for_a_pack_reaches_its_law_alone(both_laws: Session, embedder: Embedder) -> None: + run = run_set(both_laws, embedder, read_questions(TUNING), KEYWORD, pack="uae-pdpl") + + assert run.beside == () + assert {law for outcome in run.outcomes for law in laws_reached(outcome.returned)} == { + "uae-pdpl" + } + + +def test_beside_every_law_a_run_reaches_them_all_and_says_which( + both_laws: Session, embedder: Embedder +) -> None: + questions = read_questions(TUNING) + + run = run_set( + both_laws, + embedder, + questions, + KEYWORD, + pack="uae-pdpl", + question_set="tuning.en", + every_law=True, + ) + + assert (run.beside, run.name) == (("gdpr",), "tuning.en beside gdpr") + assert {law for outcome in run.outcomes for law in laws_reached(outcome.returned)} == { + "gdpr", + "uae-pdpl", + } + assert [outcome.expected for outcome in run.outcomes] == [ + question.expected_keys for question in questions + ] + + +def test_the_command_keeps_such_a_run_as_a_set_of_its_own( + both_laws: Session, migrated: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Kept beside the run of the law alone, the two are two sets, so one folder holds both + and a change is judged on each.""" + monkeypatch.chdir(REPO) + monkeypatch.setenv("DPOLENS_DATABASE_URL", migrated) + runner = CliRunner() + asked_for = ["--pack", "uae-pdpl", "--set", "tuning.en", "--config", KEYWORD] + kept_in = tmp_path / "kept" + alone, beside = kept_in / "alone.json", kept_in / "beside.json" + + ran_alone = runner.invoke(app, ["evals", "run", *asked_for, "--outcomes", str(alone)]) + ran = runner.invoke(app, ["evals", "run", *asked_for, "--every-law", "--outcomes", str(beside)]) + compared = runner.invoke(app, ["evals", "compare", str(kept_in), str(kept_in)]) + + assert (ran_alone.exit_code, ran.exit_code) == (0, 0), ran.output + assert "uae-pdpl tuning.en beside gdpr: 58 questions" in ran.output + kept = json.loads(beside.read_text(encoding="utf-8")) + assert [asked["question_set"] for asked in kept["sets"]] == ["tuning.en beside gdpr"] + assert compared.exit_code == 0, compared.output + assert "uae-pdpl tuning.en: 58 questions" in compared.output + assert "uae-pdpl tuning.en beside gdpr: 58 questions" in compared.output diff --git a/tests/unit/test_scoring_together.py b/tests/unit/test_scoring_together.py index 7cc384d..0450937 100644 --- a/tests/unit/test_scoring_together.py +++ b/tests/unit/test_scoring_together.py @@ -20,7 +20,12 @@ def outcome(question_id: str, hit: bool) -> Outcome: ) -def a_run(question_set: str, hits: list[bool], configuration: str = "convex:0.5") -> Run: +def a_run( + question_set: str, + hits: list[bool], + configuration: str = "convex:0.5", + beside: tuple[str, ...] = (), +) -> Run: outcomes = tuple(outcome(f"topic-{number}", hit) for number, hit in enumerate(hits)) return Run( pack="law", @@ -30,6 +35,7 @@ def a_run(question_set: str, hits: list[bool], configuration: str = "convex:0.5" recipe="v1", score=score(outcomes), outcomes=outcomes, + beside=beside, ) @@ -64,3 +70,23 @@ def test_the_sets_are_named_and_each_miss_says_which_set_it_is_from() -> None: def test_runs_of_different_configurations_are_not_scored_together() -> None: with pytest.raises(ValueError, match="one pack, configuration and model"): run_together([a_run("held_out.ar", HITS), a_run("held_out.en", HITS, "rrf:10")]) + + +def test_a_run_beside_other_laws_is_named_with_them() -> None: + assert a_run("tuning.ar", HITS).name == "tuning.ar" + assert a_run("tuning.ar", HITS, beside=("gdpr", "sa-pdpl")).name == ( + "tuning.ar beside gdpr, sa-pdpl" + ) + + +def test_sets_scored_together_beside_other_laws_keep_saying_which() -> None: + together = run_together( + [a_run("tuning.ar", HITS, beside=("gdpr",)), a_run("tuning.en", HITS, beside=("gdpr",))] + ) + + assert together.name == "tuning.ar+tuning.en beside gdpr" + + +def test_a_law_alone_and_beside_others_are_not_scored_together() -> None: + with pytest.raises(ValueError, match="searched beside the same laws"): + run_together([a_run("tuning.ar", HITS), a_run("tuning.en", HITS, beside=("gdpr",))])