diff --git a/evals/README.md b/evals/README.md index c7c6516..75c942b 100644 --- a/evals/README.md +++ b/evals/README.md @@ -88,6 +88,38 @@ means that law. Keyword statistics are kept across every loaded law, so a run on holding other laws can differ slightly from one holding that law alone. The published scores are measured with every pack loaded, and so is CI's gate. +## Judging a change + +A change to search is judged on the tuning sets, by a rule fixed before the change is +measured. Each question scores by the place of the first clause that answers it among the ten +returned: 1 for first place, a half for second, down to a tenth, and nothing when none answers. +A count of hits at five would not see a clause climb from ninth to sixth, or fall from first to +fifth. + +Two runs are compared question by question. A topic asked more than once, in another language +or the way people type in chat, is one topic: its questions move together, so their changes are +counted together. The change is kept when both of these hold: + +- **The gain is more than chance gives.** Each topic's change is given a random sign, ten + thousand times, and the measured gain has to be larger than 95% of those. +- **No set is worse.** A set is worse when its score falls by more than chance explains, by the + same test on that set alone, with the 5% shared between the sets. + +The held-out sets take no part in it. They are measured for the published score. + +```bash +uv run dpolens evals run --pack gdpr --outcomes before/gdpr.json +uv run dpolens evals run --pack uae-pdpl --outcomes before/uae-pdpl.json +# with the change in place, and whatever it needs rebuilt: +uv run dpolens evals run --pack gdpr --outcomes after/gdpr.json +uv run dpolens evals run --pack uae-pdpl --outcomes after/uae-pdpl.json +uv run dpolens evals compare before after +``` + +`--outcomes` writes, for every question of the configuration being scored, the clauses returned +and the place of the first that answers. `compare` takes two such files, or two folders with a +file for each law, and prints each set before and after, then whether the change is kept. + ## Running them ```bash diff --git a/packages/dpolens/src/dpolens/cli/evals.py b/packages/dpolens/src/dpolens/cli/evals.py index c6bcf22..acf1457 100644 --- a/packages/dpolens/src/dpolens/cli/evals.py +++ b/packages/dpolens/src/dpolens/cli/evals.py @@ -14,12 +14,18 @@ from dpolens.engine.embedding import DEFAULT_MODEL, Embedder, get_model from dpolens.engine.evals import ( EvalSetError, + Judgement, + Placed, Run, + Unpaired, check_configuration, + judge, + placed_in, read_questions, run_set, run_together, sets_named, + topic_of, ) from dpolens.engine.session import session_scope from dpolens.settings import load_settings @@ -82,6 +88,15 @@ def run( typer.Option(help="Also score the sets as one, a topic asked in each counted once"), ] = False, write: Annotated[bool, typer.Option(help="Write evals/results/.json")] = False, + outcomes: Annotated[ + Path | None, + typer.Option( + help=( + "Write where each question's right clause came to this file, for " + "`dpolens evals compare`" + ) + ), + ] = None, as_json: Annotated[ bool, typer.Option("--json", help="Print one run as JSON: the sets together, or the first"), @@ -128,6 +143,8 @@ def run( if together else [] ) + if outcomes is not None: + _keep(pack, [result for result in runs if result.configuration == configuration], outcomes) runs += combined if as_json: @@ -152,6 +169,106 @@ def run( _write(pack, runs) +@app.command() +def compare( + before: Annotated[ + Path, + typer.Argument(help="An outcomes file from before a change, or a folder of them"), + ], + after: Annotated[Path, typer.Argument(help="The same, from after it")], +) -> None: + """Judge a change to search from two runs over the same questions.""" + try: + judgement = judge(_placed(before), _placed(after)) + except Unpaired as problem: + raise typer.BadParameter(str(problem)) from problem + + for change in judgement.sets: + typer.echo( + f"{change.pack} {change.question_set}: {change.questions} questions " + f"hits at five {change.hits_before} to {change.hits_after} " + f"place score {change.score_before:.3f} to {change.score_after:.3f}" + ) + typer.echo( + f"\nAll: {judgement.questions} questions on {judgement.topics} topics " + f"hits at five {judgement.hits_before} to {judgement.hits_after} " + f"place score {judgement.score_before:.3f} to {judgement.score_after:.3f}" + ) + typer.echo(f"By topic at five: {judgement.topics_found} found, {judgement.topics_lost} lost\n") + typer.echo(_verdict(judgement)) + + +def _verdict(judgement: Judgement) -> str: + said = [] + if judgement.score_after <= judgement.score_before: + said.append("No gain: the right clauses sit no higher than before.") + else: + said.append( + f"Chance alone gives a gain this large {_share(judgement.chance)} of the time, " + "and a change is kept under 5%." + ) + for change in judgement.sets: + if change.worse: + said.append( + f"{change.pack} {change.question_set} is worse: chance alone gives it a loss " + f"this large {_share(change.chance_of_loss)} of the time." + ) + said.append("Kept." if judgement.kept else "Not kept.") + return "\n".join(said) + + +def _share(chance: float) -> str: + return f"{100 * chance:.2f}%" + + +def _placed(path: Path) -> list[Placed]: + """Every question in an outcomes file, or in each file of a folder.""" + files = sorted(path.glob("*.json")) if path.is_dir() else [path] + if not files: + raise typer.BadParameter(f"{path} holds no outcomes file") + placed: list[Placed] = [] + for file in files: + try: + placed += placed_in(json.loads(file.read_text(encoding="utf-8"))) + except (OSError, ValueError, KeyError, TypeError) as problem: + raise typer.BadParameter( + f"{file} is not an outcomes file, as `dpolens evals run --outcomes` writes one" + ) from problem + return placed + + +def _keep(pack: str, runs: list[Run], path: Path) -> None: + """The configuration scored, with where each question's right clause came.""" + payload = { + "pack": pack, + "measured_at": datetime.now(UTC).date().isoformat(), + "commit": _commit(), + "configuration": runs[0].configuration, + "model": runs[0].model, + "context_recipe": runs[0].recipe, + "sets": [ + { + "question_set": result.question_set, + "questions": [ + { + "id": outcome.question_id, + "topic": topic_of(outcome.question_id), + "expected": list(outcome.expected), + "returned": list(outcome.returned), + "place": outcome.first_hit, + } + for outcome in result.outcomes + ], + } + for result in runs + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + # Not to standard output, which `--json` keeps to the one record a script reads. + typer.echo(f"Wrote {path}", err=True) + + def _as_record(result: Run) -> dict[str, Any]: return { "question_set": result.question_set, diff --git a/packages/dpolens/src/dpolens/engine/evals/__init__.py b/packages/dpolens/src/dpolens/engine/evals/__init__.py index c0f3776..e3af926 100644 --- a/packages/dpolens/src/dpolens/engine/evals/__init__.py +++ b/packages/dpolens/src/dpolens/engine/evals/__init__.py @@ -1,6 +1,15 @@ """Measuring whether retrieval finds the right clause.""" from dpolens.engine.evals.gate import Verdict, compare +from dpolens.engine.evals.judge import ( + Judgement, + Placed, + SetChange, + Unpaired, + judge, + placed_in, + topic_of, +) from dpolens.engine.evals.run import ( EvalSetError, Question, @@ -23,18 +32,25 @@ __all__ = [ "EvalSetError", "Interval", + "Judgement", "Outcome", + "Placed", "Question", "Run", "Score", + "SetChange", + "Unpaired", "Verdict", "bootstrap_interval", "check_configuration", "compare", "grouped_interval", + "judge", + "placed_in", "read_questions", "recall_at", "run_set", "run_together", "sets_named", + "topic_of", ] diff --git a/packages/dpolens/src/dpolens/engine/evals/judge.py b/packages/dpolens/src/dpolens/engine/evals/judge.py new file mode 100644 index 0000000..a78b4b8 --- /dev/null +++ b/packages/dpolens/src/dpolens/engine/evals/judge.py @@ -0,0 +1,198 @@ +"""Judging whether a change to search made it better. + +Two runs over the same questions are compared question by question. What counts is how far +up the right clause sits, not only whether it reached the top five: a clause that climbs +from ninth to sixth has moved, and a count of hits at five does not see it. +""" + +from __future__ import annotations + +import random +from collections import defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +CHANCE = 0.05 +"""A gain counts when chance alone would give one as large less often than this.""" + +SHUFFLES = 10_000 +"""How many times each topic's change is given a random sign.""" + +DEPTH = 10 +"""How many results a question is scored over, which is what an eval run returns.""" + +HIT = 5 + + +class Unpaired(Exception): + """The two runs did not ask the same questions, so there is nothing to compare.""" + + +@dataclass(frozen=True) +class Placed: + """Where the first clause that answers one question came, in one run.""" + + pack: str + question_set: str + question_id: str + topic: str + place: int | None + """Counting from one, or None when no returned clause answers.""" + + @property + def score(self) -> float: + """One for first place, a half for second, down to a tenth, and nothing below.""" + return 1.0 / self.place if self.place is not None and self.place <= DEPTH else 0.0 + + @property + def hit(self) -> bool: + return self.place is not None and self.place <= HIT + + +@dataclass(frozen=True) +class SetChange: + """One question set, before and after.""" + + pack: str + question_set: str + questions: int + hits_before: int + hits_after: int + score_before: float + score_after: float + chance_of_loss: float + """How often chance alone gives this set a loss as large as the one it shows.""" + worse: bool + + +@dataclass(frozen=True) +class Judgement: + questions: int + topics: int + hits_before: int + hits_after: int + score_before: float + score_after: float + topics_found: int + topics_lost: int + chance: float + """How often chance alone gives a gain as large as the one measured.""" + sets: tuple[SetChange, ...] + + @property + def kept(self) -> bool: + return self.chance < CHANCE and not any(change.worse for change in self.sets) + + +def topic_of(question_id: str) -> str: + """The topic a question asks about. A set that asks a topic twice, formally and the way + people type in chat, gives the second question the first's id with -chat after it.""" + return question_id.removesuffix("-chat") + + +def placed_in(record: Mapping[str, Any]) -> list[Placed]: + """Every question of an outcomes file, as `dpolens evals run --outcomes` writes it.""" + return [ + Placed( + pack=str(record["pack"]), + question_set=str(asked["question_set"]), + question_id=str(question["id"]), + topic=str(question["topic"]), + place=question["place"], + ) + for asked in record["sets"] + for question in asked["questions"] + ] + + +def chance_of_gain(changes: Sequence[float], seed: int = 0) -> float: + """How often giving each change a random sign adds up to at least what they add up to. + + If a change to search did nothing, a topic would be as likely to move down as up, so + each topic's change could as well have had the other sign. The seed is fixed, so judging + the same two runs twice gives the same answer. + """ + measured = sum(changes) + if measured <= 0: + return 1.0 + chooser = random.Random(seed) + at_least = sum( + sum(change if chooser.random() < 0.5 else -change for change in changes) >= measured - 1e-12 + for _ in range(SHUFFLES) + ) + return (at_least + 1) / (SHUFFLES + 1) + + +def judge(before: Sequence[Placed], after: Sequence[Placed]) -> Judgement: + """Compare two runs over the same questions. + + A topic asked more than once, in another language or as chat, is one topic: its + questions move together, so their changes are added before any sign is shuffled. The + change is kept when its gain is more than chance gives and no set is worse. A set is + worse when its score falls by more than chance explains, by the same test on that set + alone, with the chance allowed shared across the sets so that looking at many does not + find a loss in one by accident. + """ + earlier = {(placed.pack, placed.question_set, placed.question_id): placed for placed in before} + later = {(placed.pack, placed.question_set, placed.question_id): placed for placed in after} + if len(earlier) != len(before) or len(later) != len(after): + raise Unpaired("a run holds a question twice, as two files for one law would") + if earlier.keys() != later.keys(): + unmatched = sorted(earlier.keys() ^ later.keys()) + named = ", ".join("/".join(key) for key in unmatched[:3]) + raise Unpaired( + f"the two runs do not ask the same questions: {len(unmatched)} in one run only, " + f"such as {named}" + ) + if not earlier: + raise Unpaired("the runs hold no questions") + + by_topic: dict[tuple[str, str], float] = defaultdict(float) + hits_by_topic: dict[tuple[str, str], list[int]] = defaultdict(lambda: [0, 0]) + by_set: dict[tuple[str, str], dict[str, float]] = defaultdict(lambda: defaultdict(float)) + for key, was in earlier.items(): + now = later[key] + change = now.score - was.score + by_topic[(was.pack, was.topic)] += change + by_set[(was.pack, was.question_set)][was.topic] += change + hits_by_topic[(was.pack, was.topic)][0] += was.hit + hits_by_topic[(was.pack, was.topic)][1] += now.hit + + sets = [] + for (pack, question_set), changes in by_set.items(): + was_asked = [ + placed + for placed in before + if (placed.pack, placed.question_set) == (pack, question_set) + ] + now_asked = [later[(p.pack, p.question_set, p.question_id)] for p in was_asked] + score_before = sum(placed.score for placed in was_asked) / len(was_asked) + score_after = sum(placed.score for placed in now_asked) / len(now_asked) + chance_of_loss = chance_of_gain([-change for change in changes.values()]) + sets.append( + SetChange( + pack=pack, + question_set=question_set, + questions=len(was_asked), + hits_before=sum(placed.hit for placed in was_asked), + hits_after=sum(placed.hit for placed in now_asked), + score_before=score_before, + score_after=score_after, + chance_of_loss=chance_of_loss, + worse=score_after < score_before and chance_of_loss < CHANCE / len(by_set), + ) + ) + + return Judgement( + questions=len(earlier), + topics=len(by_topic), + hits_before=sum(placed.hit for placed in earlier.values()), + hits_after=sum(placed.hit for placed in later.values()), + score_before=sum(placed.score for placed in earlier.values()) / len(earlier), + score_after=sum(placed.score for placed in later.values()) / len(later), + topics_found=sum(1 for was, now in hits_by_topic.values() if now > was), + topics_lost=sum(1 for was, now in hits_by_topic.values() if now < was), + chance=chance_of_gain(list(by_topic.values())), + sets=tuple(sets), + ) diff --git a/tests/integration/test_evals.py b/tests/integration/test_evals.py index 337357f..c3ea6d1 100644 --- a/tests/integration/test_evals.py +++ b/tests/integration/test_evals.py @@ -2,16 +2,20 @@ from __future__ import annotations +import json from collections.abc import Callable, Iterator from pathlib import Path import pytest from sqlalchemy import Engine, text from sqlalchemy.orm import Session +from typer.testing import CliRunner +from dpolens.cli.main import app from dpolens.engine.embedding import Embedder, get_model from dpolens.engine.embedding.index import build_index from dpolens.engine.evals import EvalSetError, read_questions, run_set +from dpolens.engine.evals.score import answers from dpolens.engine.packs.load import load_pack pytestmark = pytest.mark.integration @@ -132,3 +136,36 @@ def test_excluding_explaining_text_is_measurable(indexed: Session, embedder: Emb normative = run_set(indexed, embedder, questions, "meaning+normative") assert normative.score.recall_at_5 >= everything.score.recall_at_5 + + +def test_a_run_keeps_where_each_questions_right_clause_came( + indexed: Session, migrated: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Written by the command and read back: every question, the clauses returned for it, + and the place of the first that answers, which is what comparing two runs needs.""" + monkeypatch.chdir(REPO) + monkeypatch.setenv("DPOLENS_DATABASE_URL", migrated) + kept = tmp_path / "before" / "gdpr.json" + runner = CliRunner() + + asked_for = ["--pack", "gdpr", "--set", "tuning.en", "--config", "meaning+normative"] + ran = runner.invoke(app, ["evals", "run", *asked_for, "--outcomes", str(kept)]) + compared = runner.invoke(app, ["evals", "compare", str(kept), str(kept)]) + + assert ran.exit_code == 0, ran.output + record = json.loads(kept.read_text(encoding="utf-8")) + asked = record["sets"][0]["questions"] + assert (record["pack"], record["configuration"]) == ("gdpr", "meaning+normative") + assert record["sets"][0]["question_set"] == "tuning.en" + assert [question["id"] for question in asked] == [q.id for q in read_questions(TUNING)] + for question in asked: + assert len(question["returned"]) == 10 + answered = [ + place + for place, key in enumerate(question["returned"], start=1) + if any(answers(key, expected) for expected in question["expected"]) + ] + assert question["place"] == (answered[0] if answered else None) + assert any(question["place"] for question in asked) + assert compared.exit_code == 0, compared.output + assert "Not kept." in compared.output diff --git a/tests/unit/test_evals_compare.py b/tests/unit/test_evals_compare.py new file mode 100644 index 0000000..e4f6a04 --- /dev/null +++ b/tests/unit/test_evals_compare.py @@ -0,0 +1,101 @@ +"""`dpolens evals compare`, on outcomes files written by hand: it reads no database.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import click +from typer.testing import CliRunner, Result + +from dpolens.cli.main import app + +runner = CliRunner() +BORDERS = str.maketrans(dict.fromkeys("│╭╮╰╯─", " ")) + + +def said(result: Result) -> str: + """The output on one line, without the colour and borders CI draws an error in.""" + return " ".join(click.unstyle(result.output).translate(BORDERS).split()) + + +def outcomes(path: Path, pack: str, places: dict[str, int | None]) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "pack": pack, + "sets": [ + { + "question_set": "tuning.en", + "questions": [ + {"id": question, "topic": question, "place": place} + for question, place in places.items() + ], + } + ], + } + ), + encoding="utf-8", + ) + return path + + +def all_at(place: int | None) -> dict[str, int | None]: + return {f"topic-{number}": place for number in range(20)} + + +def test_a_clear_gain_is_kept(tmp_path: Path) -> None: + before = outcomes(tmp_path / "before.json", "law", all_at(None)) + after = outcomes(tmp_path / "after.json", "law", all_at(1)) + + result = runner.invoke(app, ["evals", "compare", str(before), str(after)]) + + assert result.exit_code == 0, result.output + assert "law tuning.en: 20 questions hits at five 0 to 20" in said(result) + assert "By topic at five: 20 found, 0 lost" in said(result) + assert said(result).endswith("Kept.") + + +def test_the_same_run_twice_is_no_gain(tmp_path: Path) -> None: + before = outcomes(tmp_path / "before.json", "law", all_at(3)) + + result = runner.invoke(app, ["evals", "compare", str(before), str(before)]) + + assert result.exit_code == 0, result.output + assert "No gain: the right clauses sit no higher than before." in said(result) + assert said(result).endswith("Not kept.") + + +def test_a_folder_holds_one_file_for_each_law(tmp_path: Path) -> None: + outcomes(tmp_path / "before" / "first.json", "first", all_at(None)) + outcomes(tmp_path / "before" / "second.json", "second", all_at(None)) + outcomes(tmp_path / "after" / "first.json", "first", all_at(1)) + outcomes(tmp_path / "after" / "second.json", "second", all_at(2)) + + result = runner.invoke( + app, ["evals", "compare", str(tmp_path / "before"), str(tmp_path / "after")] + ) + + assert result.exit_code == 0, result.output + assert "All: 40 questions on 40 topics hits at five 0 to 40" in said(result) + + +def test_runs_over_different_questions_are_refused(tmp_path: Path) -> None: + before = outcomes(tmp_path / "before.json", "law", {"a": 1, "b": 2}) + after = outcomes(tmp_path / "after.json", "law", {"a": 1}) + + result = runner.invoke(app, ["evals", "compare", str(before), str(after)]) + + assert result.exit_code == 2 + assert "the two runs do not ask the same questions" in said(result) + + +def test_a_file_that_is_not_an_outcomes_file_is_named(tmp_path: Path) -> None: + other = tmp_path / "results.json" + other.write_text('{"pack": "law", "runs": []}', encoding="utf-8") + + result = runner.invoke(app, ["evals", "compare", str(other), str(other)]) + + assert result.exit_code == 2 + assert "results.json is not an outcomes file" in said(result) diff --git a/tests/unit/test_judge.py b/tests/unit/test_judge.py new file mode 100644 index 0000000..2b3e804 --- /dev/null +++ b/tests/unit/test_judge.py @@ -0,0 +1,173 @@ +"""Judging a change to search. + +The arithmetic behind "kept" and "not kept", on hand-built runs, so it does not depend on +what search happens to return today. +""" + +from __future__ import annotations + +import pytest + +from dpolens.engine.evals.judge import ( + Placed, + Unpaired, + chance_of_gain, + judge, + placed_in, + topic_of, +) + + +def run( + places: dict[str, int | None], question_set: str = "tuning.en", pack: str = "law" +) -> list[Placed]: + return [ + Placed(pack, question_set, question_id, topic_of(question_id), place) + for question_id, place in places.items() + ] + + +def topics(count: int, place: int | None) -> dict[str, int | None]: + return {f"topic-{number}": place for number in range(count)} + + +def test_a_place_scores_by_how_high_it_is() -> None: + assert Placed("law", "tuning.en", "a", "a", 1).score == 1.0 + assert Placed("law", "tuning.en", "a", "a", 4).score == 0.25 + assert Placed("law", "tuning.en", "a", "a", 10).score == 0.1 + assert Placed("law", "tuning.en", "a", "a", None).score == 0.0 + assert Placed("law", "tuning.en", "a", "a", 5).hit + assert not Placed("law", "tuning.en", "a", "a", 6).hit + + +def test_a_chat_question_shares_its_topic() -> None: + assert topic_of("erasure-chat") == "erasure" + assert topic_of("erasure") == "erasure" + + +def test_nothing_changing_is_not_a_gain() -> None: + same = run(topics(20, 3)) + + judged = judge(same, same) + + assert judged.chance == 1.0 + assert not judged.kept + assert (judged.topics_found, judged.topics_lost) == (0, 0) + + +def test_a_gain_on_many_topics_is_kept() -> None: + judged = judge(run(topics(20, None)), run(topics(20, 1))) + + assert judged.kept + assert judged.chance < 0.001 + assert (judged.hits_before, judged.hits_after) == (0, 20) + assert (judged.topics_found, judged.topics_lost) == (20, 0) + assert (judged.score_before, judged.score_after) == (0.0, 1.0) + + +def test_a_gain_on_three_topics_is_what_chance_gives_one_time_in_eight() -> None: + """Three changes all upward is one of the eight ways three signs can fall.""" + before = run({**topics(20, 3), "a": None, "b": None, "c": None}) + after = run({**topics(20, 3), "a": 1, "b": 1, "c": 1}) + + judged = judge(before, after) + + assert judged.chance == pytest.approx(0.125, abs=0.02) + assert not judged.kept + + +def test_a_topic_asked_many_ways_counts_once() -> None: + """Three topics, each asked formally and as chat in two languages, are three topics: + twelve questions moving together are not twelve gains.""" + ways = [ + (f"{topic}{style}", language) + for topic in "abc" + for style in ("", "-chat") + for language in ("ar", "en") + ] + before = [ + Placed("law", f"tuning.{language}", question, topic_of(question), None) + for question, language in ways + ] + after = [ + Placed("law", f"tuning.{language}", question, topic_of(question), 1) + for question, language in ways + ] + + judged = judge(before, after) + + assert judged.questions == 12 + assert judged.topics == 3 + assert judged.chance == pytest.approx(0.125, abs=0.02) + assert not judged.kept + + +def test_a_clause_that_climbs_inside_the_top_five_counts() -> None: + """Hits at five do not change, and the right clauses sit higher.""" + judged = judge(run(topics(20, 5)), run(topics(20, 1))) + + assert (judged.hits_before, judged.hits_after) == (20, 20) + assert judged.kept + + +def test_a_set_that_loses_more_than_chance_explains_stops_the_change() -> None: + before = run(topics(40, None), "tuning.en") + run(topics(20, 1), "tuning.ar") + after = run(topics(40, 1), "tuning.en") + run(topics(20, None), "tuning.ar") + + judged = judge(before, after) + worse = {change.question_set: change.worse for change in judged.sets} + + assert judged.score_after > judged.score_before + assert worse == {"tuning.en": False, "tuning.ar": True} + assert not judged.kept + + +def test_a_set_that_slips_by_one_question_is_not_called_worse() -> None: + before = run(topics(40, None), "tuning.en") + run(topics(20, 5), "tuning.ar") + slipped = {**topics(20, 5), "topic-0": 6} + after = run(topics(40, 1), "tuning.en") + run(slipped, "tuning.ar") + + judged = judge(before, after) + arabic = next(change for change in judged.sets if change.question_set == "tuning.ar") + + assert (arabic.hits_before, arabic.hits_after) == (20, 19) + assert not arabic.worse + assert judged.kept + + +def test_runs_over_different_questions_are_refused() -> None: + with pytest.raises(Unpaired, match=r"1 in one run only, such as law/tuning\.en/b"): + judge(run({"a": 1, "b": 2}), run({"a": 1})) + + +def test_a_run_that_holds_a_question_twice_is_refused() -> None: + once = run({"a": 1, "b": 2}) + + with pytest.raises(Unpaired, match="holds a question twice"): + judge(once + run({"a": 3}), once) + + +def test_judging_twice_gives_the_same_answer() -> None: + changes = [0.5, -0.2, 0.9, 0.1, -0.4, 0.3, 0.75] + + assert chance_of_gain(changes) == chance_of_gain(changes) + + +def test_an_outcomes_file_is_read_question_by_question() -> None: + record = { + "pack": "law", + "sets": [ + { + "question_set": "tuning.ar", + "questions": [ + {"id": "erasure", "topic": "erasure", "place": 2}, + {"id": "erasure-chat", "topic": "erasure", "place": None}, + ], + } + ], + } + + assert placed_in(record) == [ + Placed("law", "tuning.ar", "erasure", "erasure", 2), + Placed("law", "tuning.ar", "erasure-chat", "erasure", None), + ]