diff --git a/evals/__init__.py b/evals/__init__.py new file mode 100644 index 000000000..5af0170ea --- /dev/null +++ b/evals/__init__.py @@ -0,0 +1 @@ +"""Version-controlled evaluation harnesses; never shipped with the package.""" diff --git a/evals/benchmark/README.md b/evals/benchmark/README.md new file mode 100644 index 000000000..200f5f7e3 --- /dev/null +++ b/evals/benchmark/README.md @@ -0,0 +1,121 @@ +# Benchmark harness: lilbee vs RAGFlow + +A preregistered, reproducible A/B of retrieval quality. Two arms answer the +same public, human-labeled datasets with the same served model, so retrieval is +the only variable: + +- **Arm A - lilbee**: the `test/retrieval-parity` branch with every parity + feature enabled. +- **Arm B - RAGFlow**: the DeepDoc pipeline at its defaults. + +One embedder and one generator are served once and pointed at by both arms, so +generation is identical and only the retrieval stack changes. Lives outside +`src/` on purpose: it never ships in the package. + +## Two tiers + +1. **Tier 1 - retrieval, no model opinion.** Each arm returns a ranked list per + query, written as a TREC run file. `pytrec_eval` scores it against the + dataset's published relevance labels (qrels): nDCG@10, Recall@20, MRR@10. + Nothing is graded by a model, so the numbers are bit-for-bit reproducible by + anyone with the same run files and qrels. +2. **Tier 2 - answer quality.** Both arms answer with the same model; RAGAS + scores faithfulness, answer relevancy, and context precision/recall. The + blind duplicate-arm judge from `evals/retrieval` is reused as a corroborating + signal, with its noise floor measured by grading one arm twice. + +Every cross-arm difference is paired per query and gets a bootstrap 95% CI and a +randomization-test p-value. A difference whose CI crosses zero is reported as +not significant, not as a win. + +## Native vs derived labels + +Passage sets (BEIR SciFact/FiQA/NFCorpus, HotpotQA, TREC-COVID) ship native TREC +qrels, used as published. The document-structured QA sets (TAT-DQA, OTT-QA) have +no retrieval labels of their own, so their qrels are **derived** from human +gold-evidence annotations by one documented pure function +(`datasets.derive_qrels_from_evidence`). Every derived dataset is marked +`label_kind: derived` in the manifest and flagged in the report. + +## Pod workflow + +One A100-80GB in a named tmux so a reclaimed pod reattaches. Preregister before +any data moves, then run each arm, score, and power off. + +```bash +cd ~/lilbee # the repo checkout; run everything from the repo root +RUN=/tmp/bench + +# 0. Freeze the preregistration. Nothing can be cherry-picked after this. +uv run python -m evals.benchmark preregister \ + --manifest evals/benchmark/manifest.example.yaml --out "$RUN/manifest.frozen.json" + +# 1. Serve the shared model with lilbee (all parity features on) and stand up +# RAGFlow pointed at the same OpenAI-compatible endpoint. +export LILBEE_TITLE_SEARCH=true LILBEE_NEIGHBOR_EXPANSION=2 \ + LILBEE_TABLE_EXTRACTION=true LILBEE_LAYOUT_DETECTION=true LILBEE_INTENT_LLM=true +lilbee serve --port 8080 & +uv run python -m evals.benchmark.bootstrap_ragflow \ + --base-url http://127.0.0.1:9380 --api-key "$RAGFLOW_KEY" \ + --corpus-dir "$RUN/corpus" --llm-model qwen2.5-72b-instruct \ + --embedding-model qwen3-embedding + +# 2. Ingest the same corpus into both systems (lilbee add; bootstrap uploaded +# RAGFlow's copy above). + +# 3. Collect a TREC run file per arm. Each query is checkpointed, so a killed +# run resumes. +uv run python -m evals.benchmark collect --system lilbee \ + --queries "$RUN/queries.jsonl" --base-url http://127.0.0.1:8080 \ + --run-tag lilbee --run "$RUN/run-lilbee.trec" --checkpoint "$RUN/ck-lilbee.jsonl" +uv run python -m evals.benchmark collect --system ragflow \ + --queries "$RUN/queries.jsonl" --base-url http://127.0.0.1:9380 \ + --api-key "$RAGFLOW_KEY" --dataset-id "$RAGFLOW_DATASET" \ + --run-tag ragflow --run "$RUN/run-ragflow.trec" --checkpoint "$RUN/ck-ragflow.jsonl" + +# 4. Tier 1: score each run against the qrels with pytrec_eval. +uv run python -m evals.benchmark score-ir --qrels "$RUN/qrels.json" \ + --run "$RUN/run-lilbee.trec" --dataset scifact --run-tag lilbee \ + --out "$RUN/ir-lilbee.jsonl" +uv run python -m evals.benchmark score-ir --qrels "$RUN/qrels.json" \ + --run "$RUN/run-ragflow.trec" --dataset scifact --run-tag ragflow \ + --out "$RUN/ir-ragflow.jsonl" + +# 5. Tier 2: generate answers on each arm, then score with RAGAS. +uv run python -m evals.benchmark answer --queries "$RUN/queries.jsonl" \ + --ground-truth "$RUN/references.json" --base-url http://127.0.0.1:8080 \ + --arm lilbee --out "$RUN/answers-lilbee.jsonl" +uv run python -m evals.benchmark score-ragas \ + --samples "$RUN/answers-lilbee.jsonl" --out "$RUN/results.jsonl" + +# 6. Paired statistics (CI + p) into the same results file. +uv run python -m evals.benchmark stats --manifest "$RUN/manifest.frozen.json" \ + --metrics-a "$RUN/ir-lilbee.jsonl" --metrics-b "$RUN/ir-ragflow.jsonl" \ + --arm-a-label lilbee --arm-b-label ragflow --out "$RUN/results.jsonl" + +# 7. Render the report, then power the pod off. +uv run python -m evals.benchmark report \ + --results "$RUN/results.jsonl" --out "$RUN/report.md" +``` + +Keep the frozen manifest, run files, and qrels: with those, a third party can +re-score the whole Tier-1 comparison without the pod. + +## Optional dependencies + +The heavy scorers and dataset loaders are imported lazily, so the harness (and +its tests) load without them. They are kept out of the shipped lock on purpose +(installing them would otherwise drag core dependencies backward), so install +them from the standalone requirements file on the benchmark pod: + +```bash +uv pip install -r evals/benchmark/requirements.txt # pytrec_eval, ragas, beir, datasets +``` + +## Determinism and resume + +- The frozen manifest pins the bootstrap seed, resamples, and alpha, so every + CI is reproducible. +- `collect` and `answer` checkpoint per query. Kill and re-run freely; only + unfinished queries repeat, and the run file is rebuilt from the full + checkpoint each time. diff --git a/evals/benchmark/RESULTS.md b/evals/benchmark/RESULTS.md new file mode 100644 index 000000000..ff0a34992 --- /dev/null +++ b/evals/benchmark/RESULTS.md @@ -0,0 +1,107 @@ +# Retrieval benchmark results — Qwen3-Embedding-0.6B + +A significance-tested measurement of lilbee's hybrid retrieval against its own +vector-only baseline on three public, human-labeled BEIR datasets. The point of +this run is to answer two questions with evidence, not opinion: does lilbee's +rank fusion actually improve retrieval over the raw embedder, and is a single +fusion weight the right design. + +## Method + +- **Embedder:** Qwen3-Embedding-0.6B (GGUF, Q8_0), served once; every arm points + at the same model, so retrieval is the only variable. +- **Datasets:** BEIR SciFact, NFCorpus, FiQA, using their published TREC qrels + as-is. No labels were derived or altered. +- **Metrics:** nDCG@10, Recall@20, MRR@10, scored with `pytrec_eval` against the + native qrels. Nothing is graded by a model, so the numbers are reproducible by + anyone with the run files and qrels. +- **Arms:** `dense` is vector-only (the lexical and title arms silenced); + `w=X` is hybrid fusion with the BM25 arm at `lexical_fusion_weight=X` and the + title arm on. This isolates what fusion adds over the embedder alone. +- **Significance:** every hybrid-vs-dense difference is paired per query and gets + a bootstrap 95% CI and a randomization-test p-value. A difference whose CI + crosses zero is reported as not significant, not as a win. +- **Condition:** query expansion was off (no chat model served), so these are + pure first-pass retrieval numbers with no LLM query rewriting on either arm. + +## Results (nDCG@10) + +| Dataset | dense | w=1.0 | w=0.75 | w=0.5 | w=0.25 | +|---|---|---|---|---|---| +| NFCorpus (n=322) | 0.3666 | 0.3734 | 0.3743 | 0.3756 | **0.3767** | +| SciFact (n=300) | 0.6981 | **0.7280** | 0.7226 | 0.7167 | 0.7155 | +| FiQA (n=648) | **0.4598** | 0.4033 | 0.4289 | 0.4333 | 0.4363 | + +## Significance (best hybrid weight vs. dense, nDCG@10) + +| Dataset | best hybrid | Δ vs dense | 95% CI | p | verdict | +|---|---|---|---|---|---| +| NFCorpus | w=0.25 | **+0.0100** | [+0.0022, +0.0180] | 0.012 | significant win | +| SciFact | w=1.0 | **+0.0299** | [+0.0073, +0.0528] | 0.011 | significant win | +| FiQA | w=0.25 | **−0.0235** | [−0.0356, −0.0118] | 0.0004 | significant regression | + +On FiQA every hybrid weight loses to dense (w=1.0 is worst at −0.0564, p=0.0001); +lowering the weight reduces the damage but never recovers it. On NFCorpus the +win is only significant at lower weights (w=1.0 is +0.0068, p=0.21, not +significant). On SciFact every weight wins, and higher is better. + +## What this establishes + +1. **Hybrid fusion adds real, significant value on two of three corpora** — once + it actually runs (see the bug below). This is the raw embedder plus lilbee's + fusion beating the raw embedder alone, measured, not asserted. +2. **No single fixed fusion weight wins everywhere.** SciFact wants a strong + lexical arm (w=1.0), NFCorpus wants a weak one (w=0.25), and FiQA wants none. + The optimal weight is corpus-dependent, which is the empirical case for + gating the lexical weight per query rather than fixing it — the adaptive + fusion this run motivates. + +## The bug this run caught + +Before any of these numbers meant anything, every hybrid config scored +identically to the vector-only baseline. The cause was a production defect, not +a benchmark artifact: `ensure_fts_index()` runs `table.optimize()` on an +existing index, which on a real-sized corpus hits a LanceDB encoding bug and +raises; the failure was swallowed and left hybrid search disabled, so every +query silently fell back to vector-only. It only surfaced on real data — a +tiny local corpus never tripped the LanceDB bug. Fixed by marking an existing, +queryable index ready before the best-effort optimize, so an optimize failure +degrades to a warning instead of disabling retrieval corpus-wide. + +## Reproducing + +The frozen manifest, the TREC run files, and the qrels are kept under +`results/`. With those, anyone can re-score Tier 1 without a GPU: + +```bash +gunzip -k results/scifact/run-w1.0.trec.gz +python -m evals.benchmark score-ir --qrels results/scifact/qrels.json \ + --run results/scifact/run-w1.0.trec --dataset scifact --run-tag w1.0 --out /tmp/ir.jsonl +``` + +## Adaptive fusion + +Because no fixed weight wins everywhere, a follow-up run measured `adaptive_fusion` +(the BM25 weight scaled per query by how peaked the vector ranking is) at three +confidence margins, on the same three datasets. Best margin was 0.15. + +| Dataset | dense | w=1.0 | adaptive (m=0.15) | adaptive vs dense | +|---|---|---|---|---| +| NFCorpus | 0.3666 | 0.3732 | 0.3751 | +0.0085, p=0.033, **sig win** | +| SciFact | 0.6981 | 0.7280 | 0.7186 | +0.0205, p=0.009, **sig win** | +| FiQA | 0.4593 | 0.4027 | 0.4410 | −0.0184, p=0.003, sig regression | + +Adaptive fusion is the best single policy found. It beats the fixed w=1.0 default +on all three datasets, keeps the significant NFCorpus and SciFact wins, and gives +the smallest FiQA regression of any config (−0.018 vs −0.056 at w=1.0 and −0.023 +at the best fixed weight). It does not fully erase FiQA's regression — pure dense +still wins there — so the vector-margin confidence signal reduces, but does not +eliminate, the cases where the lexical arm hurts. Run files and stats are under +`results-adaptive/`. + +## Scope + +This is the 0.6B-embedder study: it validates the fusion mechanism and the +FTS-optimize fix, and it sizes the corpus-dependence of the lexical weight. It +is deliberately not a headline "lilbee vs. everyone" number — a larger embedder +and the RAGFlow A/B arm are separate runs. diff --git a/evals/benchmark/__init__.py b/evals/benchmark/__init__.py new file mode 100644 index 000000000..6cb76c08d --- /dev/null +++ b/evals/benchmark/__init__.py @@ -0,0 +1,9 @@ +"""Preregistered, reproducible A/B benchmark: lilbee vs RAGFlow retrieval. + +Two arms answer the same public, human-labeled datasets with the same served +model, so retrieval is the only variable. Tier 1 scores ranked results against +the datasets' own relevance labels with pytrec_eval (no model opinion); Tier 2 +layers RAGAS answer-quality metrics on top, corroborated by the blind judge +from ``evals.retrieval``. Lives outside ``src/`` on purpose: it never ships in +the package. +""" diff --git a/evals/benchmark/__main__.py b/evals/benchmark/__main__.py new file mode 100644 index 000000000..025eb5474 --- /dev/null +++ b/evals/benchmark/__main__.py @@ -0,0 +1,6 @@ +"""Runs the benchmark CLI: python -m evals.benchmark .""" + +from evals.benchmark.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/benchmark/bootstrap_ragflow.py b/evals/benchmark/bootstrap_ragflow.py new file mode 100644 index 000000000..5d05902e3 --- /dev/null +++ b/evals/benchmark/bootstrap_ragflow.py @@ -0,0 +1,141 @@ +"""Bring up a RAGFlow arm bound to the shared model, ready for collection. + +Given a running RAGFlow, this registers the shared OpenAI-compatible LLM and +embedding endpoint (so generation is identical to lilbee's), creates a DeepDoc +dataset, uploads a corpus directory, waits for parsing, creates a chat +assistant bound to the dataset, and prints the api_key and assistant_id the +collector and answer steps consume. + +This is a live operational script: it only talks to a real RAGFlow and is never +imported by the test suite. Run it on the pod once RAGFlow's stack is up. +""" + +from __future__ import annotations + +import argparse +import sys +import time +from pathlib import Path +from typing import Any + +import httpx + +DATASETS_ROUTE = "/api/v1/datasets" +DOCUMENTS_ROUTE = "/api/v1/datasets/{dataset_id}/documents" +PARSE_ROUTE = "/api/v1/datasets/{dataset_id}/chunks" +CHATS_ROUTE = "/api/v1/chats" +PARSE_POLL_SECONDS = 10.0 +PARSE_MAX_POLLS = 360 +REQUEST_TIMEOUT_SECONDS = 120.0 +DONE_RATIO = 1.0 + + +def _client(base_url: str, api_key: str) -> httpx.Client: + return httpx.Client( + base_url=base_url.rstrip("/"), + headers={"Authorization": f"Bearer {api_key}"}, + timeout=REQUEST_TIMEOUT_SECONDS, + ) + + +def _data(response: httpx.Response) -> Any: + response.raise_for_status() + body = response.json() + if body.get("code", 0) != 0: + raise RuntimeError(f"ragflow error: {body.get('message', body)}") + return body.get("data") + + +def create_dataset(client: httpx.Client, name: str, embedding_model: str) -> str: + """Create a DeepDoc dataset and return its id.""" + data = _data( + client.post( + DATASETS_ROUTE, + json={ + "name": name, + "chunk_method": "naive", + "embedding_model": embedding_model, + "parser_config": {"layout_recognize": "DeepDOC"}, + }, + ) + ) + return str(data["id"]) + + +def upload_corpus(client: httpx.Client, dataset_id: str, corpus_dir: Path) -> list[str]: + """Upload every file under ``corpus_dir`` and return the document ids.""" + route = DOCUMENTS_ROUTE.format(dataset_id=dataset_id) + files = [("file", (path.name, path.read_bytes())) for path in sorted(corpus_dir.iterdir())] + data = _data(client.post(route, files=files)) + return [str(doc["id"]) for doc in data] + + +def parse_and_wait(client: httpx.Client, dataset_id: str, document_ids: list[str]) -> None: + """Trigger parsing and block until every document finishes.""" + route = PARSE_ROUTE.format(dataset_id=dataset_id) + _data(client.post(route, json={"document_ids": document_ids})) + docs_route = DOCUMENTS_ROUTE.format(dataset_id=dataset_id) + for _ in range(PARSE_MAX_POLLS): + docs = _data(client.get(docs_route))["docs"] + if all(doc.get("progress", 0.0) >= DONE_RATIO for doc in docs): + return + time.sleep(PARSE_POLL_SECONDS) + raise RuntimeError("ragflow parsing did not finish within the poll budget") + + +def create_assistant(client: httpx.Client, name: str, dataset_id: str, llm_model: str) -> str: + """Create a chat assistant bound to the dataset and return its id.""" + data = _data( + client.post( + CHATS_ROUTE, + json={ + "name": name, + "dataset_ids": [dataset_id], + "llm": {"model_name": llm_model, "temperature": 0.0}, + }, + ) + ) + return str(data["id"]) + + +def bootstrap(args: argparse.Namespace) -> int: + client = _client(args.base_url, args.api_key) + dataset_id = create_dataset(client, args.dataset_name, args.embedding_model) + document_ids = upload_corpus(client, dataset_id, args.corpus_dir) + parse_and_wait(client, dataset_id, document_ids) + assistant_id = create_assistant(client, args.assistant_name, dataset_id, args.llm_model) + print(f"api_key={args.api_key}") + print(f"dataset_id={dataset_id}") + print(f"assistant_id={assistant_id}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", required=True, help="RAGFlow base URL") + parser.add_argument("--api-key", required=True, help="RAGFlow API key") + parser.add_argument("--corpus-dir", type=Path, required=True) + parser.add_argument("--dataset-name", default="benchmark-corpus") + parser.add_argument("--assistant-name", default="benchmark-assistant") + parser.add_argument( + "--llm-model", + required=True, + help="the shared OpenAI-compatible generator, e.g. qwen2.5-72b-instruct", + ) + parser.add_argument( + "--embedding-model", required=True, help="the shared embedder, e.g. qwen3-embedding" + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return bootstrap(args) + except (httpx.HTTPError, RuntimeError, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/benchmark/cli.py b/evals/benchmark/cli.py new file mode 100644 index 000000000..059aaa7ad --- /dev/null +++ b/evals/benchmark/cli.py @@ -0,0 +1,293 @@ +"""Command-line entry point for the lilbee-vs-RAGFlow benchmark. + +Subcommands mirror the run stages: preregister, collect, score-ir, answer, +score-ragas, stats, report. Heavy scorers (pytrec_eval, ragas) are imported +lazily inside the modules they live in, so the CLI loads without them. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +import httpx + +from evals.benchmark import ir_metrics, stats +from evals.benchmark.collectors import ( + DEFAULT_TOP_K, + LilbeeCollector, + RagflowCollector, + collect_run, + load_queries, +) +from evals.benchmark.manifest import Manifest +from evals.benchmark.ragas_tier import Sample, score_ragas +from evals.benchmark.report import render_report +from evals.retrieval.checkpoint import JsonlCheckpoint, load_jsonl + +DEFAULT_METRICS = ["nDCG@10", "Recall@20", "MRR@10"] +ASK_ROUTE = "/api/ask" +ASK_TIMEOUT_SECONDS = 600.0 + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + +def _append_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as fh: + for row in rows: + fh.write(json.dumps(row) + "\n") + + +def _cmd_preregister(args: argparse.Namespace) -> int: + manifest = Manifest.load(args.manifest) + fingerprint = manifest.freeze(args.out) + print(f"froze manifest {manifest.run_id} -> {args.out} ({fingerprint[:12]})") + return 0 + + +def _build_collector(args: argparse.Namespace) -> LilbeeCollector | RagflowCollector: + if args.system == "lilbee": + return LilbeeCollector(args.base_url, run_tag=args.run_tag, top_k=args.top_k) + if not args.api_key or not args.dataset_id: + raise ValueError("ragflow collection needs --api-key and at least one --dataset-id") + return RagflowCollector( + args.base_url, args.api_key, args.dataset_id, run_tag=args.run_tag, top_k=args.top_k + ) + + +def _cmd_collect(args: argparse.Namespace) -> int: + queries = load_queries(args.queries) + collector = _build_collector(args) + hits = collect_run( + collector, + queries, + args.run, + args.checkpoint, + on_query=lambda qid: print(f"[{args.run_tag}] {qid}", flush=True), + ) + print(f"collected {len(queries)} queries, {len(hits)} hits -> {args.run}") + return 0 + + +def _cmd_score_ir(args: argparse.Namespace) -> int: + from evals.benchmark.runfile import read_run, run_to_pytrec + + qrels = json.loads(args.qrels.read_text()) + run = run_to_pytrec(read_run(args.run)) + scores = ir_metrics.score_run(qrels, run, args.metrics) + _write_jsonl( + args.out, + [ + { + "dataset": args.dataset, + "run_tag": args.run_tag, + "aggregated": scores.aggregated, + "per_query": scores.per_query, + } + ], + ) + print(f"scored {args.run_tag} on {args.dataset}: {scores.aggregated} -> {args.out}") + return 0 + + +def _ask(client: httpx.Client, base_url: str, question: str, top_k: int) -> tuple[str, list[str]]: + response = client.post( + f"{base_url.rstrip('/')}{ASK_ROUTE}", json={"question": question, "top_k": top_k} + ) + response.raise_for_status() + body = response.json() + contexts = [chunk["chunk"] for chunk in body.get("sources", [])] + return body["answer"], contexts + + +def _cmd_answer(args: argparse.Namespace) -> int: + queries = load_queries(args.queries) + references = json.loads(args.ground_truth.read_text()) if args.ground_truth else {} + checkpoint = JsonlCheckpoint(args.out, "query_id") + client = httpx.Client(timeout=ASK_TIMEOUT_SECONDS) + for query_id, text in queries.items(): + if query_id in checkpoint: + continue + answer, contexts = _ask(client, args.base_url, text, args.top_k) + checkpoint.append( + { + "query_id": query_id, + "question": text, + "answer": answer, + "contexts": contexts, + "ground_truth": references.get(query_id, ""), + } + ) + print(f"[{args.arm}] {query_id}", flush=True) + print(f"answered {len(queries)} queries -> {args.out}") + return 0 + + +def _cmd_score_ragas(args: argparse.Namespace) -> int: + samples = [ + Sample( + question=row["question"], + answer=row["answer"], + contexts=list(row["contexts"]), + ground_truth=row.get("ground_truth", ""), + ) + for row in load_jsonl(args.samples) + ] + scores_a = score_ragas(samples, args.metrics) + rows = [ + {"row_type": "ragas", "metric": metric, "arm_a": value, "arm_b": value} + for metric, value in scores_a.items() + ] + _append_jsonl(args.out, rows) + print(f"scored {len(samples)} answers with RAGAS -> {args.out}") + return 0 + + +def _load_metrics_file(path: Path) -> dict[str, Any]: + rows = load_jsonl(path) + if not rows: + raise ValueError(f"no metrics rows in {path}") + return rows[0] + + +def _cmd_stats(args: argparse.Namespace) -> int: + manifest = Manifest.load(args.manifest) + file_a = _load_metrics_file(args.metrics_a) + file_b = _load_metrics_file(args.metrics_b) + dataset = file_a["dataset"] + rows: list[dict[str, Any]] = [ + { + "row_type": "meta", + "run_id": manifest.run_id, + "fingerprint": manifest.fingerprint(), + "arm_a": args.arm_a_label, + "arm_b": args.arm_b_label, + } + ] + for metric in manifest.metrics: + per_query_a = file_a["per_query"].get(metric, {}) + per_query_b = file_b["per_query"].get(metric, {}) + result = stats.compare( + metric, + per_query_a, + per_query_b, + resamples=manifest.stats.bootstrap_resamples, + seed=manifest.stats.seed, + alpha=manifest.stats.alpha, + ) + row = {"row_type": "ir", "dataset": dataset, **result.to_dict()} + rows.append(row) + _append_jsonl(args.out, rows) + print(f"wrote {len(rows) - 1} paired IR comparisons -> {args.out}") + return 0 + + +def _cmd_report(args: argparse.Namespace) -> int: + report = render_report(load_jsonl(args.results)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(report) + print(f"wrote {args.out}") + return 0 + + +def _add_preregister(sub: argparse._SubParsersAction) -> None: + parser = sub.add_parser("preregister", help="validate and freeze the manifest") + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.set_defaults(handler=_cmd_preregister) + + +def _add_collect(sub: argparse._SubParsersAction) -> None: + parser = sub.add_parser("collect", help="build a TREC run file for one arm") + parser.add_argument("--system", choices=("lilbee", "ragflow"), required=True) + parser.add_argument("--queries", type=Path, required=True) + parser.add_argument("--run", type=Path, required=True) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--run-tag", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--top-k", type=int, default=DEFAULT_TOP_K) + parser.add_argument("--api-key", default="", help="ragflow only") + parser.add_argument( + "--dataset-id", action="append", default=[], help="ragflow only, repeatable" + ) + parser.set_defaults(handler=_cmd_collect) + + +def _add_score_ir(sub: argparse._SubParsersAction) -> None: + parser = sub.add_parser("score-ir", help="score a run file against qrels with pytrec_eval") + parser.add_argument("--qrels", type=Path, required=True) + parser.add_argument("--run", type=Path, required=True) + parser.add_argument("--dataset", required=True) + parser.add_argument("--run-tag", required=True) + parser.add_argument("--metrics", nargs="+", default=DEFAULT_METRICS) + parser.add_argument("--out", type=Path, required=True) + parser.set_defaults(handler=_cmd_score_ir) + + +def _add_answer(sub: argparse._SubParsersAction) -> None: + parser = sub.add_parser("answer", help="generate answers for the RAGAS tier") + parser.add_argument("--queries", type=Path, required=True) + parser.add_argument("--ground-truth", type=Path, default=None) + parser.add_argument("--base-url", required=True) + parser.add_argument("--arm", required=True) + parser.add_argument("--top-k", type=int, default=0) + parser.add_argument("--out", type=Path, required=True) + parser.set_defaults(handler=_cmd_answer) + + +def _add_score_ragas(sub: argparse._SubParsersAction) -> None: + parser = sub.add_parser("score-ragas", help="score generated answers with RAGAS") + parser.add_argument("--samples", type=Path, required=True) + parser.add_argument("--metrics", nargs="+", default=None) + parser.add_argument("--out", type=Path, required=True) + parser.set_defaults(handler=_cmd_score_ragas) + + +def _add_stats(sub: argparse._SubParsersAction) -> None: + parser = sub.add_parser("stats", help="paired bootstrap CI and randomization test") + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--metrics-a", type=Path, required=True) + parser.add_argument("--metrics-b", type=Path, required=True) + parser.add_argument("--arm-a-label", default="lilbee") + parser.add_argument("--arm-b-label", default="ragflow") + parser.add_argument("--out", type=Path, required=True) + parser.set_defaults(handler=_cmd_stats) + + +def _add_report(sub: argparse._SubParsersAction) -> None: + parser = sub.add_parser("report", help="render results.jsonl as markdown") + parser.add_argument("--results", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.set_defaults(handler=_cmd_report) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="evals.benchmark", description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + for register in ( + _add_preregister, + _add_collect, + _add_score_ir, + _add_answer, + _add_score_ragas, + _add_stats, + _add_report, + ): + register(sub) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return int(args.handler(args)) + except (ValueError, FileNotFoundError, RuntimeError, httpx.HTTPError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 diff --git a/evals/benchmark/collectors.py b/evals/benchmark/collectors.py new file mode 100644 index 000000000..b818399e7 --- /dev/null +++ b/evals/benchmark/collectors.py @@ -0,0 +1,160 @@ +"""Per-system collectors that turn a query set into a TREC run file. + +Each collector hits one system's retrieval API and returns chunk hits tagged +with their parent document; the shared driver collapses those to a document run +and writes the TREC file. Every query is checkpointed as it lands, so a killed +pod run resumes without re-querying completed queries. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any, Protocol + +import httpx + +from evals.benchmark.runfile import ChunkHit, collapse_hits, write_run +from evals.retrieval.checkpoint import JsonlCheckpoint, load_jsonl + +SEARCH_ROUTE = "/api/search" +RAGFLOW_RETRIEVAL_ROUTE = "/api/v1/retrieval" +RETRIEVE_TIMEOUT_SECONDS = 120.0 +DEFAULT_TOP_K = 20 + + +class Collector(Protocol): + """Retrieves ranked chunk hits for one query from one system.""" + + run_tag: str + + def retrieve(self, query_id: str, query_text: str) -> list[ChunkHit]: ... + + +def make_http_client() -> httpx.Client: + return httpx.Client(timeout=RETRIEVE_TIMEOUT_SECONDS) + + +class LilbeeCollector: + """Reads ranked results from a running lilbee server's ``/api/search``. + + lilbee already groups results by source document, so each result maps to one + document hit scored by ``best_relevance``. + """ + + def __init__( + self, + base_url: str, + *, + run_tag: str = "lilbee", + top_k: int = DEFAULT_TOP_K, + client: httpx.Client | None = None, + ) -> None: + self._base_url = base_url.rstrip("/") + self.run_tag = run_tag + self._top_k = top_k + self._client = client or make_http_client() + + def retrieve(self, query_id: str, query_text: str) -> list[ChunkHit]: + response = self._client.get( + f"{self._base_url}{SEARCH_ROUTE}", + params={"q": query_text, "top_k": self._top_k}, + ) + response.raise_for_status() + return [ + ChunkHit(query_id=query_id, doc_id=doc["source"], score=float(doc["best_relevance"])) + for doc in response.json() + ] + + +class RagflowCollector: + """Reads ranked chunks from RAGFlow's retrieval API and tags parent docs.""" + + def __init__( + self, + base_url: str, + api_key: str, + dataset_ids: list[str], + *, + run_tag: str = "ragflow", + top_k: int = DEFAULT_TOP_K, + client: httpx.Client | None = None, + ) -> None: + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._dataset_ids = dataset_ids + self.run_tag = run_tag + self._top_k = top_k + self._client = client or make_http_client() + + def retrieve(self, query_id: str, query_text: str) -> list[ChunkHit]: + response = self._client.post( + f"{self._base_url}{RAGFLOW_RETRIEVAL_ROUTE}", + headers={"Authorization": f"Bearer {self._api_key}"}, + json={ + "question": query_text, + "dataset_ids": self._dataset_ids, + "page": 1, + "page_size": self._top_k, + }, + ) + response.raise_for_status() + chunks = response.json().get("data", {}).get("chunks", []) + return [ + ChunkHit( + query_id=query_id, + doc_id=chunk["document_id"], + score=float(chunk["similarity"]), + ) + for chunk in chunks + ] + + +def _row_hits(query_id: str, row: dict[str, Any]) -> list[ChunkHit]: + return [ + ChunkHit(query_id=query_id, doc_id=doc_id, score=score) for doc_id, score in row["hits"] + ] + + +def collect_run( + collector: Collector, + queries: dict[str, str], + run_path: Path, + checkpoint_path: Path, + *, + on_query: Callable[[str], None] | None = None, +) -> list[ChunkHit]: + """Retrieve every query (resuming from the checkpoint) and write the run file. + + Each query's hits are appended to ``checkpoint_path`` as they land; the run + file is (re)built from the full checkpoint at the end, so an interrupted run + resumes without re-querying and still produces a complete run file. + """ + checkpoint = JsonlCheckpoint(checkpoint_path, "query_id") + for query_id, query_text in queries.items(): + if query_id in checkpoint: + continue + hits = collector.retrieve(query_id, query_text) + checkpoint.append({"query_id": query_id, "hits": [[hit.doc_id, hit.score] for hit in hits]}) + if on_query is not None: + on_query(query_id) + all_hits: list[ChunkHit] = [] + for row in load_jsonl(checkpoint_path): + all_hits.extend(_row_hits(row["query_id"], row)) + entries = collapse_hits(all_hits, collector.run_tag) + write_run(run_path, entries) + return all_hits + + +def load_queries(path: Path) -> dict[str, str]: + """Read a queries file: one ``{"query_id": ..., "text": ...}`` object per line.""" + return {row["query_id"]: row["text"] for row in load_jsonl(path)} + + +def write_queries(path: Path, queries: dict[str, str]) -> None: + """Write a queries file the collector reads back.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps({"query_id": qid, "text": text}) + "\n" for qid, text in queries.items()) + ) diff --git a/evals/benchmark/datasets.py b/evals/benchmark/datasets.py new file mode 100644 index 000000000..bf2efdfb2 --- /dev/null +++ b/evals/benchmark/datasets.py @@ -0,0 +1,171 @@ +"""Dataset loading, normalized to a single (corpus, queries, qrels) triple. + +Two label kinds: + +- NATIVE qrels ship with the dataset (BEIR, MS MARCO, HotpotQA). They are used + as published. +- DERIVED qrels are computed from human gold-evidence annotations on QA + datasets that have no retrieval labels of their own (TAT-DQA, OTT-QA). The + derivation is the single documented pure function ``derive_qrels_from_evidence``, + and every derived dataset is recorded as such in the manifest. + +Real downloads live behind a lazily-imported loader. The normalization and +derivation logic is pure and is what the unit tests exercise on tiny fixtures. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +LABEL_NATIVE = "native" +LABEL_DERIVED = "derived" + +# Relevance grade assigned to a document named by human gold evidence. +DERIVED_RELEVANCE = 1 + +Corpus = dict[str, dict[str, str]] +Queries = dict[str, str] +Qrels = dict[str, dict[str, int]] + + +@dataclass(frozen=True) +class Dataset: + """A retrieval benchmark normalized to corpus, queries, and qrels.""" + + name: str + label_kind: str + corpus: Corpus + queries: Queries + qrels: Qrels + + +def normalize_corpus(rows: Mapping[str, Mapping[str, Any]]) -> Corpus: + """Normalize BEIR-style corpus rows to ``doc_id -> {title, text}``. + + Documents whose title and text are both empty are dropped: they can never + be a relevant hit and only inflate the index. + """ + corpus: Corpus = {} + for doc_id, row in rows.items(): + title = str(row.get("title", "")).strip() + text = str(row.get("text", "")).strip() + if title or text: + corpus[str(doc_id)] = {"title": title, "text": text} + return corpus + + +def normalize_queries(rows: Mapping[str, Any]) -> Queries: + """Normalize query rows to ``query_id -> text``, dropping empty queries.""" + queries: Queries = {} + for qid, text in rows.items(): + cleaned = str(text).strip() + if cleaned: + queries[str(qid)] = cleaned + return queries + + +def normalize_qrels(rows: Mapping[str, Mapping[str, Any]]) -> Qrels: + """Normalize native qrels to ``query_id -> {doc_id: grade}`` with int grades. + + Non-positive grades are dropped so the qrels hold only judged-relevant + documents, matching how pytrec_eval treats them. + """ + qrels: Qrels = {} + for qid, judged in rows.items(): + graded = {str(doc): int(grade) for doc, grade in judged.items() if int(grade) > 0} + if graded: + qrels[str(qid)] = graded + return qrels + + +def derive_qrels_from_evidence( + evidence: Mapping[str, Iterable[str]], relevance: int = DERIVED_RELEVANCE +) -> Qrels: + """Derive retrieval qrels from human gold-evidence document annotations. + + Each query maps to the set of document ids a human annotator marked as gold + evidence for its answer; every such document is labeled relevant at + ``relevance``. This is the documented derivation for QA datasets that carry + gold evidence but no native retrieval labels (TAT-DQA, OTT-QA). Queries with + no evidence are omitted, since an unjudged query cannot be scored. + """ + qrels: Qrels = {} + for qid, doc_ids in evidence.items(): + graded = {str(doc_id): relevance for doc_id in doc_ids} + if graded: + qrels[str(qid)] = graded + return qrels + + +def build_native_dataset( + name: str, + corpus_rows: Mapping[str, Mapping[str, Any]], + query_rows: Mapping[str, Any], + qrel_rows: Mapping[str, Mapping[str, Any]], +) -> Dataset: + """Assemble a native-qrel dataset from raw BEIR-style rows.""" + return Dataset( + name=name, + label_kind=LABEL_NATIVE, + corpus=normalize_corpus(corpus_rows), + queries=normalize_queries(query_rows), + qrels=normalize_qrels(qrel_rows), + ) + + +def build_derived_dataset( + name: str, + corpus_rows: Mapping[str, Mapping[str, Any]], + query_rows: Mapping[str, Any], + evidence: Mapping[str, Iterable[str]], +) -> Dataset: + """Assemble a derived-qrel dataset from a QA corpus plus gold evidence.""" + return Dataset( + name=name, + label_kind=LABEL_DERIVED, + corpus=normalize_corpus(corpus_rows), + queries=normalize_queries(query_rows), + qrels=derive_qrels_from_evidence(evidence), + ) + + +# Loaders for real downloads. Each returns the raw rows the pure builders above +# consume; they are only called by ``load_dataset`` and import heavy deps lazily. +RawNativeLoader = Callable[[Path], tuple[Mapping, Mapping, Mapping]] +RawDerivedLoader = Callable[[Path], tuple[Mapping, Mapping, Mapping]] + + +def _load_beir(cache_dir: Path, dataset_key: str, split: str) -> tuple[Mapping, Mapping, Mapping]: + """Download and read a BEIR dataset (SciFact, FiQA, NFCorpus, ...).""" + from beir import util + from beir.datasets.data_loader import GenericDataLoader + + url = f"https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{dataset_key}.zip" + data_path = util.download_and_unzip(url, str(cache_dir)) + return GenericDataLoader(data_folder=data_path).load(split=split) + + +def load_dataset( + spec: Any, + cache_dir: Path, + *, + native_loader: RawNativeLoader | None = None, + derived_loader: RawDerivedLoader | None = None, +) -> Dataset: + """Load a dataset described by a manifest ``DatasetSpec`` to a normalized triple. + + The real download is injected (or lazily selected by ``spec.loader``); the + normalization and derivation are the pure functions above. Tests inject + tiny in-memory loaders so nothing touches the network. + """ + if spec.label_kind == LABEL_DERIVED: + if derived_loader is None: + raise ValueError(f"no derived loader available for dataset '{spec.name}'") + corpus_rows, query_rows, evidence = derived_loader(cache_dir) + return build_derived_dataset(spec.name, corpus_rows, query_rows, evidence) + loader = native_loader or (lambda path: _load_beir(path, spec.loader, spec.split)) + corpus_rows, query_rows, qrel_rows = loader(cache_dir) + return build_native_dataset(spec.name, corpus_rows, query_rows, qrel_rows) diff --git a/evals/benchmark/ir_metrics.py b/evals/benchmark/ir_metrics.py new file mode 100644 index 000000000..f83f5c97a --- /dev/null +++ b/evals/benchmark/ir_metrics.py @@ -0,0 +1,84 @@ +"""Tier-1 retrieval metrics: score a run file against qrels with pytrec_eval. + +pytrec_eval is the TREC-standard scorer, so these numbers carry no model +opinion and are exactly reproducible from the run file and the qrels. It is an +optional dependency (a C extension), imported lazily; the per-query shaping and +aggregation are pulled out so they can be unit-tested with a fake evaluator and +no C extension installed. +""" + +from __future__ import annotations + +import statistics +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any, Protocol + +Qrels = dict[str, dict[str, int]] +Run = dict[str, dict[str, float]] + +# Display name -> the pytrec_eval measure that computes it. recip_rank is the +# uncut MRR; pytrec_eval has no built-in cut-at-10 variant, and MRR is +# dominated by the top hit, so at realistic depths the two agree. +METRIC_MEASURES: dict[str, str] = { + "nDCG@10": "ndcg_cut_10", + "Recall@20": "recall_20", + "MRR@10": "recip_rank", +} + +PYTREC_INSTALL_HINT = ( + "pytrec_eval is required to score retrieval; install the benchmark deps: " + "uv pip install -r evals/benchmark/requirements.txt" +) + + +class Evaluator(Protocol): + """The slice of pytrec_eval.RelevanceEvaluator this module uses.""" + + def evaluate(self, run: Run) -> dict[str, dict[str, float]]: ... + + +EvaluatorFactory = Callable[[Qrels, set[str]], Evaluator] + + +@dataclass(frozen=True) +class MetricScores: + """Per-query and aggregated scores for the requested display metrics.""" + + per_query: dict[str, dict[str, float]] = field(default_factory=dict) + aggregated: dict[str, float] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return {"per_query": self.per_query, "aggregated": self.aggregated} + + +def _default_evaluator_factory(qrels: Qrels, measures: set[str]) -> Evaluator: + try: + import pytrec_eval + except ImportError as exc: # pragma: no cover - exercised only without the extra + raise RuntimeError(PYTREC_INSTALL_HINT) from exc + return pytrec_eval.RelevanceEvaluator(qrels, measures) + + +def score_run( + qrels: Qrels, + run: Run, + metrics: list[str], + *, + evaluator_factory: EvaluatorFactory | None = None, +) -> MetricScores: + """Score a run against qrels, returning per-query and mean-aggregated metrics.""" + unknown = [metric for metric in metrics if metric not in METRIC_MEASURES] + if unknown: + raise ValueError(f"unknown metrics: {', '.join(unknown)}") + factory = evaluator_factory or _default_evaluator_factory + measures = {METRIC_MEASURES[metric] for metric in metrics} + raw = factory(qrels, measures).evaluate(run) + per_query: dict[str, dict[str, float]] = {} + aggregated: dict[str, float] = {} + for metric in metrics: + measure = METRIC_MEASURES[metric] + scores = {qid: float(values[measure]) for qid, values in raw.items()} + per_query[metric] = scores + aggregated[metric] = round(statistics.fmean(scores.values()), 4) if scores else 0.0 + return MetricScores(per_query=per_query, aggregated=aggregated) diff --git a/evals/benchmark/manifest.example.yaml b/evals/benchmark/manifest.example.yaml new file mode 100644 index 000000000..b4889369c --- /dev/null +++ b/evals/benchmark/manifest.example.yaml @@ -0,0 +1,74 @@ +# Preregistration for one lilbee-vs-RAGFlow retrieval benchmark run. +# Freeze this with `python -m evals.benchmark preregister` before any data +# moves; the frozen copy carries a fingerprint so the run cannot be re-cut. +run_id: parity-2026-07 + +# Both arms share one served embedder and one served generator; only retrieval +# differs. The judge must differ from the generator so answers are never graded +# by the model that wrote them (validated at load time). +models: + embedder: qwen3-embedding + generator: qwen2.5-72b-instruct + generator_swap: mistral-large-2411 + judge: fable-5 + +arms: + - name: lilbee-parity + system: lilbee + description: test/retrieval-parity with every parity feature enabled + config: + title_search: true + neighbor_expansion: 2 + table_extraction: true + layout_detection: true + intent_llm: true + - name: ragflow-default + system: ragflow + description: RAGFlow DeepDoc pipeline at its defaults + config: + pipeline: deepdoc + +# Passage sets carry native TREC qrels; the document-structured QA sets have +# their retrieval labels derived from human gold-evidence annotations, marked +# label_kind: derived and disclosed here. +datasets: + - name: scifact + loader: scifact + label_kind: native + split: test + - name: fiqa + loader: fiqa + label_kind: native + split: test + - name: nfcorpus + loader: nfcorpus + label_kind: native + split: test + - name: hotpotqa + loader: hotpotqa + label_kind: native + split: test + - name: trec-covid + loader: trec-covid + label_kind: native + split: test + - name: tat-dqa + loader: tatdqa + label_kind: derived + split: test + - name: ott-qa + loader: ottqa + label_kind: derived + split: dev + +metrics: + - nDCG@10 + - Recall@20 + - MRR@10 + +stats: + bootstrap_resamples: 10000 + seed: 20260714 + alpha: 0.05 + +temperature: 0.0 diff --git a/evals/benchmark/manifest.py b/evals/benchmark/manifest.py new file mode 100644 index 000000000..0adcb73ec --- /dev/null +++ b/evals/benchmark/manifest.py @@ -0,0 +1,164 @@ +"""The preregistration manifest: everything frozen before any data moves. + +Freezing the datasets, metrics, both arms' configs, and the held-constant +models to a file (and its fingerprint) is what stops the run from being +cherry-picked after the numbers land. The key structural guarantee validated +here is that the judge model differs from the generator model, so the answer +tier is never graded by the same model that wrote the answer. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from evals.benchmark.datasets import LABEL_DERIVED, LABEL_NATIVE + +LILBEE_SYSTEM = "lilbee" +RAGFLOW_SYSTEM = "ragflow" +FROZEN_TEMPERATURE = 0.0 + + +@dataclass(frozen=True) +class ArmConfig: + """One system under test and the configuration it runs with.""" + + name: str + system: str + description: str + config: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ModelConfig: + """The models held constant across both arms. + + ``embedder`` is served once and shared by both arms. ``generator`` writes + every answer; ``judge`` grades them and MUST differ from ``generator``. + ``generator_swap`` records an alternate generator noted for a sensitivity + check, not used in the main run. + """ + + embedder: str + generator: str + judge: str + generator_swap: str = "" + + +@dataclass(frozen=True) +class DatasetSpec: + """One dataset, its loader, and whether its qrels are native or derived.""" + + name: str + loader: str + label_kind: str + split: str = "test" + + +@dataclass(frozen=True) +class StatsConfig: + """Paired-statistics configuration, frozen so CIs are reproducible.""" + + bootstrap_resamples: int = 10000 + seed: int = 20260714 + alpha: float = 0.05 + + +@dataclass(frozen=True) +class Manifest: + """The full frozen preregistration for one benchmark run.""" + + run_id: str + arms: list[ArmConfig] + models: ModelConfig + datasets: list[DatasetSpec] + metrics: list[str] + stats: StatsConfig + temperature: float = FROZEN_TEMPERATURE + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + def fingerprint(self) -> str: + """Stable sha256 over the canonical JSON; the preregistration's identity.""" + canonical = json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode()).hexdigest() + + @property + def derived_datasets(self) -> list[str]: + return [ds.name for ds in self.datasets if ds.label_kind == LABEL_DERIVED] + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Manifest: + manifest = cls( + run_id=data["run_id"], + arms=[ArmConfig(**arm) for arm in data["arms"]], + models=ModelConfig(**data["models"]), + datasets=[DatasetSpec(**ds) for ds in data["datasets"]], + metrics=list(data["metrics"]), + stats=StatsConfig(**data.get("stats", {})), + temperature=data.get("temperature", FROZEN_TEMPERATURE), + ) + manifest.validate() + return manifest + + @classmethod + def load(cls, path: Path) -> Manifest: + """Load and validate a manifest from a YAML or JSON file.""" + text = path.read_text() + if path.suffix in (".yaml", ".yml"): + import yaml + + data = yaml.safe_load(text) + else: + data = json.loads(text) + return cls.from_dict(data) + + def freeze(self, path: Path) -> str: + """Write the canonical manifest to ``path`` and return its fingerprint.""" + self.validate() + path.parent.mkdir(parents=True, exist_ok=True) + payload = self.to_dict() + payload["fingerprint"] = self.fingerprint() + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + return payload["fingerprint"] + + def validate(self) -> None: + """Fail loudly on any preregistration invariant the run depends on.""" + _validate_arms(self.arms) + _validate_models(self.models) + _validate_datasets(self.datasets) + if not self.metrics: + raise ValueError("manifest lists no metrics") + if self.temperature != FROZEN_TEMPERATURE: + raise ValueError(f"temperature must be {FROZEN_TEMPERATURE} for a deterministic run") + + +def _validate_arms(arms: list[ArmConfig]) -> None: + systems = {arm.system for arm in arms} + if len(arms) != 2: # noqa: PLR2004 - an A/B has exactly two arms + raise ValueError("an A/B benchmark needs exactly two arms") + if len({arm.name for arm in arms}) != len(arms): + raise ValueError("arm names must be distinct") + if systems != {LILBEE_SYSTEM, RAGFLOW_SYSTEM}: + raise ValueError(f"arms must be one {LILBEE_SYSTEM} and one {RAGFLOW_SYSTEM}") + + +def _validate_models(models: ModelConfig) -> None: + if not models.embedder: + raise ValueError("a shared embedder model is required") + if not models.generator: + raise ValueError("a generator model is required") + if models.judge == models.generator: + raise ValueError("judge model must differ from the generator model") + + +def _validate_datasets(datasets: list[DatasetSpec]) -> None: + if not datasets: + raise ValueError("manifest lists no datasets") + for ds in datasets: + if ds.label_kind not in (LABEL_NATIVE, LABEL_DERIVED): + raise ValueError(f"dataset '{ds.name}' has unknown label_kind '{ds.label_kind}'") diff --git a/evals/benchmark/ragas_tier.py b/evals/benchmark/ragas_tier.py new file mode 100644 index 000000000..d3fafa2f7 --- /dev/null +++ b/evals/benchmark/ragas_tier.py @@ -0,0 +1,119 @@ +"""Tier-2 answer quality: RAGAS metrics plus the reused blind judge. + +RAGAS scores the generated answers (faithfulness, answer relevancy, context +precision/recall). The blind duplicate-arm judge from ``evals.retrieval`` is +reused unchanged as a corroborating signal, with its own noise floor measured +by grading one arm twice. ragas is an optional heavy dependency, imported +lazily; the wiring is injectable so tests need neither ragas nor a real model. +""" + +from __future__ import annotations + +import random +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from evals.retrieval.answers import AnswerRow +from evals.retrieval.blinding import build_blind_rows, unblind +from evals.retrieval.judging import DIMENSIONS, judge_rows +from evals.retrieval.llm import ChatFn +from evals.retrieval.questions import Question +from evals.retrieval.scoring import ARM_REPLICATE, NOISE_REPLICATE, dimension_means, noise_floor + +RAGAS_METRICS = ("faithfulness", "answer_relevancy", "context_precision", "context_recall") + +RAGAS_INSTALL_HINT = ( + "ragas is required for the answer tier; install the benchmark deps: " + "uv pip install -r evals/benchmark/requirements.txt" +) + +# question, answer, contexts, ground_truth -> per-metric score. +RagasEvaluateFn = Callable[[list[dict[str, Any]], list[str]], dict[str, float]] + + +@dataclass(frozen=True) +class Sample: + """One generated answer with the context it was grounded on.""" + + question: str + answer: str + contexts: list[str] + ground_truth: str + + def to_ragas_row(self) -> dict[str, Any]: + return { + "user_input": self.question, + "response": self.answer, + "retrieved_contexts": list(self.contexts), + "reference": self.ground_truth, + } + + +def _default_ragas_evaluate(rows: list[dict[str, Any]], metrics: list[str]) -> dict[str, float]: + try: + from ragas import EvaluationDataset, evaluate + from ragas import metrics as ragas_metrics + except ImportError as exc: # pragma: no cover - exercised only without the extra + raise RuntimeError(RAGAS_INSTALL_HINT) from exc + selected = [getattr(ragas_metrics, name) for name in metrics] + dataset = EvaluationDataset.from_list(rows) + result = evaluate(dataset=dataset, metrics=selected) + scores = result.to_pandas()[metrics].mean().to_dict() + return {metric: float(scores[metric]) for metric in metrics} + + +def score_ragas( + samples: list[Sample], + metrics: list[str] | None = None, + *, + evaluate_fn: RagasEvaluateFn | None = None, +) -> dict[str, float]: + """Mean RAGAS score per metric over the samples.""" + evaluator = evaluate_fn or _default_ragas_evaluate + rows = [sample.to_ragas_row() for sample in samples] + return evaluator(rows, list(metrics if metrics is not None else RAGAS_METRICS)) + + +@dataclass(frozen=True) +class JudgeSummary: + """The corroborating judge's per-arm means and its measured noise floor.""" + + noise_floor: float + means: dict[str, dict[str, float]] + + def to_dict(self) -> dict[str, Any]: + return {"noise_floor": self.noise_floor, "means": self.means} + + +def run_corroborating_judge( + questions: list[Question], + answers_by_arm: dict[str, dict[str, AnswerRow]], + noise_arm: str, + chat: ChatFn, + work_dir: Path, + *, + seed: int, +) -> JudgeSummary: + """Run the reused blind judge and summarize per-arm means and its noise floor. + + The noise arm is graded twice under fresh opaque ids; the disagreement + between those two passes is the judge's noise floor, so a small answer-tier + gap is never oversold. + """ + work_dir.mkdir(parents=True, exist_ok=True) + blind = build_blind_rows(questions, answers_by_arm, noise_arm, random.Random(seed)) + grades = judge_rows(blind.rows, chat, work_dir / "grades.jsonl") + for gid in blind.prefailed: + grades[gid] = dict.fromkeys(DIMENSIONS, 0) + unblinded = unblind(blind.assignments, grades) + means = { + arm: dimension_means(replicates.get(ARM_REPLICATE, {})) + for arm, replicates in unblinded.items() + } + noise_replicates = unblinded.get(noise_arm, {}) + floor = noise_floor( + noise_replicates.get(ARM_REPLICATE, {}), noise_replicates.get(NOISE_REPLICATE, {}) + ) + return JudgeSummary(noise_floor=floor, means=means) diff --git a/evals/benchmark/report.py b/evals/benchmark/report.py new file mode 100644 index 000000000..101b7a546 --- /dev/null +++ b/evals/benchmark/report.py @@ -0,0 +1,117 @@ +"""Markdown rendering of a scored benchmark run. + +Leads with the Tier-1 label-scored retrieval numbers (the reproducible core), +then the Tier-2 RAGAS answer scores, then the feature-to-dataset coverage +matrix with derived-qrel datasets clearly marked. +""" + +from __future__ import annotations + +from typing import Any + + +def _rows_of(rows: list[dict[str, Any]], row_type: str) -> list[dict[str, Any]]: + return [row for row in rows if row.get("row_type") == row_type] + + +def _arm_labels(rows: list[dict[str, Any]]) -> tuple[str, str]: + meta = _rows_of(rows, "meta") + if meta: + return meta[0]["arm_a"], meta[0]["arm_b"] + return "arm A", "arm B" + + +def _sig_cell(row: dict[str, Any]) -> str: + ci = f"[{row['ci_low']:+.4f}, {row['ci_high']:+.4f}]" + flag = "" if row["significant"] else " (n.s.)" + return f"{ci}{flag}" + + +def _ir_section(rows: list[dict[str, Any]], arm_a: str, arm_b: str) -> list[str]: + ir_rows = _rows_of(rows, "ir") + if not ir_rows: + return [] + lines = [ + "## Tier 1 - retrieval, scored against human labels", + "", + "Computed by pytrec_eval against each dataset's published relevance " + "labels. No model judges anything, so these numbers are exactly " + "reproducible from the run files and qrels.", + "", + f"| dataset | metric | {arm_a} | {arm_b} | delta | 95% CI (B - A) | p |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for row in ir_rows: + lines.append( + f"| {row['dataset']} | {row['metric']} | {row['mean_a']:.4f} | " + f"{row['mean_b']:.4f} | {row['mean_diff']:+.4f} | {_sig_cell(row)} | " + f"{row['p_value']:.3f} |" + ) + lines.append("") + return lines + + +def _ragas_section(rows: list[dict[str, Any]], arm_a: str, arm_b: str) -> list[str]: + ragas_rows = _rows_of(rows, "ragas") + if not ragas_rows: + return [] + lines = [ + "## Tier 2 - answer quality (RAGAS)", + "", + f"| metric | {arm_a} | {arm_b} |", + "| --- | --- | --- |", + ] + for row in ragas_rows: + lines.append(f"| {row['metric']} | {row['arm_a']:.4f} | {row['arm_b']:.4f} |") + for judge in _rows_of(rows, "judge"): + lines += [ + "", + f"Corroborating blind judge noise floor: plus or minus " + f"{judge['noise_floor']} per dimension (one arm graded twice). " + "Answer-tier gaps at or below it are not oversold.", + ] + lines.append("") + return lines + + +def _coverage_section(rows: list[dict[str, Any]]) -> list[str]: + coverage_rows = _rows_of(rows, "coverage") + if not coverage_rows: + return [] + lines = [ + "## Feature coverage", + "", + "Each feature under test maps to a dataset that stresses it. Datasets " + "whose retrieval labels are derived from human gold evidence are marked.", + "", + "| feature under test | proven by | tier-1 metric | qrels |", + "| --- | --- | --- | --- |", + ] + for row in coverage_rows: + qrels = "derived" if row.get("derived") else "native" + lines.append(f"| {row['feature']} | {row['dataset']} | {row['metric']} | {qrels} |") + lines.append("") + return lines + + +def render_report(rows: list[dict[str, Any]]) -> str: + """Render results rows to the human-readable benchmark markdown report.""" + arm_a, arm_b = _arm_labels(rows) + meta = _rows_of(rows, "meta") + header = ["# lilbee vs RAGFlow retrieval benchmark", ""] + if meta: + header.append( + f"Run `{meta[0].get('run_id', '?')}` " + f"(manifest `{meta[0].get('fingerprint', '?')[:12]}`). " + f"Arm A = {arm_a}, arm B = {arm_b}." + ) + header.append("") + lines = ( + header + + _ir_section(rows, arm_a, arm_b) + + _ragas_section(rows, arm_a, arm_b) + + _coverage_section(rows) + ) + if not _rows_of(rows, "ir") and not _rows_of(rows, "ragas"): + raise ValueError("results contain no ir or ragas rows; run score-ir first") + return "\n".join(lines).rstrip() + "\n" diff --git a/evals/benchmark/requirements.txt b/evals/benchmark/requirements.txt new file mode 100644 index 000000000..9eac502c4 --- /dev/null +++ b/evals/benchmark/requirements.txt @@ -0,0 +1,10 @@ +# Heavy scorers and dataset loaders for the retrieval benchmark. Pod-only and +# research-only: imported lazily, never touched by the core package or the test +# suite, and deliberately kept OUT of the shipped lock so installing them cannot +# move a core dependency. Install on the benchmark pod with: +# uv pip install -r evals/benchmark/requirements.txt +# pytrec_eval-terrier is the maintained pytrec_eval fork; it imports as pytrec_eval. +pytrec_eval-terrier>=0.5.6 +ragas>=0.2 +beir>=2.0 +datasets>=2.16 diff --git a/evals/benchmark/results-adaptive/fiqa/ir-adm0.15.jsonl b/evals/benchmark/results-adaptive/fiqa/ir-adm0.15.jsonl new file mode 100644 index 000000000..bc3cc2dfd --- /dev/null +++ b/evals/benchmark/results-adaptive/fiqa/ir-adm0.15.jsonl @@ -0,0 +1 @@ +{"dataset": "fiqa", "run_tag": "adm0.15", "aggregated": {"nDCG@10": 0.441, "Recall@20": 0.6192, "MRR@10": 0.5143}, "per_query": {"nDCG@10": {"8": 0.6131471927654584, "15": 0.6309297535714575, "18": 0.6309297535714575, "26": 0.23719771276929622, "34": 0.0, "42": 0.46927872602275644, "56": 0.3562071871080222, "68": 0.0, "89": 0.286381299268402, "90": 0.6309297535714575, "94": 1.0, "98": 0.6131471927654584, "104": 0.6131471927654584, "106": 0.5, "109": 1.0, "475": 1.0, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.3333333333333333, "559": 0.0, "570": 0.0, "585": 0.2640681225725909, "588": 0.5714285040141098, "594": 0.6131471927654584, "603": 0.0, "604": 0.34870224750355494, "620": 0.0, "622": 0.38685280723454163, "659": 0.5067837010870873, "672": 0.0, "684": 0.0, "687": 0.19342640361727081, "689": 0.0, "691": 0.6309297535714575, "699": 1.0, "701": 0.13565197343244778, "715": 0.0, "721": 1.0, "744": 0.18154179253735267, "750": 0.8772153153380493, "753": 0.38685280723454163, "766": 0.18154179253735267, "776": 0.4690000933206748, "810": 0.3333333333333333, "813": 0.20438239758848611, "849": 0.3333333333333333, "852": 0.7653606369886217, "853": 0.0, "858": 0.0, "859": 0.38685280723454163, "864": 0.7247145167543899, "879": 0.6309297535714575, "885": 0.19342640361727081, "895": 1.0, "904": 1.0, "928": 0.6131471927654584, "929": 1.0, "932": 0.3065735963827292, "939": 0.5, "945": 0.3010299956639812, "957": 0.7653606369886217, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.23719771276929622, "1150": 0.0, "1157": 0.6131471927654584, "1159": 0.0, "1198": 0.6309297535714575, "1230": 0.0, "1281": 0.38685280723454163, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.6469523342593044, "1310": 1.0, "1321": 0.0, "1322": 0.24630238874073, "1391": 0.18154179253735267, "1393": 0.30260241349881345, "1415": 0.9178287402913377, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.431733884398313, "1670": 0.7653606369886217, "1676": 0.5101671483174705, "1736": 0.5531464700081437, "1748": 0.6105456988825855, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.286381299268402, "1824": 0.31546487678572877, "1826": 0.9828920819566879, "1832": 0.5, "1871": 0.6713527276121545, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.21398626473452756, "2010": 0.9325210919548239, "2051": 0.0, "2070": 0.18154179253735267, "2075": 0.40129869165981646, "2076": 0.6934264036172708, "2088": 0.0, "2108": 0.16716045496620227, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 0.7526136409516659, "2204": 0.8984972075197749, "2264": 0.38685280723454163, "2296": 0.33627415762678553, "2306": 0.6131471927654584, "2316": 0.8772153153380493, "2318": 0.3010299956639812, "2330": 0.6131471927654584, "2334": 0.46927872602275644, "2348": 0.43231813227099475, "2376": 0.9830168297915423, "2383": 0.6131471927654584, "2384": 0.6309297535714575, "2385": 0.6131471927654584, "2388": 1.0, "2395": 1.0, "2398": 0.0, "2399": 1.0, "2400": 0.5307212739772434, "2407": 0.24630238874073, "2416": 0.8838242945899706, "2423": 0.3576223542196109, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 0.43067655807339306, "2513": 0.0, "2516": 0.6366824387328317, "2549": 0.6131471927654584, "2551": 1.0, "2568": 0.25288473642100684, "2579": 0.2423936099728648, "2580": 0.46927872602275644, "2587": 1.0, "2589": 0.7653606369886217, "2590": 0.46927872602275644, "2593": 0.8854598815714874, "2598": 0.9197207891481876, "2648": 1.0, "2676": 0.5, "2685": 0.35079429740281753, "2695": 1.0, "2713": 0.38685280723454163, "2724": 0.8772153153380493, "2737": 1.0, "2747": 0.5, "2749": 0.8772153153380493, "2790": 0.499028327865556, "2801": 0.6131471927654584, "2856": 0.7095272044910244, "2857": 0.6595629128335778, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.3562071871080222, "2923": 0.8207658584763088, "2964": 0.0, "2968": 0.9197207891481876, "2994": 0.46927872602275644, "3006": 0.25909036127391766, "3008": 0.24630238874073, "3014": 1.0, "3033": 0.6131471927654584, "3039": 0.38685280723454163, "3049": 0.24630238874073, "3051": 1.0, "3067": 0.0, "3085": 0.19342640361727081, "3091": 0.9020870746500321, "3103": 1.0, "3115": 0.4935232796777481, "3125": 1.0, "3148": 0.5531464700081437, "3149": 0.0, "3177": 0.15642624200758548, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.7039180890341347, "3264": 0.8048099750039491, "3357": 1.0, "3369": 0.0, "3394": 0.5805485932475565, "3404": 0.3932591258696538, "3405": 0.0, "3446": 0.46704774694449785, "3451": 0.07945707943276742, "3453": 0.5706417189553201, "3480": 1.0, "3490": 0.3010299956639812, "3500": 0.644824486427155, "3503": 0.6891341765546151, "3512": 0.9197207891481876, "3528": 0.6309297535714575, "3530": 0.4776237035032179, "3534": 0.0, "3566": 0.8194270280062366, "3569": 0.5, "3594": 0.123151194370365, "3612": 0.11751610475642714, "3615": 0.45749452618986164, "3625": 0.20210734650054757, "3682": 0.20438239758848611, "3683": 0.7021799053685303, "3694": 0.6131471927654584, "3724": 0.0, "3735": 0.13565197343244778, "3759": 0.0, "3767": 0.2716774977597197, "3771": 0.4935232796777481, "3781": 0.9197207891481876, "3789": 0.787702056960637, "3791": 0.370291336443826, "3801": 0.0, "3822": 0.0, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5517854393872873, "3932": 0.0, "3934": 0.16812753627111746, "3995": 0.0, "4007": 0.6364391809889587, "4011": 1.0, "4019": 0.31608365985743925, "4031": 0.0, "4037": 0.18457569677956817, "4047": 0.8048099750039491, "4071": 0.0, "4084": 0.5012658353418871, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 0.6131471927654584, "4125": 0.0, "4142": 0.44927383782962327, "4153": 0.31546487678572877, "4179": 0.6309297535714575, "4188": 0.0, "4205": 0.4525081529734507, "4233": 0.8539316501572937, "4265": 0.7751482523375255, "4286": 0.6131471927654584, "4289": 0.3065735963827292, "4306": 0.7598336331031966, "4312": 0.6370339733570289, "4335": 0.8318724637288826, "4339": 1.0, "4394": 0.0, "4409": 0.19518900079066107, "4411": 0.46927872602275644, "4414": 0.8315546295836225, "4415": 0.0, "4433": 0.0, "4447": 1.0, "4464": 0.5, "4465": 0.2960819109658652, "4484": 1.0, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 0.4034698846650846, "4523": 0.21398626473452756, "4539": 0.21840743681816419, "4571": 0.7653606369886217, "4600": 1.0, "4605": 0.5585830187272559, "4615": 0.4909156065077737, "4640": 0.0, "4641": 0.252016380211836, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75369761125927, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.6131471927654584, "4813": 1.0, "4823": 0.46927872602275644, "4827": 0.38685280723454163, "4837": 0.0, "4844": 1.0, "4845": 0.7653606369886217, "4846": 0.0, "4863": 0.5307212739772434, "4865": 0.0, "4920": 1.0, "4942": 0.31546487678572877, "4946": 1.0, "4955": 0.0, "4962": 0.6131471927654584, "4968": 0.6309297535714575, "4981": 0.6508205185601091, "4999": 0.5855700749881525, "5021": 0.0, "5030": 0.0, "5045": 0.3010299956639812, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.3333333333333333, "5080": 0.0, "5083": 0.23719771276929622, "5085": 1.0, "5086": 1.0, "5090": 0.2640681225725909, "5125": 1.0, "5134": 0.38685280723454163, "5150": 0.0, "5155": 1.0, "5172": 0.0, "5178": 0.252016380211836, "5185": 0.0, "5196": 0.2960819109658652, "5206": 0.0, "5228": 0.31546487678572877, "5231": 0.0, "5241": 0.44615333764087994, "5254": 0.6131471927654584, "5255": 1.0, "5264": 0.7227265726449519, "5271": 0.41618115554873086, "5331": 1.0, "5343": 0.3333333333333333, "5347": 1.0, "5356": 0.1208113026994942, "5369": 0.0, "5374": 0.3010299956639812, "5380": 1.0, "5402": 0.0, "5410": 0.2960819109658652, "5422": 0.24630238874073, "5427": 0.23719771276929622, "5460": 0.0, "5464": 0.16716045496620227, "5503": 0.6131471927654584, "5505": 1.0, "5511": 0.1610425878239398, "5534": 0.38685280723454163, "5549": 0.0, "5585": 0.31546487678572877, "5592": 1.0, "5616": 0.3562071871080222, "5620": 0.0, "5646": 0.9217868789962071, "5653": 1.0, "5683": 0.6934264036172708, "5710": 0.0, "5741": 0.0, "5763": 0.23463936301137822, "5782": 0.4371994911049732, "5790": 0.0, "5808": 0.5, "5853": 0.21078558563971425, "5862": 0.29127873064148246, "5888": 0.2640681225725909, "5903": 0.0, "5906": 0.44130740935663865, "5940": 0.0, "5951": 0.42750379661745125, "5970": 0.23463936301137822, "5981": 1.0, "5993": 0.31488013066763093, "6002": 0.7720617383222417, "6004": 0.0, "6005": 0.5771493379489847, "6009": 0.0, "6041": 0.6366824387328317, "6080": 0.6131471927654584, "6110": 0.41918605213740834, "6121": 0.38685280723454163, "6122": 0.46927872602275644, "6131": 0.5810820381115676, "6133": 0.6131471927654584, "6142": 0.6131471927654584, "6146": 1.0, "6199": 0.5135312443624667, "6219": 0.6934264036172708, "6221": 0.0, "6252": 0.0, "6262": 0.49818925746641285, "6278": 0.20438239758848611, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.3562071871080222, "6467": 0.0, "6468": 0.6131471927654584, "6479": 1.0, "6525": 0.49818925746641285, "6554": 0.6131471927654584, "6562": 0.11751610475642714, "6611": 0.7903864795495061, "6612": 0.16716045496620227, "6625": 0.31546487678572877, "6629": 0.0, "6635": 0.6366824387328317, "6644": 0.0, "6647": 0.7653606369886217, "6668": 0.8503449055347546, "6679": 0.5307212739772434, "6683": 0.8503449055347546, "6713": 0.6131471927654584, "6715": 0.6131471927654584, "6746": 0.3065735963827292, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.5413996682199069, "6807": 1.0, "6814": 0.0, "6832": 0.430624116386567, "6835": 0.6131471927654584, "6849": 0.3333333333333333, "6862": 1.0, "6867": 0.22471947398559278, "6875": 0.0, "6890": 0.5012658353418871, "6891": 0.617319681505689, "6896": 0.0, "6901": 0.21840743681816419, "6907": 0.0, "6909": 0.0, "6959": 0.38685280723454163, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 0.8772153153380493, "7096": 0.6309297535714575, "7098": 0.6131471927654584, "7105": 0.5, "7109": 0.23463936301137822, "7124": 0.7246262544989281, "7141": 0.0, "7145": 0.13565197343244778, "7178": 0.5, "7188": 0.31546487678572877, "7205": 1.0, "7206": 0.0, "7218": 0.43067655807339306, "7221": 0.0, "7269": 0.0, "7279": 0.44229834886928765, "7295": 0.0, "7311": 0.0, "7326": 0.0, "7329": 0.46845052016107697, "7344": 0.6131471927654584, "7345": 0.15642624200758548, "7377": 1.0, "7431": 0.0, "7441": 0.20210734650054757, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.14606834984270645, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.16716045496620227, "7529": 0.3065735963827292, "7533": 0.0, "7534": 0.46927872602275644, "7590": 0.9197207891481876, "7592": 0.0, "7594": 0.3333333333333333, "7622": 0.0, "7633": 0.3903800499921017, "7674": 0.0, "7700": 0.123151194370365, "7702": 0.38685280723454163, "7705": 0.49818925746641285, "7734": 0.6131471927654584, "7747": 0.21840743681816419, "7754": 0.0, "7758": 1.0, "7801": 1.0, "7803": 0.19342640361727081, "7823": 0.617319681505689, "7876": 0.5, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.19092086617893467, "7928": 0.24630238874073, "7936": 0.4632423659320675, "7992": 0.38685280723454163, "8002": 0.6257049680303419, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.31546487678572877, "8040": 1.0, "8072": 0.6508205185601091, "8079": 0.7039180890341347, "8102": 0.11284514134893527, "8116": 0.45749452618986164, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 0.6131471927654584, "8275": 0.43067655807339306, "8296": 0.9197207891481876, "8332": 0.9060254355346823, "8351": 0.6131471927654584, "8378": 0.1480409554829326, "8456": 0.617319681505689, "8475": 0.7957503936198023, "8507": 0.0, "8512": 1.0, "8513": 0.0, "8532": 0.9325210919548239, "8537": 0.3562071871080222, "8539": 0.0, "8544": 0.6131471927654584, "8592": 0.23463936301137822, "8632": 0.2640681225725909, "8635": 0.0, "8702": 0.7653606369886217, "8779": 0.448643819352159, "8789": 0.7653606369886217, "8795": 1.0, "8832": 1.0, "8834": 0.3980628465882808, "8855": 0.0, "8874": 0.31546487678572877, "8934": 0.8065735963827293, "8937": 0.43067655807339306, "8947": 0.23719771276929622, "8959": 0.38685280723454163, "8970": 0.6934264036172708, "8974": 0.41253177989255346, "8982": 0.0, "9060": 0.6131471927654584, "9088": 0.0, "9108": 0.0, "9115": 0.5833416105149332, "9126": 0.0, "9164": 0.16716045496620227, "9174": 0.0, "9188": 0.6131471927654584, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.20438239758848611, "9329": 0.8065735963827293, "9332": 0.43067655807339306, "9381": 0.6934264036172708, "9385": 0.3562071871080222, "9391": 0.5143371794949736, "9403": 0.0, "9481": 0.5706417189553201, "9487": 0.0, "9548": 0.43067655807339306, "9556": 1.0, "9565": 1.0, "9598": 0.3562071871080222, "9617": 0.3152014104491349, "9633": 0.6131471927654584, "9643": 0.5, "9644": 0.6309297535714575, "9646": 0.45749452618986164, "9668": 0.6508205185601091, "9701": 0.0, "9733": 0.18154179253735267, "9735": 0.3333333333333333, "9737": 0.7653606369886217, "9771": 0.0, "9808": 0.2960819109658652, "9824": 1.0, "9871": 0.24630238874073, "9882": 0.0, "9925": 0.8175295903539445, "9929": 0.5714285040141098, "9961": 0.0, "9979": 0.3903800499921017, "10034": 0.38685280723454163, "10039": 0.6309297535714575, "10109": 0.5186511804016661, "10122": 0.19519002499605084, "10136": 0.34337431936037655, "10137": 0.38685280723454163, "10152": 1.0, "10183": 0.3065735963827292, "10213": 0.46927872602275644, "10246": 0.0, "10267": 0.0, "10414": 0.9197207891481876, "10447": 0.18154179253735267, "10462": 0.0, "10482": 0.0, "10497": 0.48864468918810133, "10526": 0.20438239758848611, "10547": 0.0, "10558": 0.0, "10596": 0.9427731067700877, "10601": 1.0, "10628": 0.3010299956639812, "10639": 0.21398626473452756, "10645": 0.38685280723454163, "10674": 0.8048099750039491, "10710": 0.3333333333333333, "10734": 0.8696762340896295, "10792": 0.8503449055347546, "10808": 0.3295827480202853, "10809": 0.6049306994552042, "10812": 0.9493885684853097, "10827": 0.14606834984270645, "10845": 1.0, "10912": 0.0, "10932": 1.0, "10975": 0.0, "10979": 0.6366824387328317, "10994": 0.5, "11039": 0.315526956404728, "11054": 1.0, "11088": 0.43067655807339306}, "Recall@20": {"8": 0.5, "15": 1.0, "18": 1.0, "26": 0.5, "34": 0.0, "42": 0.3333333333333333, "56": 1.0, "68": 0.0, "89": 0.5, "90": 1.0, "94": 1.0, "98": 0.5, "104": 0.5, "106": 1.0, "109": 1.0, "475": 1.0, "503": 1.0, "504": 0.3333333333333333, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 1.0, "585": 0.5, "588": 1.0, "594": 1.0, "603": 0.0, "604": 1.0, "620": 0.2, "622": 0.5, "659": 0.7, "672": 0.5, "684": 1.0, "687": 0.5, "689": 0.0, "691": 1.0, "699": 1.0, "701": 0.3333333333333333, "715": 0.0, "721": 1.0, "744": 0.3333333333333333, "750": 1.0, "753": 0.5, "766": 0.3333333333333333, "776": 0.25, "810": 1.0, "813": 1.0, "849": 1.0, "852": 0.6666666666666666, "853": 0.5, "858": 0.0, "859": 0.5, "864": 0.6666666666666666, "879": 1.0, "885": 0.5, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.5, "939": 1.0, "945": 1.0, "957": 0.6666666666666666, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.5, "1150": 0.0, "1157": 0.5, "1159": 0.0, "1198": 1.0, "1230": 1.0, "1281": 1.0, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.75, "1310": 1.0, "1321": 0.0, "1322": 0.25, "1391": 0.6666666666666666, "1393": 0.5, "1415": 1.0, "1416": 1.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 1.0, "1670": 0.6666666666666666, "1676": 1.0, "1736": 0.4, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 1.0, "1826": 1.0, "1832": 1.0, "1871": 0.75, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.4, "2010": 1.0, "2051": 1.0, "2070": 0.6666666666666666, "2075": 0.5555555555555556, "2076": 1.0, "2088": 0.0, "2108": 0.3333333333333333, "2118": 0.0, "2154": 0.0, "2181": 0.5, "2183": 0.8333333333333334, "2204": 1.0, "2264": 0.5, "2296": 0.2222222222222222, "2306": 0.5, "2316": 1.0, "2318": 1.0, "2330": 0.5, "2334": 0.6666666666666666, "2348": 0.26666666666666666, "2376": 1.0, "2383": 0.5, "2384": 1.0, "2385": 0.5, "2388": 1.0, "2395": 1.0, "2398": 0.2, "2399": 1.0, "2400": 0.6666666666666666, "2407": 0.5, "2416": 1.0, "2423": 0.8571428571428571, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.3333333333333333, "2472": 0.0, "2486": 1.0, "2498": 1.0, "2513": 1.0, "2516": 0.75, "2549": 0.5, "2551": 1.0, "2568": 0.2857142857142857, "2579": 0.5, "2580": 0.3333333333333333, "2587": 1.0, "2589": 0.6666666666666666, "2590": 0.6666666666666666, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 1.0, "2685": 0.25, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 1.0, "2790": 0.42857142857142855, "2801": 1.0, "2856": 0.75, "2857": 0.75, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 1.0, "2923": 0.8, "2964": 0.5, "2968": 1.0, "2994": 0.3333333333333333, "3006": 0.5, "3008": 0.25, "3014": 1.0, "3033": 0.5, "3039": 1.0, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 1.0, "3091": 0.8571428571428571, "3103": 1.0, "3115": 0.5, "3125": 1.0, "3148": 0.6, "3149": 0.0, "3177": 0.6666666666666666, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.6666666666666666, "3264": 0.75, "3357": 1.0, "3369": 0.16666666666666666, "3394": 0.75, "3404": 0.42857142857142855, "3405": 1.0, "3446": 0.8, "3451": 0.42857142857142855, "3453": 1.0, "3480": 1.0, "3490": 1.0, "3500": 0.5, "3503": 0.7142857142857143, "3512": 1.0, "3528": 1.0, "3530": 0.6666666666666666, "3534": 0.0, "3566": 1.0, "3569": 1.0, "3594": 0.25, "3612": 0.75, "3615": 1.0, "3625": 0.3333333333333333, "3682": 1.0, "3683": 0.6666666666666666, "3694": 0.5, "3724": 0.0, "3735": 1.0, "3759": 0.0, "3767": 0.4, "3771": 0.3333333333333333, "3781": 1.0, "3789": 0.75, "3791": 0.6666666666666666, "3801": 0.0, "3822": 0.25, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 0.6666666666666666, "4011": 1.0, "4019": 0.4, "4031": 0.0, "4037": 0.5, "4047": 0.75, "4071": 0.0, "4084": 1.0, "4102": 0.25, "4103": 0.0, "4105": 0.0, "4116": 0.5, "4125": 0.0, "4142": 0.75, "4153": 1.0, "4179": 1.0, "4188": 1.0, "4205": 0.6666666666666666, "4233": 0.8, "4265": 0.6666666666666666, "4286": 0.5, "4289": 0.5, "4306": 0.75, "4312": 0.8571428571428571, "4335": 0.75, "4339": 1.0, "4394": 0.0, "4409": 0.2, "4411": 0.6666666666666666, "4414": 1.0, "4415": 0.0, "4433": 0.5, "4447": 1.0, "4464": 1.0, "4465": 0.6666666666666666, "4484": 1.0, "4499": 0.3333333333333333, "4500": 0.5, "4504": 1.0, "4514": 0.5, "4523": 0.2, "4539": 0.5, "4571": 0.6666666666666666, "4600": 1.0, "4605": 0.7142857142857143, "4615": 0.6666666666666666, "4640": 0.0, "4641": 0.6, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 0.6666666666666666, "4846": 0.0, "4863": 1.0, "4865": 0.0, "4920": 1.0, "4942": 1.0, "4946": 1.0, "4955": 0.0, "4962": 0.5, "4968": 1.0, "4981": 0.6666666666666666, "4999": 0.75, "5021": 0.0, "5030": 0.0, "5045": 1.0, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 1.0, "5080": 0.0, "5083": 0.5, "5085": 1.0, "5086": 1.0, "5090": 0.5, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 1.0, "5178": 0.4, "5185": 0.0, "5196": 0.3333333333333333, "5206": 0.0, "5228": 1.0, "5231": 1.0, "5241": 0.4, "5254": 0.5, "5255": 1.0, "5264": 0.6, "5271": 1.0, "5331": 1.0, "5343": 1.0, "5347": 1.0, "5356": 0.4, "5369": 0.0, "5374": 1.0, "5380": 1.0, "5402": 0.5, "5410": 0.3333333333333333, "5422": 0.75, "5427": 0.5, "5460": 0.0, "5464": 0.3333333333333333, "5503": 0.5, "5505": 1.0, "5511": 0.3, "5534": 0.5, "5549": 0.0, "5585": 1.0, "5592": 1.0, "5616": 1.0, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.5, "5763": 0.6666666666666666, "5782": 0.4, "5790": 0.0, "5808": 1.0, "5853": 0.375, "5862": 0.5, "5888": 0.5, "5903": 0.0, "5906": 1.0, "5940": 0.0, "5951": 0.5555555555555556, "5970": 0.6666666666666666, "5981": 1.0, "5993": 0.2, "6002": 0.9090909090909091, "6004": 0.0, "6005": 0.38461538461538464, "6009": 1.0, "6041": 0.75, "6080": 0.5, "6110": 0.375, "6121": 0.5, "6122": 0.3333333333333333, "6131": 0.4166666666666667, "6133": 0.5, "6142": 0.5, "6146": 1.0, "6199": 0.5, "6219": 1.0, "6221": 0.125, "6252": 0.0, "6262": 0.6666666666666666, "6278": 1.0, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 1.0, "6467": 0.0, "6468": 0.5, "6479": 1.0, "6525": 0.6666666666666666, "6554": 0.5, "6562": 0.5, "6611": 1.0, "6612": 0.3333333333333333, "6625": 1.0, "6629": 0.0, "6635": 0.5, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 1.0, "6683": 1.0, "6713": 0.5, "6715": 1.0, "6746": 0.5, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.75, "6807": 1.0, "6814": 0.0, "6832": 1.0, "6835": 0.5, "6849": 1.0, "6862": 1.0, "6867": 0.42857142857142855, "6875": 0.0, "6890": 1.0, "6891": 0.6666666666666666, "6896": 0.0, "6901": 0.5, "6907": 0.0, "6909": 0.0, "6959": 1.0, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 1.0, "7109": 0.6666666666666666, "7124": 0.75, "7141": 0.5, "7145": 0.3333333333333333, "7178": 1.0, "7188": 1.0, "7205": 1.0, "7206": 0.0, "7218": 1.0, "7221": 0.0, "7269": 1.0, "7279": 0.75, "7295": 0.0, "7311": 1.0, "7326": 0.0, "7329": 1.0, "7344": 0.5, "7345": 0.6666666666666666, "7377": 1.0, "7431": 0.0, "7441": 0.3333333333333333, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.4, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 1.0, "7513": 0.6666666666666666, "7529": 0.5, "7533": 0.5, "7534": 0.3333333333333333, "7590": 1.0, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 0.5, "7674": 0.3333333333333333, "7700": 0.25, "7702": 1.0, "7705": 0.6666666666666666, "7734": 0.5, "7747": 0.5, "7754": 1.0, "7758": 1.0, "7801": 1.0, "7803": 0.5, "7823": 0.6666666666666666, "7876": 1.0, "7879": 0.0, "7880": 0.5, "7911": 1.0, "7925": 0.16666666666666666, "7928": 0.25, "7936": 0.6666666666666666, "7992": 1.0, "8002": 0.6666666666666666, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 1.0, "8040": 1.0, "8072": 0.6666666666666666, "8079": 1.0, "8102": 0.25, "8116": 1.0, "8121": 1.0, "8202": 0.0, "8230": 1.0, "8247": 0.0, "8271": 0.5, "8275": 1.0, "8296": 1.0, "8332": 1.0, "8351": 0.5, "8378": 0.3333333333333333, "8456": 0.6666666666666666, "8475": 0.8571428571428571, "8507": 0.3333333333333333, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 1.0, "8539": 0.25, "8544": 0.5, "8592": 0.6666666666666666, "8632": 1.0, "8635": 0.0, "8702": 0.6666666666666666, "8779": 1.0, "8789": 0.6666666666666666, "8795": 1.0, "8832": 1.0, "8834": 0.5, "8855": 0.0, "8874": 1.0, "8934": 1.0, "8937": 1.0, "8947": 0.5, "8959": 1.0, "8970": 1.0, "8974": 0.25, "8982": 0.0, "9060": 0.5, "9088": 0.0, "9108": 0.0, "9115": 1.0, "9126": 0.0, "9164": 0.3333333333333333, "9174": 0.2, "9188": 0.5, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.5, "9329": 1.0, "9332": 1.0, "9381": 1.0, "9385": 1.0, "9391": 0.75, "9403": 0.0, "9481": 1.0, "9487": 1.0, "9548": 1.0, "9556": 1.0, "9565": 1.0, "9598": 1.0, "9617": 0.6666666666666666, "9633": 0.5, "9643": 1.0, "9644": 1.0, "9646": 1.0, "9668": 1.0, "9701": 0.0, "9733": 0.3333333333333333, "9735": 1.0, "9737": 0.6666666666666666, "9771": 0.0, "9808": 0.3333333333333333, "9824": 1.0, "9871": 0.25, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 0.0, "9979": 0.5, "10034": 0.5, "10039": 1.0, "10109": 0.42857142857142855, "10122": 0.5, "10136": 0.6666666666666666, "10137": 1.0, "10152": 1.0, "10183": 1.0, "10213": 0.3333333333333333, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.3333333333333333, "10462": 0.0, "10482": 0.0, "10497": 0.7, "10526": 0.5, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 1.0, "10639": 0.4, "10645": 1.0, "10674": 1.0, "10710": 1.0, "10734": 1.0, "10792": 1.0, "10808": 1.0, "10809": 0.6666666666666666, "10812": 1.0, "10827": 0.2, "10845": 1.0, "10912": 1.0, "10932": 1.0, "10975": 0.0, "10979": 0.75, "10994": 1.0, "11039": 0.5, "11054": 1.0, "11088": 1.0}, "MRR@10": {"8": 1.0, "15": 0.5, "18": 0.5, "26": 0.2, "34": 0.0, "42": 1.0, "56": 0.16666666666666666, "68": 0.0, "89": 0.5, "90": 0.5, "94": 1.0, "98": 1.0, "104": 1.0, "106": 0.3333333333333333, "109": 1.0, "475": 1.0, "503": 0.07692307692307693, "504": 0.05555555555555555, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.14285714285714285, "559": 0.0, "570": 0.06666666666666667, "585": 0.25, "588": 0.5, "594": 1.0, "603": 0.0, "604": 0.2, "620": 0.08333333333333333, "622": 0.5, "659": 1.0, "672": 0.07692307692307693, "684": 0.05555555555555555, "687": 0.125, "689": 0.0, "691": 0.5, "699": 1.0, "701": 0.1, "715": 0.0, "721": 1.0, "744": 0.2, "750": 1.0, "753": 0.5, "766": 0.2, "776": 1.0, "810": 0.14285714285714285, "813": 0.14285714285714285, "849": 0.14285714285714285, "852": 1.0, "853": 0.09090909090909091, "858": 0.0, "859": 0.5, "864": 1.0, "879": 0.5, "885": 0.125, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.3333333333333333, "939": 0.3333333333333333, "945": 0.1111111111111111, "957": 1.0, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.2, "1150": 0.0, "1157": 1.0, "1159": 0.0, "1198": 0.5, "1230": 0.06666666666666667, "1281": 0.2, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 1.0, "1310": 1.0, "1321": 0.0, "1322": 0.5, "1391": 0.2, "1393": 1.0, "1415": 1.0, "1416": 0.058823529411764705, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.5, "1670": 1.0, "1676": 0.5, "1736": 1.0, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 0.125, "1826": 1.0, "1832": 0.3333333333333333, "1871": 1.0, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.5, "2010": 1.0, "2051": 0.07142857142857142, "2070": 0.2, "2075": 0.5, "2076": 0.5, "2088": 0.0, "2108": 0.16666666666666666, "2118": 0.0, "2154": 0.0, "2181": 0.05263157894736842, "2183": 1.0, "2204": 1.0, "2264": 0.5, "2296": 1.0, "2306": 1.0, "2316": 1.0, "2318": 0.1111111111111111, "2330": 1.0, "2334": 1.0, "2348": 1.0, "2376": 1.0, "2383": 1.0, "2384": 0.5, "2385": 1.0, "2388": 1.0, "2395": 1.0, "2398": 0.08333333333333333, "2399": 1.0, "2400": 0.5, "2407": 0.5, "2416": 1.0, "2423": 1.0, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0625, "2472": 0.0, "2486": 1.0, "2498": 0.25, "2513": 0.0625, "2516": 1.0, "2549": 1.0, "2551": 1.0, "2568": 0.5, "2579": 0.3333333333333333, "2580": 1.0, "2587": 1.0, "2589": 1.0, "2590": 1.0, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 0.3333333333333333, "2685": 1.0, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 0.3333333333333333, "2749": 1.0, "2790": 1.0, "2801": 1.0, "2856": 1.0, "2857": 1.0, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.16666666666666666, "2923": 1.0, "2964": 0.07692307692307693, "2968": 1.0, "2994": 1.0, "3006": 0.3333333333333333, "3008": 0.5, "3014": 1.0, "3033": 1.0, "3039": 0.2, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 0.125, "3091": 1.0, "3103": 1.0, "3115": 1.0, "3125": 1.0, "3148": 1.0, "3149": 0.0, "3177": 0.14285714285714285, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 1.0, "3264": 1.0, "3357": 1.0, "3369": 0.09090909090909091, "3394": 0.5, "3404": 1.0, "3405": 0.09090909090909091, "3446": 0.5, "3451": 0.1, "3453": 0.3333333333333333, "3480": 1.0, "3490": 0.1111111111111111, "3500": 1.0, "3503": 1.0, "3512": 1.0, "3528": 0.5, "3530": 0.5, "3534": 0.0, "3566": 1.0, "3569": 0.3333333333333333, "3594": 0.125, "3612": 0.1111111111111111, "3615": 0.25, "3625": 0.25, "3682": 0.14285714285714285, "3683": 1.0, "3694": 1.0, "3724": 0.0, "3735": 0.1, "3759": 0.0, "3767": 0.3333333333333333, "3771": 1.0, "3781": 1.0, "3789": 1.0, "3791": 0.3333333333333333, "3801": 0.0, "3822": 0.05263157894736842, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 1.0, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 1.0, "4011": 1.0, "4019": 0.5, "4031": 0.0, "4037": 0.1111111111111111, "4047": 1.0, "4071": 0.0, "4084": 0.25, "4102": 0.06666666666666667, "4103": 0.0, "4105": 0.0, "4116": 1.0, "4125": 0.0, "4142": 0.25, "4153": 0.125, "4179": 0.5, "4188": 0.0625, "4205": 0.5, "4233": 1.0, "4265": 1.0, "4286": 1.0, "4289": 0.3333333333333333, "4306": 1.0, "4312": 1.0, "4335": 1.0, "4339": 1.0, "4394": 0.0, "4409": 0.3333333333333333, "4411": 1.0, "4414": 1.0, "4415": 0.0, "4433": 0.09090909090909091, "4447": 1.0, "4464": 0.3333333333333333, "4465": 0.5, "4484": 1.0, "4499": 0.05555555555555555, "4500": 0.07142857142857142, "4504": 1.0, "4514": 1.0, "4523": 0.5, "4539": 0.16666666666666666, "4571": 1.0, "4600": 1.0, "4605": 1.0, "4615": 0.5, "4640": 0.0, "4641": 0.2, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 1.0, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 0.2, "4837": 0.0, "4844": 1.0, "4845": 1.0, "4846": 0.0, "4863": 0.5, "4865": 0.0, "4920": 1.0, "4942": 0.125, "4946": 1.0, "4955": 0.0, "4962": 1.0, "4968": 0.5, "4981": 1.0, "4999": 1.0, "5021": 0.0, "5030": 0.0, "5045": 0.1111111111111111, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.14285714285714285, "5080": 0.0, "5083": 0.2, "5085": 1.0, "5086": 1.0, "5090": 0.25, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 0.09090909090909091, "5178": 0.2, "5185": 0.0, "5196": 0.5, "5206": 0.0, "5228": 0.125, "5231": 0.058823529411764705, "5241": 1.0, "5254": 1.0, "5255": 1.0, "5264": 1.0, "5271": 0.3333333333333333, "5331": 1.0, "5343": 0.14285714285714285, "5347": 1.0, "5356": 0.16666666666666666, "5369": 0.0, "5374": 0.1111111111111111, "5380": 1.0, "5402": 0.0625, "5410": 0.5, "5422": 0.5, "5427": 0.2, "5460": 0.0, "5464": 0.16666666666666666, "5503": 1.0, "5505": 1.0, "5511": 0.25, "5534": 0.5, "5549": 0.0, "5585": 0.125, "5592": 1.0, "5616": 0.16666666666666666, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 0.5, "5710": 0.0, "5741": 0.0625, "5763": 0.3333333333333333, "5782": 1.0, "5790": 0.0, "5808": 0.3333333333333333, "5853": 0.3333333333333333, "5862": 0.25, "5888": 0.25, "5903": 0.0, "5906": 0.25, "5940": 0.0, "5951": 0.5, "5970": 0.3333333333333333, "5981": 1.0, "5993": 1.0, "6002": 1.0, "6004": 0.0, "6005": 1.0, "6009": 0.09090909090909091, "6041": 1.0, "6080": 1.0, "6110": 1.0, "6121": 0.5, "6122": 1.0, "6131": 1.0, "6133": 1.0, "6142": 1.0, "6146": 1.0, "6199": 1.0, "6219": 0.5, "6221": 0.05, "6252": 0.0, "6262": 0.5, "6278": 0.14285714285714285, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.16666666666666666, "6467": 0.0, "6468": 1.0, "6479": 1.0, "6525": 0.5, "6554": 1.0, "6562": 0.1111111111111111, "6611": 1.0, "6612": 0.16666666666666666, "6625": 0.125, "6629": 0.0, "6635": 1.0, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 0.5, "6683": 1.0, "6713": 1.0, "6715": 1.0, "6746": 0.3333333333333333, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 1.0, "6807": 1.0, "6814": 0.0, "6832": 0.2, "6835": 1.0, "6849": 0.14285714285714285, "6862": 1.0, "6867": 0.25, "6875": 0.0, "6890": 0.25, "6891": 1.0, "6896": 0.0, "6901": 0.16666666666666666, "6907": 0.0, "6909": 0.0, "6959": 0.5, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 0.5, "7098": 1.0, "7105": 0.3333333333333333, "7109": 0.3333333333333333, "7124": 1.0, "7141": 0.0625, "7145": 0.1, "7178": 0.3333333333333333, "7188": 0.125, "7205": 1.0, "7206": 0.0, "7218": 0.25, "7221": 0.0, "7269": 0.07142857142857142, "7279": 0.25, "7295": 0.0, "7311": 0.09090909090909091, "7326": 0.0, "7329": 0.25, "7344": 1.0, "7345": 0.14285714285714285, "7377": 1.0, "7431": 0.0, "7441": 0.25, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.25, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 0.0625, "7513": 0.16666666666666666, "7529": 0.3333333333333333, "7533": 0.08333333333333333, "7534": 1.0, "7590": 1.0, "7592": 0.0, "7594": 0.14285714285714285, "7622": 0.0, "7633": 1.0, "7674": 0.0625, "7700": 0.125, "7702": 0.5, "7705": 0.5, "7734": 1.0, "7747": 0.16666666666666666, "7754": 0.08333333333333333, "7758": 1.0, "7801": 1.0, "7803": 0.125, "7823": 1.0, "7876": 0.3333333333333333, "7879": 0.0, "7880": 0.06666666666666667, "7911": 1.0, "7925": 0.5, "7928": 0.5, "7936": 0.5, "7992": 0.5, "8002": 1.0, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.125, "8040": 1.0, "8072": 1.0, "8079": 1.0, "8102": 0.1, "8116": 0.25, "8121": 1.0, "8202": 0.0, "8230": 0.0625, "8247": 0.0, "8271": 1.0, "8275": 0.25, "8296": 1.0, "8332": 1.0, "8351": 1.0, "8378": 0.125, "8456": 1.0, "8475": 1.0, "8507": 0.09090909090909091, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 0.16666666666666666, "8539": 0.09090909090909091, "8544": 1.0, "8592": 0.3333333333333333, "8632": 0.25, "8635": 0.0, "8702": 1.0, "8779": 0.25, "8789": 1.0, "8795": 1.0, "8832": 1.0, "8834": 1.0, "8855": 0.0, "8874": 0.125, "8934": 1.0, "8937": 0.25, "8947": 0.2, "8959": 0.2, "8970": 0.5, "8974": 1.0, "8982": 0.0, "9060": 1.0, "9088": 0.0, "9108": 0.0, "9115": 0.3333333333333333, "9126": 0.0, "9164": 0.16666666666666666, "9174": 0.07142857142857142, "9188": 1.0, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.14285714285714285, "9329": 1.0, "9332": 0.25, "9381": 0.5, "9385": 0.16666666666666666, "9391": 0.3333333333333333, "9403": 0.0, "9481": 0.3333333333333333, "9487": 0.08333333333333333, "9548": 0.25, "9556": 1.0, "9565": 1.0, "9598": 0.16666666666666666, "9617": 0.16666666666666666, "9633": 1.0, "9643": 0.3333333333333333, "9644": 0.5, "9646": 0.25, "9668": 1.0, "9701": 0.0, "9733": 0.2, "9735": 0.14285714285714285, "9737": 1.0, "9771": 0.0, "9808": 0.5, "9824": 1.0, "9871": 0.5, "9882": 0.0, "9925": 1.0, "9929": 0.5, "9961": 0.0, "9979": 1.0, "10034": 0.5, "10039": 0.5, "10109": 1.0, "10122": 0.3333333333333333, "10136": 0.25, "10137": 0.2, "10152": 1.0, "10183": 0.3333333333333333, "10213": 1.0, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.2, "10462": 0.0, "10482": 0.0, "10497": 1.0, "10526": 0.14285714285714285, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 0.1111111111111111, "10639": 0.5, "10645": 0.5, "10674": 1.0, "10710": 0.14285714285714285, "10734": 1.0, "10792": 1.0, "10808": 0.2, "10809": 1.0, "10812": 1.0, "10827": 0.25, "10845": 1.0, "10912": 0.08333333333333333, "10932": 1.0, "10975": 0.0, "10979": 1.0, "10994": 0.3333333333333333, "11039": 0.5, "11054": 1.0, "11088": 0.25}}} diff --git a/evals/benchmark/results-adaptive/fiqa/ir-adm0.30.jsonl b/evals/benchmark/results-adaptive/fiqa/ir-adm0.30.jsonl new file mode 100644 index 000000000..faddfe7dd --- /dev/null +++ b/evals/benchmark/results-adaptive/fiqa/ir-adm0.30.jsonl @@ -0,0 +1 @@ +{"dataset": "fiqa", "run_tag": "adm0.30", "aggregated": {"nDCG@10": 0.4323, "Recall@20": 0.6152, "MRR@10": 0.5032}, "per_query": {"nDCG@10": {"8": 0.6131471927654584, "15": 0.3562071871080222, "18": 0.6309297535714575, "26": 0.23719771276929622, "34": 0.0, "42": 0.46927872602275644, "56": 0.3562071871080222, "68": 0.0, "89": 0.286381299268402, "90": 0.6309297535714575, "94": 1.0, "98": 0.6131471927654584, "104": 0.6131471927654584, "106": 0.5, "109": 1.0, "475": 1.0, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.3333333333333333, "559": 0.0, "570": 0.0, "585": 0.2640681225725909, "588": 0.5714285040141098, "594": 0.6131471927654584, "603": 0.0, "604": 0.34870224750355494, "620": 0.0, "622": 0.38685280723454163, "659": 0.43828689407543014, "672": 0.0, "684": 0.0, "687": 0.19342640361727081, "689": 0.0, "691": 0.6309297535714575, "699": 1.0, "701": 0.13565197343244778, "715": 0.0, "721": 1.0, "744": 0.18154179253735267, "750": 0.8503449055347546, "753": 0.38685280723454163, "766": 0.18154179253735267, "776": 0.3557772116892465, "810": 0.3333333333333333, "813": 0.20438239758848611, "849": 0.3333333333333333, "852": 0.7653606369886217, "853": 0.0, "858": 0.0, "859": 0.38685280723454163, "864": 0.7247145167543899, "879": 0.6309297535714575, "885": 0.19342640361727081, "895": 1.0, "904": 1.0, "928": 0.6131471927654584, "929": 1.0, "932": 0.3065735963827292, "939": 0.5, "945": 0.3010299956639812, "957": 0.7653606369886217, "988": 0.0, "1074": 0.6309297535714575, "1085": 0.0, "1090": 0.23719771276929622, "1150": 0.0, "1157": 0.6131471927654584, "1159": 0.0, "1198": 1.0, "1230": 0.0, "1281": 0.38685280723454163, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.5319460297682747, "1310": 1.0, "1321": 0.0, "1322": 0.24630238874073, "1391": 0.18154179253735267, "1393": 0.19092086617893467, "1415": 0.9709286432396583, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.431733884398313, "1670": 0.7653606369886217, "1676": 0.6542448095688422, "1736": 0.5531464700081437, "1748": 0.6105456988825855, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.286381299268402, "1824": 0.31546487678572877, "1826": 1.0, "1832": 0.5, "1871": 0.6422813708518126, "1877": 1.0, "1889": 0.0, "1915": 0.8772153153380493, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.21398626473452756, "2010": 0.8194270280062366, "2051": 0.0, "2070": 0.23463936301137822, "2075": 0.4880470105041527, "2076": 0.6934264036172708, "2088": 0.0, "2108": 0.16716045496620227, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 0.7526136409516659, "2204": 0.9830168297915423, "2264": 0.38685280723454163, "2296": 0.33627415762678553, "2306": 0.6131471927654584, "2316": 1.0, "2318": 0.3010299956639812, "2330": 0.6131471927654584, "2334": 0.46927872602275644, "2348": 0.43231813227099475, "2376": 0.8910635073822442, "2383": 0.6131471927654584, "2384": 0.6309297535714575, "2385": 0.6131471927654584, "2388": 1.0, "2395": 1.0, "2398": 0.0, "2399": 1.0, "2400": 0.5307212739772434, "2407": 0.24630238874073, "2416": 0.9058653015743079, "2423": 0.3576223542196109, "2443": 0.0, "2445": 1.0, "2460": 0.9469024295259745, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 0.43067655807339306, "2513": 0.0, "2516": 0.5585075862632192, "2549": 0.6131471927654584, "2551": 1.0, "2568": 0.21689524588991285, "2579": 0.2214161691338287, "2580": 0.46927872602275644, "2587": 1.0, "2589": 0.7653606369886217, "2590": 0.46927872602275644, "2593": 0.8854598815714874, "2598": 0.8503449055347546, "2648": 1.0, "2676": 0.5, "2685": 0.35079429740281753, "2695": 1.0, "2713": 0.6131471927654584, "2724": 0.8175295903539445, "2737": 1.0, "2747": 0.5, "2749": 0.8772153153380493, "2790": 0.47997295436377346, "2801": 0.6131471927654584, "2856": 0.7095272044910244, "2857": 0.6595629128335778, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.3010299956639812, "2923": 0.8207658584763088, "2964": 0.0, "2968": 0.8772153153380493, "2994": 0.46927872602275644, "3006": 0.25909036127391766, "3008": 0.24630238874073, "3014": 1.0, "3033": 0.38685280723454163, "3039": 0.43067655807339306, "3049": 0.24630238874073, "3051": 1.0, "3067": 0.0, "3085": 0.19342640361727081, "3091": 0.9020870746500321, "3103": 1.0, "3115": 0.4935232796777481, "3125": 1.0, "3148": 0.5531464700081437, "3149": 0.0, "3177": 0.15642624200758548, "3179": 0.8503449055347546, "3186": 1.0, "3189": 0.0, "3254": 0.6713860725233041, "3264": 0.8048099750039491, "3357": 1.0, "3369": 0.0, "3394": 0.5805485932475565, "3404": 0.3932591258696538, "3405": 0.0, "3446": 0.4340327988596634, "3451": 0.07945707943276742, "3453": 0.5706417189553201, "3480": 1.0, "3490": 0.2890648263178879, "3500": 0.644824486427155, "3503": 0.6183313187362977, "3512": 0.8503449055347546, "3528": 0.6309297535714575, "3530": 0.4776237035032179, "3534": 0.0, "3566": 0.8194270280062366, "3569": 0.5, "3594": 0.123151194370365, "3612": 0.11751610475642714, "3615": 0.430624116386567, "3625": 0.20210734650054757, "3682": 0.20438239758848611, "3683": 0.6324677375730864, "3694": 0.6131471927654584, "3724": 0.0, "3735": 0.0, "3759": 0.0, "3767": 0.2716774977597197, "3771": 0.19632790425573848, "3781": 0.9197207891481876, "3789": 0.8048099750039491, "3791": 0.370291336443826, "3801": 0.0, "3822": 0.0, "3829": 0.0, "3830": 1.0, "3837": 0.6309297535714575, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5517854393872873, "3932": 0.0, "3934": 0.16812753627111746, "3995": 0.0, "4007": 0.7039180890341347, "4011": 0.9674679834891693, "4019": 0.31608365985743925, "4031": 0.0, "4037": 0.18457569677956817, "4047": 0.8318724637288826, "4071": 0.0, "4084": 0.5012658353418871, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 0.6131471927654584, "4125": 0.0, "4142": 0.39233405721184117, "4153": 0.31546487678572877, "4179": 0.6309297535714575, "4188": 0.0, "4205": 0.4525081529734507, "4233": 0.8687949224876582, "4265": 0.7751482523375255, "4286": 0.6131471927654584, "4289": 0.23719771276929622, "4306": 0.7598336331031966, "4312": 0.7237478018511487, "4335": 0.8318724637288826, "4339": 1.0, "4394": 0.0, "4409": 0.19518900079066107, "4411": 0.46927872602275644, "4414": 0.8315546295836225, "4415": 0.0, "4433": 0.0, "4447": 1.0, "4464": 0.5, "4465": 0.2960819109658652, "4484": 0.6309297535714575, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 0.4034698846650846, "4523": 0.21398626473452756, "4539": 0.21840743681816419, "4571": 0.7653606369886217, "4600": 1.0, "4605": 0.5585830187272559, "4615": 0.4816421678943211, "4640": 0.0, "4641": 0.252016380211836, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75369761125927, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.6131471927654584, "4813": 1.0, "4823": 0.46927872602275644, "4827": 0.3562071871080222, "4837": 0.0, "4844": 1.0, "4845": 0.7653606369886217, "4846": 0.0, "4863": 0.49818925746641285, "4865": 0.0, "4920": 1.0, "4942": 0.31546487678572877, "4946": 1.0, "4955": 0.0, "4962": 0.6131471927654584, "4968": 0.6309297535714575, "4981": 0.7039180890341347, "4999": 0.5294362295028773, "5021": 0.0, "5030": 0.0, "5045": 0.3010299956639812, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.6309297535714575, "5080": 0.0, "5083": 0.23719771276929622, "5085": 1.0, "5086": 1.0, "5090": 0.2640681225725909, "5125": 1.0, "5134": 0.38685280723454163, "5150": 0.0, "5155": 1.0, "5172": 0.0, "5178": 0.252016380211836, "5185": 0.0, "5196": 0.2960819109658652, "5206": 0.0, "5228": 0.31546487678572877, "5231": 0.0, "5241": 0.44615333764087994, "5254": 0.6131471927654584, "5255": 1.0, "5264": 0.6843515475204855, "5271": 0.41618115554873086, "5331": 1.0, "5343": 0.3333333333333333, "5347": 1.0, "5356": 0.1208113026994942, "5369": 0.0, "5374": 0.3010299956639812, "5380": 1.0, "5402": 0.0, "5410": 0.2960819109658652, "5422": 0.19519002499605084, "5427": 0.23719771276929622, "5460": 0.0, "5464": 0.16716045496620227, "5503": 0.38685280723454163, "5505": 1.0, "5511": 0.1610425878239398, "5534": 0.38685280723454163, "5549": 0.0, "5585": 0.31546487678572877, "5592": 1.0, "5616": 0.3562071871080222, "5620": 0.0, "5646": 0.9217868789962071, "5653": 1.0, "5683": 0.9197207891481876, "5710": 0.0, "5741": 0.0, "5763": 0.23463936301137822, "5782": 0.4371994911049732, "5790": 0.0, "5808": 0.5, "5853": 0.21078558563971425, "5862": 0.29127873064148246, "5888": 0.3065735963827292, "5903": 0.0, "5906": 0.44130740935663865, "5940": 0.0, "5951": 0.42030067999766585, "5970": 0.23463936301137822, "5981": 0.9896062251871525, "5993": 0.31488013066763093, "6002": 0.762416491594312, "6004": 0.0, "6005": 0.5388856921828065, "6009": 0.0, "6041": 0.7668091220635322, "6080": 0.6131471927654584, "6110": 0.41918605213740834, "6121": 0.38685280723454163, "6122": 0.46927872602275644, "6131": 0.5810820381115676, "6133": 0.6131471927654584, "6142": 0.6131471927654584, "6146": 0.8315546295836225, "6199": 0.5135312443624667, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.49818925746641285, "6278": 0.2640681225725909, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.38685280723454163, "6467": 0.0, "6468": 0.6131471927654584, "6479": 1.0, "6525": 0.49818925746641285, "6554": 0.6131471927654584, "6562": 0.11751610475642714, "6611": 0.7903864795495061, "6612": 0.16716045496620227, "6625": 0.0, "6629": 0.0, "6635": 0.6366824387328317, "6644": 0.0, "6647": 0.7653606369886217, "6668": 0.8503449055347546, "6679": 0.4632423659320675, "6683": 0.8503449055347546, "6713": 0.6131471927654584, "6715": 0.6131471927654584, "6746": 0.2640681225725909, "6787": 0.0, "6792": 0.0, "6800": 0.43067655807339306, "6803": 0.5205067333228022, "6807": 1.0, "6814": 0.0, "6832": 0.430624116386567, "6835": 0.6131471927654584, "6849": 0.43067655807339306, "6862": 1.0, "6867": 0.22471947398559278, "6875": 0.0, "6890": 0.5012658353418871, "6891": 0.617319681505689, "6896": 0.0, "6901": 0.21840743681816419, "6907": 0.0, "6909": 0.0, "6959": 0.38685280723454163, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 0.8772153153380493, "7096": 0.6309297535714575, "7098": 0.6131471927654584, "7105": 0.38685280723454163, "7109": 0.23463936301137822, "7124": 0.6525874238732422, "7141": 0.0, "7145": 0.13565197343244778, "7178": 0.3333333333333333, "7188": 0.31546487678572877, "7205": 1.0, "7206": 0.0, "7218": 0.43067655807339306, "7221": 0.0, "7269": 0.0, "7279": 0.44229834886928765, "7295": 0.0, "7311": 0.0, "7326": 0.0, "7329": 0.46845052016107697, "7344": 0.6131471927654584, "7345": 0.15642624200758548, "7377": 1.0, "7431": 0.0, "7441": 0.20210734650054757, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.14606834984270645, "7467": 0.6309297535714575, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.16716045496620227, "7529": 0.3065735963827292, "7533": 0.0, "7534": 0.46927872602275644, "7590": 0.5714285040141098, "7592": 0.0, "7594": 0.3333333333333333, "7622": 0.0, "7633": 0.3903800499921017, "7674": 0.0, "7700": 0.123151194370365, "7702": 0.38685280723454163, "7705": 0.4367467095119258, "7734": 0.6131471927654584, "7747": 0.20438239758848611, "7754": 0.0, "7758": 0.6309297535714575, "7801": 1.0, "7803": 0.19342640361727081, "7823": 0.6105456988825855, "7876": 0.43067655807339306, "7879": 0.0, "7880": 0.19342640361727081, "7911": 1.0, "7925": 0.19092086617893467, "7928": 0.24630238874073, "7936": 0.39106560501896365, "7992": 0.38685280723454163, "8002": 0.617319681505689, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.31546487678572877, "8040": 1.0, "8072": 0.4776237035032179, "8079": 0.7039180890341347, "8102": 0.11284514134893527, "8116": 0.45749452618986164, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 0.6131471927654584, "8275": 0.43067655807339306, "8296": 0.8315546295836225, "8332": 0.9060254355346823, "8351": 0.6131471927654584, "8378": 0.14126697285982898, "8456": 0.6105456988825855, "8475": 0.7957503936198023, "8507": 0.0, "8512": 1.0, "8513": 0.0, "8532": 0.9325210919548239, "8537": 0.3562071871080222, "8539": 0.0, "8544": 0.6131471927654584, "8592": 0.23463936301137822, "8632": 0.21840743681816419, "8635": 0.0, "8702": 0.7653606369886217, "8779": 0.0, "8789": 0.7653606369886217, "8795": 1.0, "8832": 1.0, "8834": 0.3980628465882808, "8855": 0.0, "8874": 0.3562071871080222, "8934": 0.8065735963827293, "8937": 0.3562071871080222, "8947": 0.23719771276929622, "8959": 0.43067655807339306, "8970": 0.8772153153380493, "8974": 0.41253177989255346, "8982": 0.0, "9060": 0.6131471927654584, "9088": 0.0, "9108": 0.0, "9115": 0.5256940434743352, "9126": 0.0, "9164": 0.1480409554829326, "9174": 0.0, "9188": 0.6131471927654584, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.20438239758848611, "9329": 1.0, "9332": 0.3562071871080222, "9381": 0.5706417189553201, "9385": 0.3562071871080222, "9391": 0.5143371794949736, "9403": 0.0, "9481": 0.5706417189553201, "9487": 0.0, "9548": 0.38685280723454163, "9556": 1.0, "9565": 1.0, "9598": 0.3562071871080222, "9617": 0.3152014104491349, "9633": 0.6131471927654584, "9643": 0.5, "9644": 0.6309297535714575, "9646": 0.448643819352159, "9668": 0.6508205185601091, "9701": 0.0, "9733": 0.18154179253735267, "9735": 0.3333333333333333, "9737": 0.7653606369886217, "9771": 0.0, "9808": 0.2960819109658652, "9824": 1.0, "9871": 0.24630238874073, "9882": 0.0, "9925": 0.8175295903539445, "9929": 0.5714285040141098, "9961": 0.0, "9979": 0.3903800499921017, "10034": 0.3065735963827292, "10039": 0.6309297535714575, "10109": 0.5399294342072939, "10122": 0.19519002499605084, "10136": 0.34337431936037655, "10137": 0.3562071871080222, "10152": 1.0, "10183": 0.3065735963827292, "10213": 0.46927872602275644, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.18154179253735267, "10462": 0.0, "10482": 0.0, "10497": 0.49828993591603105, "10526": 0.20438239758848611, "10547": 0.0, "10558": 0.0, "10596": 0.9427731067700877, "10601": 1.0, "10628": 0.3010299956639812, "10639": 0.21398626473452756, "10645": 0.38685280723454163, "10674": 0.7095272044910244, "10710": 0.3333333333333333, "10734": 0.8486987932505933, "10792": 0.4415801103577823, "10808": 0.3295827480202853, "10809": 0.6049306994552042, "10812": 0.9619991470595832, "10827": 0.14606834984270645, "10845": 1.0, "10912": 0.0, "10932": 1.0, "10975": 0.0, "10979": 0.6366824387328317, "10994": 0.5, "11039": 0.315526956404728, "11054": 1.0, "11088": 0.43067655807339306}, "Recall@20": {"8": 0.5, "15": 1.0, "18": 1.0, "26": 0.5, "34": 0.0, "42": 0.3333333333333333, "56": 1.0, "68": 0.0, "89": 0.5, "90": 1.0, "94": 1.0, "98": 0.5, "104": 0.5, "106": 1.0, "109": 1.0, "475": 1.0, "503": 1.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 1.0, "585": 0.5, "588": 1.0, "594": 1.0, "603": 0.0, "604": 1.0, "620": 0.2, "622": 0.5, "659": 0.7, "672": 0.5, "684": 1.0, "687": 0.5, "689": 0.0, "691": 1.0, "699": 1.0, "701": 0.3333333333333333, "715": 0.0, "721": 1.0, "744": 0.3333333333333333, "750": 1.0, "753": 0.5, "766": 0.3333333333333333, "776": 0.25, "810": 1.0, "813": 1.0, "849": 1.0, "852": 0.6666666666666666, "853": 0.5, "858": 0.0, "859": 0.5, "864": 0.6666666666666666, "879": 1.0, "885": 0.5, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.5, "939": 1.0, "945": 1.0, "957": 0.6666666666666666, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.5, "1150": 0.0, "1157": 0.5, "1159": 0.0, "1198": 1.0, "1230": 1.0, "1281": 1.0, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.75, "1310": 1.0, "1321": 0.0, "1322": 0.25, "1391": 0.3333333333333333, "1393": 0.5, "1415": 1.0, "1416": 1.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 1.0, "1670": 0.6666666666666666, "1676": 1.0, "1736": 0.4, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 1.0, "1826": 1.0, "1832": 1.0, "1871": 0.75, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.4, "2010": 1.0, "2051": 1.0, "2070": 0.6666666666666666, "2075": 0.5555555555555556, "2076": 1.0, "2088": 0.0, "2108": 0.3333333333333333, "2118": 0.0, "2154": 0.0, "2181": 0.5, "2183": 0.8333333333333334, "2204": 1.0, "2264": 0.5, "2296": 0.3333333333333333, "2306": 0.5, "2316": 1.0, "2318": 1.0, "2330": 0.5, "2334": 0.6666666666666666, "2348": 0.26666666666666666, "2376": 1.0, "2383": 0.5, "2384": 1.0, "2385": 0.5, "2388": 1.0, "2395": 1.0, "2398": 0.2, "2399": 1.0, "2400": 0.6666666666666666, "2407": 0.5, "2416": 1.0, "2423": 0.8571428571428571, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.3333333333333333, "2472": 0.0, "2486": 1.0, "2498": 1.0, "2513": 1.0, "2516": 0.75, "2549": 0.5, "2551": 1.0, "2568": 0.2857142857142857, "2579": 0.5, "2580": 0.6666666666666666, "2587": 1.0, "2589": 0.6666666666666666, "2590": 0.6666666666666666, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 1.0, "2685": 0.25, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 1.0, "2790": 0.42857142857142855, "2801": 1.0, "2856": 0.75, "2857": 0.75, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 1.0, "2923": 0.8, "2964": 0.5, "2968": 1.0, "2994": 0.3333333333333333, "3006": 0.3333333333333333, "3008": 0.25, "3014": 1.0, "3033": 0.5, "3039": 1.0, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 1.0, "3091": 0.8571428571428571, "3103": 1.0, "3115": 0.5, "3125": 1.0, "3148": 0.6, "3149": 0.0, "3177": 0.3333333333333333, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.6666666666666666, "3264": 0.75, "3357": 1.0, "3369": 0.16666666666666666, "3394": 0.75, "3404": 0.42857142857142855, "3405": 1.0, "3446": 0.8, "3451": 0.42857142857142855, "3453": 1.0, "3480": 1.0, "3490": 1.0, "3500": 0.5, "3503": 0.7142857142857143, "3512": 1.0, "3528": 1.0, "3530": 0.6666666666666666, "3534": 0.0, "3566": 1.0, "3569": 1.0, "3594": 0.25, "3612": 0.75, "3615": 1.0, "3625": 0.3333333333333333, "3682": 1.0, "3683": 0.6666666666666666, "3694": 0.5, "3724": 0.0, "3735": 1.0, "3759": 0.0, "3767": 0.4, "3771": 0.3333333333333333, "3781": 1.0, "3789": 0.75, "3791": 0.6666666666666666, "3801": 0.5, "3822": 0.0, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 0.6666666666666666, "4011": 1.0, "4019": 0.4, "4031": 0.0, "4037": 0.5, "4047": 0.75, "4071": 0.0, "4084": 1.0, "4102": 0.25, "4103": 0.0, "4105": 0.0, "4116": 0.5, "4125": 0.0, "4142": 0.75, "4153": 1.0, "4179": 1.0, "4188": 1.0, "4205": 0.6666666666666666, "4233": 0.8, "4265": 0.6666666666666666, "4286": 0.5, "4289": 0.5, "4306": 0.75, "4312": 0.8571428571428571, "4335": 0.75, "4339": 1.0, "4394": 0.0, "4409": 0.2, "4411": 0.6666666666666666, "4414": 1.0, "4415": 0.0, "4433": 0.5, "4447": 1.0, "4464": 1.0, "4465": 0.6666666666666666, "4484": 1.0, "4499": 0.0, "4500": 0.5, "4504": 1.0, "4514": 0.5, "4523": 0.2, "4539": 0.5, "4571": 0.6666666666666666, "4600": 1.0, "4605": 0.7142857142857143, "4615": 0.6666666666666666, "4640": 0.0, "4641": 0.6, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 0.6666666666666666, "4846": 0.0, "4863": 1.0, "4865": 0.0, "4920": 1.0, "4942": 1.0, "4946": 1.0, "4955": 0.0, "4962": 0.5, "4968": 1.0, "4981": 0.6666666666666666, "4999": 0.75, "5021": 0.0, "5030": 0.0, "5045": 1.0, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 1.0, "5080": 0.0, "5083": 0.5, "5085": 1.0, "5086": 1.0, "5090": 0.5, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 0.5, "5178": 0.4, "5185": 0.0, "5196": 0.3333333333333333, "5206": 0.0, "5228": 1.0, "5231": 0.0, "5241": 0.4, "5254": 0.5, "5255": 1.0, "5264": 0.6, "5271": 1.0, "5331": 1.0, "5343": 1.0, "5347": 1.0, "5356": 0.4, "5369": 0.0, "5374": 1.0, "5380": 1.0, "5402": 0.5, "5410": 0.3333333333333333, "5422": 0.75, "5427": 0.5, "5460": 0.0, "5464": 0.3333333333333333, "5503": 0.5, "5505": 1.0, "5511": 0.3, "5534": 0.5, "5549": 0.0, "5585": 1.0, "5592": 1.0, "5616": 1.0, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.5, "5763": 0.6666666666666666, "5782": 0.4, "5790": 0.0, "5808": 1.0, "5853": 0.25, "5862": 0.5, "5888": 0.5, "5903": 0.0, "5906": 1.0, "5940": 0.0, "5951": 0.5555555555555556, "5970": 0.6666666666666666, "5981": 1.0, "5993": 0.2, "6002": 0.9090909090909091, "6004": 0.0, "6005": 0.38461538461538464, "6009": 1.0, "6041": 0.75, "6080": 0.5, "6110": 0.5, "6121": 0.5, "6122": 0.3333333333333333, "6131": 0.4166666666666667, "6133": 0.5, "6142": 0.5, "6146": 1.0, "6199": 0.5, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.6666666666666666, "6278": 1.0, "6395": 0.0, "6410": 1.0, "6420": 1.0, "6441": 1.0, "6467": 0.0, "6468": 0.5, "6479": 1.0, "6525": 0.6666666666666666, "6554": 0.5, "6562": 0.5, "6611": 1.0, "6612": 0.3333333333333333, "6625": 1.0, "6629": 0.0, "6635": 0.5, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 1.0, "6683": 1.0, "6713": 0.5, "6715": 1.0, "6746": 0.5, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.75, "6807": 1.0, "6814": 0.0, "6832": 1.0, "6835": 0.5, "6849": 1.0, "6862": 1.0, "6867": 0.42857142857142855, "6875": 0.0, "6890": 1.0, "6891": 0.6666666666666666, "6896": 0.0, "6901": 0.5, "6907": 0.0, "6909": 0.0, "6959": 1.0, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 1.0, "7109": 0.6666666666666666, "7124": 0.75, "7141": 0.0, "7145": 0.3333333333333333, "7178": 1.0, "7188": 1.0, "7205": 1.0, "7206": 0.0, "7218": 1.0, "7221": 0.0, "7269": 1.0, "7279": 0.75, "7295": 0.0, "7311": 1.0, "7326": 0.0, "7329": 1.0, "7344": 0.5, "7345": 0.6666666666666666, "7377": 1.0, "7431": 0.0, "7441": 0.3333333333333333, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.4, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 1.0, "7513": 0.3333333333333333, "7529": 0.5, "7533": 0.5, "7534": 0.3333333333333333, "7590": 1.0, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 0.5, "7674": 0.0, "7700": 0.25, "7702": 1.0, "7705": 0.6666666666666666, "7734": 0.5, "7747": 0.5, "7754": 1.0, "7758": 1.0, "7801": 1.0, "7803": 0.5, "7823": 0.6666666666666666, "7876": 1.0, "7879": 0.0, "7880": 0.5, "7911": 1.0, "7925": 0.16666666666666666, "7928": 0.25, "7936": 0.6666666666666666, "7992": 1.0, "8002": 0.6666666666666666, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 1.0, "8040": 1.0, "8072": 0.6666666666666666, "8079": 1.0, "8102": 0.25, "8116": 1.0, "8121": 1.0, "8202": 0.0, "8230": 1.0, "8247": 0.0, "8271": 0.5, "8275": 1.0, "8296": 1.0, "8332": 1.0, "8351": 0.5, "8378": 0.3333333333333333, "8456": 0.6666666666666666, "8475": 0.8571428571428571, "8507": 0.3333333333333333, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 1.0, "8539": 0.25, "8544": 0.5, "8592": 0.6666666666666666, "8632": 1.0, "8635": 0.0, "8702": 0.6666666666666666, "8779": 1.0, "8789": 0.6666666666666666, "8795": 1.0, "8832": 1.0, "8834": 0.5, "8855": 0.0, "8874": 1.0, "8934": 1.0, "8937": 1.0, "8947": 0.5, "8959": 1.0, "8970": 1.0, "8974": 0.25, "8982": 0.0, "9060": 0.5, "9088": 0.0, "9108": 0.0, "9115": 1.0, "9126": 0.0, "9164": 0.3333333333333333, "9174": 0.2, "9188": 0.5, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.5, "9329": 1.0, "9332": 1.0, "9381": 1.0, "9385": 1.0, "9391": 0.75, "9403": 0.0, "9481": 1.0, "9487": 1.0, "9548": 1.0, "9556": 1.0, "9565": 1.0, "9598": 1.0, "9617": 0.6666666666666666, "9633": 0.5, "9643": 1.0, "9644": 1.0, "9646": 1.0, "9668": 1.0, "9701": 0.0, "9733": 0.3333333333333333, "9735": 1.0, "9737": 0.6666666666666666, "9771": 0.0, "9808": 0.3333333333333333, "9824": 1.0, "9871": 0.25, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 0.0, "9979": 0.5, "10034": 0.5, "10039": 1.0, "10109": 0.42857142857142855, "10122": 0.5, "10136": 0.6666666666666666, "10137": 1.0, "10152": 1.0, "10183": 1.0, "10213": 0.3333333333333333, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.3333333333333333, "10462": 0.0, "10482": 0.0, "10497": 0.7, "10526": 0.5, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 1.0, "10639": 0.4, "10645": 1.0, "10674": 1.0, "10710": 1.0, "10734": 1.0, "10792": 1.0, "10808": 1.0, "10809": 0.6666666666666666, "10812": 1.0, "10827": 0.2, "10845": 1.0, "10912": 1.0, "10932": 1.0, "10975": 0.0, "10979": 0.75, "10994": 1.0, "11039": 0.5, "11054": 1.0, "11088": 1.0}, "MRR@10": {"8": 1.0, "15": 0.16666666666666666, "18": 0.5, "26": 0.2, "34": 0.0, "42": 1.0, "56": 0.16666666666666666, "68": 0.0, "89": 0.5, "90": 0.5, "94": 1.0, "98": 1.0, "104": 1.0, "106": 0.3333333333333333, "109": 1.0, "475": 1.0, "503": 0.07692307692307693, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.14285714285714285, "559": 0.0, "570": 0.06666666666666667, "585": 0.25, "588": 0.5, "594": 1.0, "603": 0.0, "604": 0.2, "620": 0.08333333333333333, "622": 0.5, "659": 1.0, "672": 0.07692307692307693, "684": 0.05555555555555555, "687": 0.125, "689": 0.0, "691": 0.5, "699": 1.0, "701": 0.1, "715": 0.0, "721": 1.0, "744": 0.2, "750": 1.0, "753": 0.5, "766": 0.2, "776": 1.0, "810": 0.14285714285714285, "813": 0.14285714285714285, "849": 0.14285714285714285, "852": 1.0, "853": 0.09090909090909091, "858": 0.0, "859": 0.5, "864": 1.0, "879": 0.5, "885": 0.125, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.3333333333333333, "939": 0.3333333333333333, "945": 0.1111111111111111, "957": 1.0, "988": 0.0, "1074": 0.5, "1085": 0.0, "1090": 0.2, "1150": 0.0, "1157": 1.0, "1159": 0.0, "1198": 1.0, "1230": 0.06666666666666667, "1281": 0.2, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.5, "1310": 1.0, "1321": 0.0, "1322": 0.5, "1391": 0.2, "1393": 0.5, "1415": 1.0, "1416": 0.058823529411764705, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.5, "1670": 1.0, "1676": 1.0, "1736": 1.0, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 0.125, "1826": 1.0, "1832": 0.3333333333333333, "1871": 1.0, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.5, "2010": 1.0, "2051": 0.07142857142857142, "2070": 0.3333333333333333, "2075": 1.0, "2076": 0.5, "2088": 0.0, "2108": 0.16666666666666666, "2118": 0.0, "2154": 0.0, "2181": 0.05263157894736842, "2183": 1.0, "2204": 1.0, "2264": 0.5, "2296": 1.0, "2306": 1.0, "2316": 1.0, "2318": 0.1111111111111111, "2330": 1.0, "2334": 1.0, "2348": 1.0, "2376": 1.0, "2383": 1.0, "2384": 0.5, "2385": 1.0, "2388": 1.0, "2395": 1.0, "2398": 0.08333333333333333, "2399": 1.0, "2400": 0.5, "2407": 0.5, "2416": 1.0, "2423": 1.0, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0625, "2472": 0.0, "2486": 1.0, "2498": 0.25, "2513": 0.05, "2516": 1.0, "2549": 1.0, "2551": 1.0, "2568": 0.3333333333333333, "2579": 0.25, "2580": 1.0, "2587": 1.0, "2589": 1.0, "2590": 1.0, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 0.3333333333333333, "2685": 1.0, "2695": 1.0, "2713": 1.0, "2724": 1.0, "2737": 1.0, "2747": 0.3333333333333333, "2749": 1.0, "2790": 1.0, "2801": 1.0, "2856": 1.0, "2857": 1.0, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.1111111111111111, "2923": 1.0, "2964": 0.07692307692307693, "2968": 1.0, "2994": 1.0, "3006": 0.3333333333333333, "3008": 0.5, "3014": 1.0, "3033": 0.5, "3039": 0.25, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 0.125, "3091": 1.0, "3103": 1.0, "3115": 1.0, "3125": 1.0, "3148": 1.0, "3149": 0.0, "3177": 0.14285714285714285, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 1.0, "3264": 1.0, "3357": 1.0, "3369": 0.09090909090909091, "3394": 0.5, "3404": 1.0, "3405": 0.09090909090909091, "3446": 0.5, "3451": 0.1, "3453": 0.3333333333333333, "3480": 1.0, "3490": 0.1, "3500": 1.0, "3503": 0.5, "3512": 1.0, "3528": 0.5, "3530": 0.5, "3534": 0.0, "3566": 1.0, "3569": 0.3333333333333333, "3594": 0.125, "3612": 0.1111111111111111, "3615": 0.2, "3625": 0.25, "3682": 0.14285714285714285, "3683": 1.0, "3694": 1.0, "3724": 0.0, "3735": 0.09090909090909091, "3759": 0.0, "3767": 0.3333333333333333, "3771": 0.14285714285714285, "3781": 1.0, "3789": 1.0, "3791": 0.3333333333333333, "3801": 0.07142857142857142, "3822": 0.0, "3829": 0.0, "3830": 1.0, "3837": 0.5, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 1.0, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 1.0, "4011": 1.0, "4019": 0.5, "4031": 0.0, "4037": 0.1111111111111111, "4047": 1.0, "4071": 0.0, "4084": 0.25, "4102": 0.06666666666666667, "4103": 0.0, "4105": 0.0, "4116": 1.0, "4125": 0.0, "4142": 0.16666666666666666, "4153": 0.125, "4179": 0.5, "4188": 0.0625, "4205": 0.5, "4233": 1.0, "4265": 1.0, "4286": 1.0, "4289": 0.2, "4306": 1.0, "4312": 1.0, "4335": 1.0, "4339": 1.0, "4394": 0.0, "4409": 0.3333333333333333, "4411": 1.0, "4414": 1.0, "4415": 0.0, "4433": 0.09090909090909091, "4447": 1.0, "4464": 0.3333333333333333, "4465": 0.5, "4484": 0.5, "4499": 0.0, "4500": 0.07142857142857142, "4504": 1.0, "4514": 1.0, "4523": 0.5, "4539": 0.16666666666666666, "4571": 1.0, "4600": 1.0, "4605": 1.0, "4615": 0.5, "4640": 0.0, "4641": 0.2, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 1.0, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 0.16666666666666666, "4837": 0.0, "4844": 1.0, "4845": 1.0, "4846": 0.0, "4863": 0.5, "4865": 0.0, "4920": 1.0, "4942": 0.125, "4946": 1.0, "4955": 0.0, "4962": 1.0, "4968": 0.5, "4981": 1.0, "4999": 1.0, "5021": 0.0, "5030": 0.0, "5045": 0.1111111111111111, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.5, "5080": 0.0, "5083": 0.2, "5085": 1.0, "5086": 1.0, "5090": 0.25, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 0.09090909090909091, "5178": 0.2, "5185": 0.0, "5196": 0.5, "5206": 0.0, "5228": 0.125, "5231": 0.0, "5241": 1.0, "5254": 1.0, "5255": 1.0, "5264": 1.0, "5271": 0.3333333333333333, "5331": 1.0, "5343": 0.14285714285714285, "5347": 1.0, "5356": 0.16666666666666666, "5369": 0.0, "5374": 0.1111111111111111, "5380": 1.0, "5402": 0.0625, "5410": 0.5, "5422": 0.3333333333333333, "5427": 0.2, "5460": 0.0, "5464": 0.16666666666666666, "5503": 0.5, "5505": 1.0, "5511": 0.25, "5534": 0.5, "5549": 0.0, "5585": 0.125, "5592": 1.0, "5616": 0.16666666666666666, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.058823529411764705, "5763": 0.3333333333333333, "5782": 1.0, "5790": 0.0, "5808": 0.3333333333333333, "5853": 0.3333333333333333, "5862": 0.25, "5888": 0.3333333333333333, "5903": 0.0, "5906": 0.25, "5940": 0.0, "5951": 0.5, "5970": 0.3333333333333333, "5981": 1.0, "5993": 1.0, "6002": 1.0, "6004": 0.0, "6005": 1.0, "6009": 0.09090909090909091, "6041": 1.0, "6080": 1.0, "6110": 1.0, "6121": 0.5, "6122": 1.0, "6131": 1.0, "6133": 1.0, "6142": 1.0, "6146": 1.0, "6199": 1.0, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.5, "6278": 0.25, "6395": 0.0, "6410": 0.0625, "6420": 1.0, "6441": 0.2, "6467": 0.0, "6468": 1.0, "6479": 1.0, "6525": 0.5, "6554": 1.0, "6562": 0.1111111111111111, "6611": 1.0, "6612": 0.16666666666666666, "6625": 0.09090909090909091, "6629": 0.0, "6635": 1.0, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 0.5, "6683": 1.0, "6713": 1.0, "6715": 1.0, "6746": 0.25, "6787": 0.0, "6792": 0.0, "6800": 0.25, "6803": 1.0, "6807": 1.0, "6814": 0.0, "6832": 0.2, "6835": 1.0, "6849": 0.25, "6862": 1.0, "6867": 0.25, "6875": 0.0, "6890": 0.25, "6891": 1.0, "6896": 0.0, "6901": 0.16666666666666666, "6907": 0.0, "6909": 0.0, "6959": 0.5, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 0.5, "7098": 1.0, "7105": 0.2, "7109": 0.3333333333333333, "7124": 1.0, "7141": 0.0, "7145": 0.1, "7178": 0.14285714285714285, "7188": 0.125, "7205": 1.0, "7206": 0.0, "7218": 0.25, "7221": 0.0, "7269": 0.07142857142857142, "7279": 0.25, "7295": 0.0, "7311": 0.07692307692307693, "7326": 0.0, "7329": 0.25, "7344": 1.0, "7345": 0.14285714285714285, "7377": 1.0, "7431": 0.0, "7441": 0.25, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.25, "7467": 0.5, "7484": 1.0, "7509": 0.0, "7512": 0.0625, "7513": 0.16666666666666666, "7529": 0.3333333333333333, "7533": 0.08333333333333333, "7534": 1.0, "7590": 0.5, "7592": 0.0, "7594": 0.14285714285714285, "7622": 0.0, "7633": 1.0, "7674": 0.0, "7700": 0.125, "7702": 0.5, "7705": 0.3333333333333333, "7734": 1.0, "7747": 0.14285714285714285, "7754": 0.08333333333333333, "7758": 0.5, "7801": 1.0, "7803": 0.125, "7823": 1.0, "7876": 0.25, "7879": 0.0, "7880": 0.125, "7911": 1.0, "7925": 0.5, "7928": 0.5, "7936": 0.3333333333333333, "7992": 0.5, "8002": 1.0, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.125, "8040": 1.0, "8072": 0.5, "8079": 1.0, "8102": 0.1, "8116": 0.25, "8121": 1.0, "8202": 0.0, "8230": 0.058823529411764705, "8247": 0.0, "8271": 1.0, "8275": 0.25, "8296": 1.0, "8332": 1.0, "8351": 1.0, "8378": 0.1111111111111111, "8456": 1.0, "8475": 1.0, "8507": 0.09090909090909091, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 0.16666666666666666, "8539": 0.09090909090909091, "8544": 1.0, "8592": 0.3333333333333333, "8632": 0.16666666666666666, "8635": 0.0, "8702": 1.0, "8779": 0.09090909090909091, "8789": 1.0, "8795": 1.0, "8832": 1.0, "8834": 1.0, "8855": 0.0, "8874": 0.16666666666666666, "8934": 1.0, "8937": 0.16666666666666666, "8947": 0.2, "8959": 0.25, "8970": 1.0, "8974": 1.0, "8982": 0.0, "9060": 1.0, "9088": 0.0, "9108": 0.0, "9115": 0.25, "9126": 0.0, "9164": 0.125, "9174": 0.07142857142857142, "9188": 1.0, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.14285714285714285, "9329": 1.0, "9332": 0.16666666666666666, "9381": 0.3333333333333333, "9385": 0.16666666666666666, "9391": 0.3333333333333333, "9403": 0.0, "9481": 0.3333333333333333, "9487": 0.08333333333333333, "9548": 0.2, "9556": 1.0, "9565": 1.0, "9598": 0.16666666666666666, "9617": 0.16666666666666666, "9633": 1.0, "9643": 0.3333333333333333, "9644": 0.5, "9646": 0.25, "9668": 1.0, "9701": 0.0, "9733": 0.2, "9735": 0.14285714285714285, "9737": 1.0, "9771": 0.0, "9808": 0.5, "9824": 1.0, "9871": 0.5, "9882": 0.0, "9925": 1.0, "9929": 0.5, "9961": 0.0, "9979": 1.0, "10034": 0.3333333333333333, "10039": 0.5, "10109": 1.0, "10122": 0.3333333333333333, "10136": 0.25, "10137": 0.16666666666666666, "10152": 1.0, "10183": 0.3333333333333333, "10213": 1.0, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.2, "10462": 0.0, "10482": 0.0, "10497": 1.0, "10526": 0.14285714285714285, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 0.1111111111111111, "10639": 0.5, "10645": 0.5, "10674": 1.0, "10710": 0.14285714285714285, "10734": 1.0, "10792": 0.2, "10808": 0.2, "10809": 1.0, "10812": 1.0, "10827": 0.25, "10845": 1.0, "10912": 0.08333333333333333, "10932": 1.0, "10975": 0.0, "10979": 1.0, "10994": 0.3333333333333333, "11039": 0.5, "11054": 1.0, "11088": 0.25}}} diff --git a/evals/benchmark/results-adaptive/fiqa/ir-adm0.50.jsonl b/evals/benchmark/results-adaptive/fiqa/ir-adm0.50.jsonl new file mode 100644 index 000000000..2430e0a27 --- /dev/null +++ b/evals/benchmark/results-adaptive/fiqa/ir-adm0.50.jsonl @@ -0,0 +1 @@ +{"dataset": "fiqa", "run_tag": "adm0.50", "aggregated": {"nDCG@10": 0.4281, "Recall@20": 0.6101, "MRR@10": 0.4972}, "per_query": {"nDCG@10": {"8": 0.6131471927654584, "15": 0.3562071871080222, "18": 0.6309297535714575, "26": 0.23719771276929622, "34": 0.0, "42": 0.46927872602275644, "56": 0.3562071871080222, "68": 0.0, "89": 0.286381299268402, "90": 0.6309297535714575, "94": 1.0, "98": 0.6131471927654584, "104": 0.6131471927654584, "106": 0.5, "109": 1.0, "475": 0.6309297535714575, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.3333333333333333, "559": 0.0, "570": 0.0, "585": 0.2640681225725909, "588": 0.5714285040141098, "594": 0.6131471927654584, "603": 0.0, "604": 0.34870224750355494, "620": 0.0, "622": 0.38685280723454163, "659": 0.43828689407543014, "672": 0.0, "684": 0.0, "687": 0.19342640361727081, "689": 0.0, "691": 0.6309297535714575, "699": 1.0, "701": 0.0, "715": 0.0, "721": 1.0, "744": 0.18154179253735267, "750": 0.8503449055347546, "753": 0.38685280723454163, "766": 0.18154179253735267, "776": 0.3557772116892465, "810": 0.3333333333333333, "813": 0.20438239758848611, "849": 0.31546487678572877, "852": 0.7653606369886217, "853": 0.0, "858": 0.0, "859": 0.38685280723454163, "864": 0.7247145167543899, "879": 0.6309297535714575, "885": 0.19342640361727081, "895": 1.0, "904": 1.0, "928": 0.6131471927654584, "929": 1.0, "932": 0.3065735963827292, "939": 0.5, "945": 0.3010299956639812, "957": 0.7039180890341347, "988": 0.0, "1074": 0.6309297535714575, "1085": 0.0, "1090": 0.23719771276929622, "1150": 0.0, "1157": 0.6131471927654584, "1159": 0.0, "1198": 1.0, "1230": 0.0, "1281": 0.38685280723454163, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.5319460297682747, "1310": 1.0, "1321": 0.0, "1322": 0.24630238874073, "1391": 0.18154179253735267, "1393": 0.19092086617893467, "1415": 0.9828920819566879, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.431733884398313, "1670": 0.7653606369886217, "1676": 0.6713527276121545, "1736": 0.5531464700081437, "1748": 0.6105456988825855, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.286381299268402, "1824": 0.31546487678572877, "1826": 1.0, "1832": 0.5, "1871": 0.6422813708518126, "1877": 1.0, "1889": 0.0, "1915": 0.8772153153380493, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.21398626473452756, "2010": 0.8194270280062366, "2051": 0.0, "2070": 0.23463936301137822, "2075": 0.4880470105041527, "2076": 0.6934264036172708, "2088": 0.0, "2108": 0.16716045496620227, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 0.7526136409516659, "2204": 0.9830168297915423, "2264": 0.38685280723454163, "2296": 0.33627415762678553, "2306": 0.6131471927654584, "2316": 1.0, "2318": 0.3010299956639812, "2330": 0.6131471927654584, "2334": 0.46927872602275644, "2348": 0.43231813227099475, "2376": 0.8910635073822442, "2383": 0.6131471927654584, "2384": 0.6309297535714575, "2385": 0.6131471927654584, "2388": 1.0, "2395": 1.0, "2398": 0.0, "2399": 1.0, "2400": 0.5307212739772434, "2407": 0.24630238874073, "2416": 0.9058653015743079, "2423": 0.3576223542196109, "2443": 0.0, "2445": 1.0, "2460": 0.9469024295259745, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 0.43067655807339306, "2513": 0.0, "2516": 0.5585075862632192, "2549": 0.6131471927654584, "2551": 1.0, "2568": 0.21689524588991285, "2579": 0.2214161691338287, "2580": 0.46927872602275644, "2587": 1.0, "2589": 0.7653606369886217, "2590": 0.46927872602275644, "2593": 0.8854598815714874, "2598": 0.8503449055347546, "2648": 1.0, "2676": 0.5, "2685": 0.35079429740281753, "2695": 1.0, "2713": 0.6131471927654584, "2724": 0.8175295903539445, "2737": 1.0, "2747": 0.5, "2749": 0.8772153153380493, "2790": 0.47997295436377346, "2801": 0.6131471927654584, "2856": 0.7095272044910244, "2857": 0.6595629128335778, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.3010299956639812, "2923": 0.8207658584763088, "2964": 0.0, "2968": 0.6509209298071326, "2994": 0.46927872602275644, "3006": 0.25909036127391766, "3008": 0.24630238874073, "3014": 1.0, "3033": 0.3065735963827292, "3039": 0.43067655807339306, "3049": 0.24630238874073, "3051": 0.9197207891481876, "3067": 0.0, "3085": 0.19342640361727081, "3091": 0.9020870746500321, "3103": 0.6309297535714575, "3115": 0.4935232796777481, "3125": 0.6309297535714575, "3148": 0.5531464700081437, "3149": 0.0, "3177": 0.15642624200758548, "3179": 0.8315546295836225, "3186": 1.0, "3189": 0.0, "3254": 0.6713860725233041, "3264": 0.8048099750039491, "3357": 1.0, "3369": 0.0, "3394": 0.5805485932475565, "3404": 0.3932591258696538, "3405": 0.0, "3446": 0.4340327988596634, "3451": 0.07945707943276742, "3453": 0.5706417189553201, "3480": 1.0, "3490": 0.2890648263178879, "3500": 0.644824486427155, "3503": 0.6183313187362977, "3512": 0.8503449055347546, "3528": 0.6309297535714575, "3530": 0.4776237035032179, "3534": 0.0, "3566": 0.8194270280062366, "3569": 0.5, "3594": 0.123151194370365, "3612": 0.11751610475642714, "3615": 0.430624116386567, "3625": 0.20210734650054757, "3682": 0.20438239758848611, "3683": 0.5982291239616433, "3694": 0.6131471927654584, "3724": 0.0, "3735": 0.0, "3759": 0.0, "3767": 0.2716774977597197, "3771": 0.19632790425573848, "3781": 0.9197207891481876, "3789": 0.8048099750039491, "3791": 0.370291336443826, "3801": 0.17723928678404774, "3822": 0.0, "3829": 0.0, "3830": 0.6309297535714575, "3837": 0.6309297535714575, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5517854393872873, "3932": 0.0, "3934": 0.16812753627111746, "3995": 0.0, "4007": 0.7653606369886217, "4011": 0.9674679834891693, "4019": 0.31608365985743925, "4031": 0.0, "4037": 0.18457569677956817, "4047": 0.8318724637288826, "4071": 0.0, "4084": 0.5012658353418871, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 0.6131471927654584, "4125": 0.0, "4142": 0.39233405721184117, "4153": 0.31546487678572877, "4179": 0.6309297535714575, "4188": 0.0, "4205": 0.4525081529734507, "4233": 0.8687949224876582, "4265": 0.7751482523375255, "4286": 0.6131471927654584, "4289": 0.23719771276929622, "4306": 0.7598336331031966, "4312": 0.7837042816946693, "4335": 0.8318724637288826, "4339": 1.0, "4394": 0.0, "4409": 0.19518900079066107, "4411": 0.46927872602275644, "4414": 0.8315546295836225, "4415": 0.0, "4433": 0.0, "4447": 1.0, "4464": 0.5, "4465": 0.2960819109658652, "4484": 0.6309297535714575, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 0.4034698846650846, "4523": 0.21398626473452756, "4539": 0.21840743681816419, "4571": 0.7653606369886217, "4600": 0.6309297535714575, "4605": 0.5585830187272559, "4615": 0.4816421678943211, "4640": 0.0, "4641": 0.252016380211836, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75369761125927, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.6131471927654584, "4813": 1.0, "4823": 0.46927872602275644, "4827": 0.3562071871080222, "4837": 0.0, "4844": 1.0, "4845": 0.7653606369886217, "4846": 0.0, "4863": 0.49818925746641285, "4865": 0.0, "4920": 1.0, "4942": 0.31546487678572877, "4946": 1.0, "4955": 0.0, "4962": 0.6131471927654584, "4968": 0.6309297535714575, "4981": 0.7653606369886217, "4999": 0.5294362295028773, "5021": 0.0, "5030": 0.0, "5045": 0.3010299956639812, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.6309297535714575, "5080": 0.0, "5083": 0.23719771276929622, "5085": 1.0, "5086": 0.6309297535714575, "5090": 0.2640681225725909, "5125": 1.0, "5134": 0.38685280723454163, "5150": 0.0, "5155": 1.0, "5172": 0.0, "5178": 0.252016380211836, "5185": 0.0, "5196": 0.2960819109658652, "5206": 0.0, "5228": 0.31546487678572877, "5231": 0.0, "5241": 0.44615333764087994, "5254": 0.6131471927654584, "5255": 1.0, "5264": 0.6843515475204855, "5271": 0.41618115554873086, "5331": 1.0, "5343": 0.3333333333333333, "5347": 1.0, "5356": 0.1208113026994942, "5369": 0.0, "5374": 0.3010299956639812, "5380": 1.0, "5402": 0.0, "5410": 0.2960819109658652, "5422": 0.19519002499605084, "5427": 0.23719771276929622, "5460": 0.0, "5464": 0.16716045496620227, "5503": 0.38685280723454163, "5505": 1.0, "5511": 0.09478836436955078, "5534": 0.38685280723454163, "5549": 0.0, "5585": 0.31546487678572877, "5592": 1.0, "5616": 0.3562071871080222, "5620": 0.0, "5646": 0.9217868789962071, "5653": 1.0, "5683": 0.9197207891481876, "5710": 0.0, "5741": 0.0, "5763": 0.23463936301137822, "5782": 0.3391602052736161, "5790": 0.0, "5808": 0.5, "5853": 0.21078558563971425, "5862": 0.29127873064148246, "5888": 0.3065735963827292, "5903": 0.0, "5906": 0.44130740935663865, "5940": 0.0, "5951": 0.42030067999766585, "5970": 0.23463936301137822, "5981": 0.9896062251871525, "5993": 0.31488013066763093, "6002": 0.762416491594312, "6004": 0.0, "6005": 0.5388856921828065, "6009": 0.0, "6041": 0.7668091220635322, "6080": 0.6131471927654584, "6110": 0.41918605213740834, "6121": 0.38685280723454163, "6122": 0.46927872602275644, "6131": 0.5810820381115676, "6133": 0.6131471927654584, "6142": 0.6131471927654584, "6146": 0.8315546295836225, "6199": 0.5135312443624667, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.49818925746641285, "6278": 0.2640681225725909, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.3562071871080222, "6467": 0.0, "6468": 0.6131471927654584, "6479": 1.0, "6525": 0.49818925746641285, "6554": 0.6131471927654584, "6562": 0.11751610475642714, "6611": 0.7903864795495061, "6612": 0.16716045496620227, "6625": 0.0, "6629": 0.0, "6635": 0.6366824387328317, "6644": 0.0, "6647": 0.7653606369886217, "6668": 0.8503449055347546, "6679": 0.4632423659320675, "6683": 0.6240505200038379, "6713": 0.6131471927654584, "6715": 0.6131471927654584, "6746": 0.2640681225725909, "6787": 0.0, "6792": 0.0, "6800": 0.43067655807339306, "6803": 0.5205067333228022, "6807": 1.0, "6814": 0.0, "6832": 0.430624116386567, "6835": 0.6131471927654584, "6849": 0.5, "6862": 1.0, "6867": 0.22471947398559278, "6875": 0.0, "6890": 0.5012658353418871, "6891": 0.617319681505689, "6896": 0.0, "6901": 0.21840743681816419, "6907": 0.0, "6909": 0.0, "6959": 0.38685280723454163, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 0.8503449055347546, "7096": 0.6309297535714575, "7098": 0.6131471927654584, "7105": 0.38685280723454163, "7109": 0.2960819109658652, "7124": 0.6525874238732422, "7141": 0.0, "7145": 0.13565197343244778, "7178": 0.3333333333333333, "7188": 0.31546487678572877, "7205": 1.0, "7206": 0.0, "7218": 0.43067655807339306, "7221": 0.0, "7269": 0.0, "7279": 0.44229834886928765, "7295": 0.0, "7311": 0.0, "7326": 0.0, "7329": 0.46845052016107697, "7344": 0.6131471927654584, "7345": 0.15642624200758548, "7377": 1.0, "7431": 0.0, "7441": 0.18154179253735267, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.14606834984270645, "7467": 0.6309297535714575, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.16716045496620227, "7529": 0.3065735963827292, "7533": 0.0, "7534": 0.46927872602275644, "7590": 0.7977228895450266, "7592": 0.0, "7594": 0.3333333333333333, "7622": 0.0, "7633": 0.3903800499921017, "7674": 0.0, "7700": 0.123151194370365, "7702": 0.38685280723454163, "7705": 0.4367467095119258, "7734": 0.6131471927654584, "7747": 0.20438239758848611, "7754": 0.0, "7758": 0.6309297535714575, "7801": 1.0, "7803": 0.19342640361727081, "7823": 0.6105456988825855, "7876": 0.38685280723454163, "7879": 0.0, "7880": 0.21840743681816419, "7911": 1.0, "7925": 0.19092086617893467, "7928": 0.24630238874073, "7936": 0.40179981797758046, "7992": 0.38685280723454163, "8002": 0.617319681505689, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.31546487678572877, "8040": 1.0, "8072": 0.4776237035032179, "8079": 0.7039180890341347, "8102": 0.11284514134893527, "8116": 0.45749452618986164, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 0.6131471927654584, "8275": 0.43067655807339306, "8296": 0.8315546295836225, "8332": 0.9060254355346823, "8351": 0.6131471927654584, "8378": 0.13565197343244778, "8456": 0.6105456988825855, "8475": 0.7957503936198023, "8507": 0.0, "8512": 0.6309297535714575, "8513": 0.0, "8532": 0.9325210919548239, "8537": 0.3562071871080222, "8539": 0.0, "8544": 0.6131471927654584, "8592": 0.23463936301137822, "8632": 0.21840743681816419, "8635": 0.0, "8702": 0.7653606369886217, "8779": 0.17723928678404774, "8789": 0.7653606369886217, "8795": 1.0, "8832": 1.0, "8834": 0.3980628465882808, "8855": 0.0, "8874": 0.38685280723454163, "8934": 0.8065735963827293, "8937": 0.3562071871080222, "8947": 0.23719771276929622, "8959": 0.43067655807339306, "8970": 0.8772153153380493, "8974": 0.41253177989255346, "8982": 0.0, "9060": 0.6131471927654584, "9088": 0.0, "9108": 0.0, "9115": 0.5256940434743352, "9126": 0.0, "9164": 0.1480409554829326, "9174": 0.0, "9188": 0.6131471927654584, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.20438239758848611, "9329": 1.0, "9332": 0.3562071871080222, "9381": 0.5706417189553201, "9385": 0.3562071871080222, "9391": 0.5143371794949736, "9403": 0.0, "9481": 0.6240505200038379, "9487": 0.0, "9548": 0.38685280723454163, "9556": 1.0, "9565": 1.0, "9598": 0.3562071871080222, "9617": 0.3152014104491349, "9633": 0.6131471927654584, "9643": 0.5, "9644": 0.6309297535714575, "9646": 0.42177340954886444, "9668": 0.6508205185601091, "9701": 0.0, "9733": 0.18154179253735267, "9735": 0.3333333333333333, "9737": 0.7653606369886217, "9771": 0.0, "9808": 0.2960819109658652, "9824": 1.0, "9871": 0.24630238874073, "9882": 0.0, "9925": 0.8315546295836225, "9929": 0.5714285040141098, "9961": 0.0, "9979": 0.3903800499921017, "10034": 0.3065735963827292, "10039": 0.6309297535714575, "10109": 0.5399294342072939, "10122": 0.19519002499605084, "10136": 0.34337431936037655, "10137": 0.3562071871080222, "10152": 1.0, "10183": 0.3065735963827292, "10213": 0.46927872602275644, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.18154179253735267, "10462": 0.0, "10482": 0.0, "10497": 0.49828993591603105, "10526": 0.20438239758848611, "10547": 0.0, "10558": 0.0, "10596": 0.9427731067700877, "10601": 1.0, "10628": 0.3010299956639812, "10639": 0.21398626473452756, "10645": 0.38685280723454163, "10674": 0.7095272044910244, "10710": 0.3333333333333333, "10734": 0.8486987932505933, "10792": 0.4227898344066503, "10808": 0.3295827480202853, "10809": 0.6049306994552042, "10812": 0.9709286432396583, "10827": 0.14606834984270645, "10845": 1.0, "10912": 0.0, "10932": 1.0, "10975": 0.0, "10979": 0.6366824387328317, "10994": 0.4911492931622974, "11039": 0.315526956404728, "11054": 1.0, "11088": 0.43067655807339306}, "Recall@20": {"8": 0.5, "15": 1.0, "18": 1.0, "26": 0.5, "34": 0.0, "42": 0.3333333333333333, "56": 1.0, "68": 0.0, "89": 0.5, "90": 1.0, "94": 1.0, "98": 0.5, "104": 0.5, "106": 1.0, "109": 1.0, "475": 1.0, "503": 1.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 1.0, "585": 0.5, "588": 1.0, "594": 1.0, "603": 0.0, "604": 1.0, "620": 0.2, "622": 0.5, "659": 0.7, "672": 0.5, "684": 0.0, "687": 0.5, "689": 0.0, "691": 1.0, "699": 1.0, "701": 0.3333333333333333, "715": 0.0, "721": 1.0, "744": 0.3333333333333333, "750": 1.0, "753": 0.5, "766": 0.3333333333333333, "776": 0.25, "810": 1.0, "813": 1.0, "849": 1.0, "852": 0.6666666666666666, "853": 0.5, "858": 0.0, "859": 0.5, "864": 0.6666666666666666, "879": 1.0, "885": 0.5, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.5, "939": 1.0, "945": 1.0, "957": 0.6666666666666666, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.5, "1150": 0.0, "1157": 0.5, "1159": 0.0, "1198": 1.0, "1230": 1.0, "1281": 1.0, "1284": 1.0, "1297": 1.0, "1306": 0.25, "1309": 0.75, "1310": 1.0, "1321": 0.0, "1322": 0.25, "1391": 0.3333333333333333, "1393": 0.5, "1415": 1.0, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 1.0, "1670": 0.6666666666666666, "1676": 1.0, "1736": 0.4, "1748": 1.0, "1783": 0.0, "1812": 0.5, "1815": 1.0, "1819": 0.3333333333333333, "1824": 1.0, "1826": 1.0, "1832": 1.0, "1871": 0.75, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.4, "2010": 1.0, "2051": 1.0, "2070": 0.6666666666666666, "2075": 0.5555555555555556, "2076": 1.0, "2088": 0.0, "2108": 0.3333333333333333, "2118": 0.0, "2154": 0.0, "2181": 0.5, "2183": 0.8333333333333334, "2204": 1.0, "2264": 0.5, "2296": 0.3333333333333333, "2306": 0.5, "2316": 1.0, "2318": 1.0, "2330": 0.5, "2334": 0.6666666666666666, "2348": 0.26666666666666666, "2376": 1.0, "2383": 0.5, "2384": 1.0, "2385": 0.5, "2388": 1.0, "2395": 1.0, "2398": 0.2, "2399": 1.0, "2400": 0.6666666666666666, "2407": 0.25, "2416": 1.0, "2423": 0.8571428571428571, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.3333333333333333, "2472": 0.0, "2486": 1.0, "2498": 1.0, "2513": 0.0, "2516": 0.75, "2549": 0.5, "2551": 1.0, "2568": 0.2857142857142857, "2579": 0.3333333333333333, "2580": 0.6666666666666666, "2587": 1.0, "2589": 0.6666666666666666, "2590": 0.6666666666666666, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 1.0, "2685": 0.25, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 1.0, "2790": 0.42857142857142855, "2801": 1.0, "2856": 0.75, "2857": 0.75, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 1.0, "2923": 0.8, "2964": 0.5, "2968": 1.0, "2994": 0.3333333333333333, "3006": 0.3333333333333333, "3008": 0.25, "3014": 1.0, "3033": 0.5, "3039": 1.0, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 1.0, "3091": 0.8571428571428571, "3103": 1.0, "3115": 0.5, "3125": 1.0, "3148": 0.6, "3149": 0.0, "3177": 0.3333333333333333, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.6666666666666666, "3264": 0.75, "3357": 1.0, "3369": 0.16666666666666666, "3394": 0.75, "3404": 0.42857142857142855, "3405": 1.0, "3446": 0.8, "3451": 0.2857142857142857, "3453": 1.0, "3480": 1.0, "3490": 1.0, "3500": 0.5, "3503": 0.7142857142857143, "3512": 1.0, "3528": 1.0, "3530": 0.6666666666666666, "3534": 0.0, "3566": 1.0, "3569": 1.0, "3594": 0.25, "3612": 0.75, "3615": 1.0, "3625": 0.3333333333333333, "3682": 0.5, "3683": 0.6666666666666666, "3694": 0.5, "3724": 0.0, "3735": 1.0, "3759": 0.0, "3767": 0.4, "3771": 0.3333333333333333, "3781": 1.0, "3789": 0.75, "3791": 0.6666666666666666, "3801": 0.5, "3822": 0.0, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 0.6666666666666666, "4011": 1.0, "4019": 0.4, "4031": 0.0, "4037": 0.5, "4047": 0.75, "4071": 0.0, "4084": 1.0, "4102": 0.25, "4103": 0.0, "4105": 0.0, "4116": 0.5, "4125": 0.0, "4142": 0.75, "4153": 1.0, "4179": 1.0, "4188": 1.0, "4205": 0.6666666666666666, "4233": 0.8, "4265": 0.6666666666666666, "4286": 0.5, "4289": 0.5, "4306": 0.75, "4312": 0.8571428571428571, "4335": 0.75, "4339": 1.0, "4394": 0.0, "4409": 0.2, "4411": 0.6666666666666666, "4414": 1.0, "4415": 0.0, "4433": 0.5, "4447": 1.0, "4464": 1.0, "4465": 0.6666666666666666, "4484": 1.0, "4499": 0.0, "4500": 0.5, "4504": 1.0, "4514": 0.5, "4523": 0.2, "4539": 0.5, "4571": 0.6666666666666666, "4600": 1.0, "4605": 0.7142857142857143, "4615": 0.6666666666666666, "4640": 0.0, "4641": 0.6, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 0.6666666666666666, "4846": 0.0, "4863": 1.0, "4865": 0.0, "4920": 1.0, "4942": 1.0, "4946": 1.0, "4955": 0.0, "4962": 0.5, "4968": 1.0, "4981": 0.6666666666666666, "4999": 0.75, "5021": 0.0, "5030": 0.0, "5045": 1.0, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 1.0, "5080": 0.0, "5083": 0.5, "5085": 1.0, "5086": 1.0, "5090": 0.5, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 0.5, "5178": 0.4, "5185": 0.0, "5196": 0.3333333333333333, "5206": 0.0, "5228": 1.0, "5231": 0.0, "5241": 0.4, "5254": 0.5, "5255": 1.0, "5264": 0.6, "5271": 1.0, "5331": 1.0, "5343": 1.0, "5347": 1.0, "5356": 0.2, "5369": 0.0, "5374": 1.0, "5380": 1.0, "5402": 0.5, "5410": 0.3333333333333333, "5422": 0.75, "5427": 0.5, "5460": 0.0, "5464": 0.3333333333333333, "5503": 0.5, "5505": 1.0, "5511": 0.3, "5534": 0.5, "5549": 0.0, "5585": 1.0, "5592": 1.0, "5616": 1.0, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.5, "5763": 0.6666666666666666, "5782": 0.4, "5790": 0.0, "5808": 1.0, "5853": 0.25, "5862": 0.75, "5888": 0.5, "5903": 0.0, "5906": 1.0, "5940": 0.0, "5951": 0.6666666666666666, "5970": 0.3333333333333333, "5981": 1.0, "5993": 0.13333333333333333, "6002": 0.8181818181818182, "6004": 0.0, "6005": 0.38461538461538464, "6009": 1.0, "6041": 0.75, "6080": 0.5, "6110": 0.5, "6121": 0.5, "6122": 0.3333333333333333, "6131": 0.4166666666666667, "6133": 0.5, "6142": 0.5, "6146": 1.0, "6199": 0.5, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.6666666666666666, "6278": 1.0, "6395": 0.0, "6410": 1.0, "6420": 1.0, "6441": 1.0, "6467": 0.0, "6468": 0.5, "6479": 1.0, "6525": 0.6666666666666666, "6554": 0.5, "6562": 0.25, "6611": 1.0, "6612": 0.3333333333333333, "6625": 1.0, "6629": 0.0, "6635": 0.5, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 1.0, "6683": 1.0, "6713": 0.5, "6715": 1.0, "6746": 0.5, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.75, "6807": 1.0, "6814": 0.0, "6832": 1.0, "6835": 0.5, "6849": 1.0, "6862": 1.0, "6867": 0.42857142857142855, "6875": 0.0, "6890": 1.0, "6891": 0.6666666666666666, "6896": 0.0, "6901": 0.5, "6907": 0.0, "6909": 0.0, "6959": 1.0, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 1.0, "7109": 0.6666666666666666, "7124": 0.75, "7141": 0.0, "7145": 0.3333333333333333, "7178": 1.0, "7188": 1.0, "7205": 1.0, "7206": 0.0, "7218": 1.0, "7221": 0.0, "7269": 1.0, "7279": 0.75, "7295": 0.0, "7311": 1.0, "7326": 0.0, "7329": 1.0, "7344": 0.5, "7345": 0.6666666666666666, "7377": 1.0, "7431": 0.0, "7441": 0.3333333333333333, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.4, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 1.0, "7513": 0.3333333333333333, "7529": 0.5, "7533": 0.5, "7534": 0.3333333333333333, "7590": 1.0, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 0.5, "7674": 0.0, "7700": 0.25, "7702": 1.0, "7705": 0.6666666666666666, "7734": 0.5, "7747": 0.5, "7754": 1.0, "7758": 1.0, "7801": 1.0, "7803": 0.5, "7823": 0.6666666666666666, "7876": 1.0, "7879": 0.0, "7880": 0.5, "7911": 1.0, "7925": 0.16666666666666666, "7928": 0.25, "7936": 0.6666666666666666, "7992": 1.0, "8002": 0.6666666666666666, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 1.0, "8040": 1.0, "8072": 0.6666666666666666, "8079": 1.0, "8102": 0.25, "8116": 1.0, "8121": 1.0, "8202": 0.0, "8230": 0.5, "8247": 0.0, "8271": 0.5, "8275": 1.0, "8296": 1.0, "8332": 1.0, "8351": 0.5, "8378": 0.3333333333333333, "8456": 0.6666666666666666, "8475": 0.8571428571428571, "8507": 0.3333333333333333, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 1.0, "8539": 0.25, "8544": 0.5, "8592": 1.0, "8632": 1.0, "8635": 0.0, "8702": 0.6666666666666666, "8779": 1.0, "8789": 0.6666666666666666, "8795": 1.0, "8832": 1.0, "8834": 0.5, "8855": 0.0, "8874": 1.0, "8934": 1.0, "8937": 1.0, "8947": 0.5, "8959": 1.0, "8970": 1.0, "8974": 0.25, "8982": 0.0, "9060": 0.5, "9088": 0.0, "9108": 0.0, "9115": 1.0, "9126": 0.0, "9164": 0.3333333333333333, "9174": 0.2, "9188": 0.5, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.5, "9329": 1.0, "9332": 1.0, "9381": 1.0, "9385": 1.0, "9391": 0.75, "9403": 0.0, "9481": 1.0, "9487": 1.0, "9548": 1.0, "9556": 1.0, "9565": 1.0, "9598": 1.0, "9617": 0.6666666666666666, "9633": 0.5, "9643": 1.0, "9644": 1.0, "9646": 1.0, "9668": 1.0, "9701": 0.0, "9733": 0.3333333333333333, "9735": 1.0, "9737": 0.6666666666666666, "9771": 0.0, "9808": 0.3333333333333333, "9824": 1.0, "9871": 0.25, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 1.0, "9979": 0.5, "10034": 0.5, "10039": 1.0, "10109": 0.42857142857142855, "10122": 0.5, "10136": 0.6666666666666666, "10137": 1.0, "10152": 1.0, "10183": 0.5, "10213": 0.3333333333333333, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.3333333333333333, "10462": 0.0, "10482": 0.0, "10497": 0.6, "10526": 0.5, "10547": 0.0, "10558": 0.5, "10596": 1.0, "10601": 1.0, "10628": 1.0, "10639": 0.4, "10645": 1.0, "10674": 1.0, "10710": 1.0, "10734": 1.0, "10792": 1.0, "10808": 1.0, "10809": 0.6666666666666666, "10812": 1.0, "10827": 0.2, "10845": 1.0, "10912": 1.0, "10932": 1.0, "10975": 0.0, "10979": 0.75, "10994": 1.0, "11039": 0.5, "11054": 1.0, "11088": 1.0}, "MRR@10": {"8": 1.0, "15": 0.16666666666666666, "18": 0.5, "26": 0.2, "34": 0.0, "42": 1.0, "56": 0.16666666666666666, "68": 0.0, "89": 0.5, "90": 0.5, "94": 1.0, "98": 1.0, "104": 1.0, "106": 0.3333333333333333, "109": 1.0, "475": 0.5, "503": 0.06666666666666667, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.14285714285714285, "559": 0.0, "570": 0.06666666666666667, "585": 0.25, "588": 0.5, "594": 1.0, "603": 0.0, "604": 0.2, "620": 0.08333333333333333, "622": 0.5, "659": 1.0, "672": 0.07692307692307693, "684": 0.0, "687": 0.125, "689": 0.0, "691": 0.5, "699": 1.0, "701": 0.07692307692307693, "715": 0.0, "721": 1.0, "744": 0.2, "750": 1.0, "753": 0.5, "766": 0.2, "776": 1.0, "810": 0.14285714285714285, "813": 0.14285714285714285, "849": 0.125, "852": 1.0, "853": 0.09090909090909091, "858": 0.0, "859": 0.5, "864": 1.0, "879": 0.5, "885": 0.125, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.3333333333333333, "939": 0.3333333333333333, "945": 0.1111111111111111, "957": 1.0, "988": 0.0, "1074": 0.5, "1085": 0.0, "1090": 0.2, "1150": 0.0, "1157": 1.0, "1159": 0.0, "1198": 1.0, "1230": 0.05, "1281": 0.2, "1284": 1.0, "1297": 1.0, "1306": 0.07692307692307693, "1309": 0.5, "1310": 1.0, "1321": 0.0, "1322": 0.5, "1391": 0.2, "1393": 0.5, "1415": 1.0, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.5, "1670": 1.0, "1676": 1.0, "1736": 1.0, "1748": 1.0, "1783": 0.0, "1812": 0.06666666666666667, "1815": 1.0, "1819": 0.5, "1824": 0.125, "1826": 1.0, "1832": 0.3333333333333333, "1871": 1.0, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.5, "2010": 1.0, "2051": 0.0625, "2070": 0.3333333333333333, "2075": 1.0, "2076": 0.5, "2088": 0.0, "2108": 0.16666666666666666, "2118": 0.0, "2154": 0.0, "2181": 0.05263157894736842, "2183": 1.0, "2204": 1.0, "2264": 0.5, "2296": 1.0, "2306": 1.0, "2316": 1.0, "2318": 0.1111111111111111, "2330": 1.0, "2334": 1.0, "2348": 1.0, "2376": 1.0, "2383": 1.0, "2384": 0.5, "2385": 1.0, "2388": 1.0, "2395": 1.0, "2398": 0.08333333333333333, "2399": 1.0, "2400": 0.5, "2407": 0.5, "2416": 1.0, "2423": 1.0, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0625, "2472": 0.0, "2486": 1.0, "2498": 0.25, "2513": 0.0, "2516": 1.0, "2549": 1.0, "2551": 1.0, "2568": 0.3333333333333333, "2579": 0.25, "2580": 1.0, "2587": 1.0, "2589": 1.0, "2590": 1.0, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 0.3333333333333333, "2685": 1.0, "2695": 1.0, "2713": 1.0, "2724": 1.0, "2737": 1.0, "2747": 0.3333333333333333, "2749": 1.0, "2790": 1.0, "2801": 1.0, "2856": 1.0, "2857": 1.0, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.1111111111111111, "2923": 1.0, "2964": 0.07692307692307693, "2968": 0.5, "2994": 1.0, "3006": 0.3333333333333333, "3008": 0.5, "3014": 1.0, "3033": 0.3333333333333333, "3039": 0.25, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 0.125, "3091": 1.0, "3103": 0.5, "3115": 1.0, "3125": 0.5, "3148": 1.0, "3149": 0.0, "3177": 0.14285714285714285, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 1.0, "3264": 1.0, "3357": 1.0, "3369": 0.09090909090909091, "3394": 0.5, "3404": 1.0, "3405": 0.09090909090909091, "3446": 0.5, "3451": 0.1, "3453": 0.3333333333333333, "3480": 1.0, "3490": 0.1, "3500": 1.0, "3503": 0.5, "3512": 1.0, "3528": 0.5, "3530": 0.5, "3534": 0.0, "3566": 1.0, "3569": 0.3333333333333333, "3594": 0.125, "3612": 0.1111111111111111, "3615": 0.2, "3625": 0.25, "3682": 0.14285714285714285, "3683": 1.0, "3694": 1.0, "3724": 0.0, "3735": 0.09090909090909091, "3759": 0.0, "3767": 0.3333333333333333, "3771": 0.14285714285714285, "3781": 1.0, "3789": 1.0, "3791": 0.3333333333333333, "3801": 0.1, "3822": 0.0, "3829": 0.0, "3830": 0.5, "3837": 0.5, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 1.0, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 1.0, "4011": 1.0, "4019": 0.5, "4031": 0.0, "4037": 0.1111111111111111, "4047": 1.0, "4071": 0.0, "4084": 0.25, "4102": 0.0625, "4103": 0.0, "4105": 0.0, "4116": 1.0, "4125": 0.0, "4142": 0.16666666666666666, "4153": 0.125, "4179": 0.5, "4188": 0.0625, "4205": 0.5, "4233": 1.0, "4265": 1.0, "4286": 1.0, "4289": 0.2, "4306": 1.0, "4312": 1.0, "4335": 1.0, "4339": 1.0, "4394": 0.0, "4409": 0.3333333333333333, "4411": 1.0, "4414": 1.0, "4415": 0.0, "4433": 0.09090909090909091, "4447": 1.0, "4464": 0.3333333333333333, "4465": 0.5, "4484": 0.5, "4499": 0.0, "4500": 0.058823529411764705, "4504": 1.0, "4514": 1.0, "4523": 0.5, "4539": 0.16666666666666666, "4571": 1.0, "4600": 0.5, "4605": 1.0, "4615": 0.5, "4640": 0.0, "4641": 0.2, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 1.0, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 0.16666666666666666, "4837": 0.0, "4844": 1.0, "4845": 1.0, "4846": 0.0, "4863": 0.5, "4865": 0.0, "4920": 1.0, "4942": 0.125, "4946": 1.0, "4955": 0.0, "4962": 1.0, "4968": 0.5, "4981": 1.0, "4999": 1.0, "5021": 0.0, "5030": 0.0, "5045": 0.1111111111111111, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.5, "5080": 0.0, "5083": 0.2, "5085": 1.0, "5086": 0.5, "5090": 0.25, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 0.08333333333333333, "5178": 0.2, "5185": 0.0, "5196": 0.5, "5206": 0.0, "5228": 0.125, "5231": 0.0, "5241": 1.0, "5254": 1.0, "5255": 1.0, "5264": 1.0, "5271": 0.3333333333333333, "5331": 1.0, "5343": 0.14285714285714285, "5347": 1.0, "5356": 0.16666666666666666, "5369": 0.0, "5374": 0.1111111111111111, "5380": 1.0, "5402": 0.0625, "5410": 0.5, "5422": 0.3333333333333333, "5427": 0.2, "5460": 0.0, "5464": 0.16666666666666666, "5503": 0.5, "5505": 1.0, "5511": 0.25, "5534": 0.5, "5549": 0.0, "5585": 0.125, "5592": 1.0, "5616": 0.16666666666666666, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.05263157894736842, "5763": 0.3333333333333333, "5782": 1.0, "5790": 0.0, "5808": 0.3333333333333333, "5853": 0.3333333333333333, "5862": 0.25, "5888": 0.3333333333333333, "5903": 0.0, "5906": 0.25, "5940": 0.0, "5951": 0.5, "5970": 0.3333333333333333, "5981": 1.0, "5993": 1.0, "6002": 1.0, "6004": 0.0, "6005": 1.0, "6009": 0.09090909090909091, "6041": 1.0, "6080": 1.0, "6110": 1.0, "6121": 0.5, "6122": 1.0, "6131": 1.0, "6133": 1.0, "6142": 1.0, "6146": 1.0, "6199": 1.0, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.5, "6278": 0.25, "6395": 0.0, "6410": 0.09090909090909091, "6420": 1.0, "6441": 0.16666666666666666, "6467": 0.0, "6468": 1.0, "6479": 1.0, "6525": 0.5, "6554": 1.0, "6562": 0.1111111111111111, "6611": 1.0, "6612": 0.16666666666666666, "6625": 0.08333333333333333, "6629": 0.0, "6635": 1.0, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 0.5, "6683": 0.5, "6713": 1.0, "6715": 1.0, "6746": 0.25, "6787": 0.0, "6792": 0.0, "6800": 0.25, "6803": 1.0, "6807": 1.0, "6814": 0.0, "6832": 0.2, "6835": 1.0, "6849": 0.3333333333333333, "6862": 1.0, "6867": 0.25, "6875": 0.0, "6890": 0.25, "6891": 1.0, "6896": 0.0, "6901": 0.16666666666666666, "6907": 0.0, "6909": 0.0, "6959": 0.5, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 0.5, "7098": 1.0, "7105": 0.2, "7109": 0.5, "7124": 1.0, "7141": 0.0, "7145": 0.1, "7178": 0.14285714285714285, "7188": 0.125, "7205": 1.0, "7206": 0.0, "7218": 0.25, "7221": 0.0, "7269": 0.07142857142857142, "7279": 0.25, "7295": 0.0, "7311": 0.0625, "7326": 0.0, "7329": 0.25, "7344": 1.0, "7345": 0.14285714285714285, "7377": 1.0, "7431": 0.0, "7441": 0.2, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.25, "7467": 0.5, "7484": 1.0, "7509": 0.0, "7512": 0.0625, "7513": 0.16666666666666666, "7529": 0.3333333333333333, "7533": 0.07692307692307693, "7534": 1.0, "7590": 1.0, "7592": 0.0, "7594": 0.14285714285714285, "7622": 0.0, "7633": 1.0, "7674": 0.0, "7700": 0.125, "7702": 0.5, "7705": 0.3333333333333333, "7734": 1.0, "7747": 0.14285714285714285, "7754": 0.08333333333333333, "7758": 0.5, "7801": 1.0, "7803": 0.125, "7823": 1.0, "7876": 0.2, "7879": 0.0, "7880": 0.16666666666666666, "7911": 1.0, "7925": 0.5, "7928": 0.5, "7936": 0.3333333333333333, "7992": 0.5, "8002": 1.0, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.125, "8040": 1.0, "8072": 0.5, "8079": 1.0, "8102": 0.1, "8116": 0.25, "8121": 1.0, "8202": 0.0, "8230": 0.05263157894736842, "8247": 0.0, "8271": 1.0, "8275": 0.25, "8296": 1.0, "8332": 1.0, "8351": 1.0, "8378": 0.1, "8456": 1.0, "8475": 1.0, "8507": 0.09090909090909091, "8512": 0.5, "8513": 0.0, "8532": 1.0, "8537": 0.16666666666666666, "8539": 0.07142857142857142, "8544": 1.0, "8592": 0.3333333333333333, "8632": 0.16666666666666666, "8635": 0.0, "8702": 1.0, "8779": 0.1, "8789": 1.0, "8795": 1.0, "8832": 1.0, "8834": 1.0, "8855": 0.0, "8874": 0.2, "8934": 1.0, "8937": 0.16666666666666666, "8947": 0.2, "8959": 0.25, "8970": 1.0, "8974": 1.0, "8982": 0.0, "9060": 1.0, "9088": 0.0, "9108": 0.0, "9115": 0.25, "9126": 0.0, "9164": 0.125, "9174": 0.07142857142857142, "9188": 1.0, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.14285714285714285, "9329": 1.0, "9332": 0.16666666666666666, "9381": 0.3333333333333333, "9385": 0.16666666666666666, "9391": 0.3333333333333333, "9403": 0.0, "9481": 0.5, "9487": 0.08333333333333333, "9548": 0.2, "9556": 1.0, "9565": 1.0, "9598": 0.16666666666666666, "9617": 0.16666666666666666, "9633": 1.0, "9643": 0.3333333333333333, "9644": 0.5, "9646": 0.2, "9668": 1.0, "9701": 0.0, "9733": 0.2, "9735": 0.14285714285714285, "9737": 1.0, "9771": 0.0, "9808": 0.5, "9824": 1.0, "9871": 0.5, "9882": 0.0, "9925": 1.0, "9929": 0.5, "9961": 0.05263157894736842, "9979": 1.0, "10034": 0.3333333333333333, "10039": 0.5, "10109": 1.0, "10122": 0.3333333333333333, "10136": 0.25, "10137": 0.16666666666666666, "10152": 1.0, "10183": 0.3333333333333333, "10213": 1.0, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.2, "10462": 0.0, "10482": 0.0, "10497": 1.0, "10526": 0.14285714285714285, "10547": 0.0, "10558": 0.05263157894736842, "10596": 1.0, "10601": 1.0, "10628": 0.1111111111111111, "10639": 0.5, "10645": 0.5, "10674": 1.0, "10710": 0.14285714285714285, "10734": 1.0, "10792": 0.16666666666666666, "10808": 0.2, "10809": 1.0, "10812": 1.0, "10827": 0.25, "10845": 1.0, "10912": 0.07692307692307693, "10932": 1.0, "10975": 0.0, "10979": 1.0, "10994": 0.3333333333333333, "11039": 0.5, "11054": 1.0, "11088": 0.25}}} diff --git a/evals/benchmark/results-adaptive/fiqa/run-adm0.15.trec.gz b/evals/benchmark/results-adaptive/fiqa/run-adm0.15.trec.gz new file mode 100644 index 000000000..f4825d81b Binary files /dev/null and b/evals/benchmark/results-adaptive/fiqa/run-adm0.15.trec.gz differ diff --git a/evals/benchmark/results-adaptive/fiqa/stats-adm0.15-vs-dense.jsonl b/evals/benchmark/results-adaptive/fiqa/stats-adm0.15-vs-dense.jsonl new file mode 100644 index 000000000..c2d0cfd77 --- /dev/null +++ b/evals/benchmark/results-adaptive/fiqa/stats-adm0.15-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "adm0.15", "arm_b": "dense"} +{"row_type": "ir", "dataset": "fiqa", "metric": "nDCG@10", "n": 648, "mean_a": 0.44095088181672676, "mean_b": 0.4593212711946586, "mean_diff": 0.018370389377931798, "ci_low": 0.0064939441616973445, "ci_high": 0.030178623637791963, "p_value": 0.0030996900309969004, "significant": true} +{"row_type": "ir", "dataset": "fiqa", "metric": "Recall@20", "n": 648, "mean_a": 0.6192455050093939, "mean_b": 0.6184739000711222, "mean_diff": -0.0007716049382716049, "ci_low": -0.0023148148148148147, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "fiqa", "metric": "MRR@10", "n": 648, "mean_a": 0.5142836262918178, "mean_b": 0.5444934096921756, "mean_diff": 0.03020978340035788, "ci_low": 0.011089687675102908, "ci_high": 0.049370468885759744, "p_value": 0.0020997900209979003, "significant": true} diff --git a/evals/benchmark/results-adaptive/fiqa/stats-adm0.30-vs-dense.jsonl b/evals/benchmark/results-adaptive/fiqa/stats-adm0.30-vs-dense.jsonl new file mode 100644 index 000000000..b32ea0813 --- /dev/null +++ b/evals/benchmark/results-adaptive/fiqa/stats-adm0.30-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "adm0.30", "arm_b": "dense"} +{"row_type": "ir", "dataset": "fiqa", "metric": "nDCG@10", "n": 648, "mean_a": 0.4323057944095525, "mean_b": 0.4593212711946586, "mean_diff": 0.027015476785106015, "ci_low": 0.013850445084198757, "ci_high": 0.04025476814035674, "p_value": 0.0004999500049995, "significant": true} +{"row_type": "ir", "dataset": "fiqa", "metric": "Recall@20", "n": 648, "mean_a": 0.615237446024483, "mean_b": 0.6184739000711222, "mean_diff": 0.003236454046639232, "ci_low": -0.002700617283950617, "ci_high": 0.009259259259259259, "p_value": 0.3060693930606939, "significant": false} +{"row_type": "ir", "dataset": "fiqa", "metric": "MRR@10", "n": 648, "mean_a": 0.5032414891220145, "mean_b": 0.5444934096921756, "mean_diff": 0.04125192057016103, "ci_low": 0.020228842287670298, "ci_high": 0.0622629414061643, "p_value": 0.0004999500049995, "significant": true} diff --git a/evals/benchmark/results-adaptive/fiqa/stats-adm0.50-vs-dense.jsonl b/evals/benchmark/results-adaptive/fiqa/stats-adm0.50-vs-dense.jsonl new file mode 100644 index 000000000..31b2f963a --- /dev/null +++ b/evals/benchmark/results-adaptive/fiqa/stats-adm0.50-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "adm0.50", "arm_b": "dense"} +{"row_type": "ir", "dataset": "fiqa", "metric": "nDCG@10", "n": 648, "mean_a": 0.4280700229017404, "mean_b": 0.4593212711946586, "mean_diff": 0.031251248292918116, "ci_low": 0.018013095401605703, "ci_high": 0.04460475559377619, "p_value": 9.999000099990002e-05, "significant": true} +{"row_type": "ir", "dataset": "fiqa", "metric": "Recall@20", "n": 648, "mean_a": 0.6101098918228548, "mean_b": 0.6184739000711222, "mean_diff": 0.008364008248267508, "ci_low": -0.0008684689153439154, "ci_high": 0.017943175972689854, "p_value": 0.08289171082891711, "significant": false} +{"row_type": "ir", "dataset": "fiqa", "metric": "MRR@10", "n": 648, "mean_a": 0.49719203004645474, "mean_b": 0.5444934096921756, "mean_diff": 0.04730137964572089, "ci_low": 0.02628181337866921, "ci_high": 0.06853884310044817, "p_value": 0.00019998000199980003, "significant": true} diff --git a/evals/benchmark/results-adaptive/manifest.frozen.json b/evals/benchmark/results-adaptive/manifest.frozen.json new file mode 100644 index 000000000..5bda42167 --- /dev/null +++ b/evals/benchmark/results-adaptive/manifest.frozen.json @@ -0,0 +1,87 @@ +{ + "arms": [ + { + "config": { + "intent_llm": true, + "layout_detection": true, + "neighbor_expansion": 2, + "table_extraction": true, + "title_search": true + }, + "description": "test/retrieval-parity with every parity feature enabled", + "name": "lilbee-parity", + "system": "lilbee" + }, + { + "config": { + "pipeline": "deepdoc" + }, + "description": "RAGFlow DeepDoc pipeline at its defaults", + "name": "ragflow-default", + "system": "ragflow" + } + ], + "datasets": [ + { + "label_kind": "native", + "loader": "scifact", + "name": "scifact", + "split": "test" + }, + { + "label_kind": "native", + "loader": "fiqa", + "name": "fiqa", + "split": "test" + }, + { + "label_kind": "native", + "loader": "nfcorpus", + "name": "nfcorpus", + "split": "test" + }, + { + "label_kind": "native", + "loader": "hotpotqa", + "name": "hotpotqa", + "split": "test" + }, + { + "label_kind": "native", + "loader": "trec-covid", + "name": "trec-covid", + "split": "test" + }, + { + "label_kind": "derived", + "loader": "tatdqa", + "name": "tat-dqa", + "split": "test" + }, + { + "label_kind": "derived", + "loader": "ottqa", + "name": "ott-qa", + "split": "dev" + } + ], + "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", + "metrics": [ + "nDCG@10", + "Recall@20", + "MRR@10" + ], + "models": { + "embedder": "qwen3-embedding", + "generator": "qwen2.5-72b-instruct", + "generator_swap": "mistral-large-2411", + "judge": "fable-5" + }, + "run_id": "parity-2026-07", + "stats": { + "alpha": 0.05, + "bootstrap_resamples": 10000, + "seed": 20260714 + }, + "temperature": 0.0 +} diff --git a/evals/benchmark/results-adaptive/nfcorpus/ir-adm0.15.jsonl b/evals/benchmark/results-adaptive/nfcorpus/ir-adm0.15.jsonl new file mode 100644 index 000000000..8e1b39a75 --- /dev/null +++ b/evals/benchmark/results-adaptive/nfcorpus/ir-adm0.15.jsonl @@ -0,0 +1 @@ +{"dataset": "nfcorpus", "run_tag": "adm0.15", "aggregated": {"nDCG@10": 0.3751, "Recall@20": 0.2141, "MRR@10": 0.603}, "per_query": {"nDCG@10": {"PLAIN-2": 0.8291570719562573, "PLAIN-12": 0.16950960215087577, "PLAIN-23": 0.46459600558095715, "PLAIN-33": 0.2640886107812416, "PLAIN-44": 0.23888170027858516, "PLAIN-56": 0.8281497384091, "PLAIN-68": 0.4509737963519493, "PLAIN-78": 0.06370866775185362, "PLAIN-91": 0.6947482816557203, "PLAIN-102": 0.37789901297017203, "PLAIN-112": 0.674411193614723, "PLAIN-123": 0.08039138809008574, "PLAIN-133": 0.13347537859706934, "PLAIN-143": 0.6347240446515069, "PLAIN-153": 0.9478557351534421, "PLAIN-165": 0.5494506864770778, "PLAIN-175": 0.42027556825478546, "PLAIN-186": 0.22009176629808017, "PLAIN-196": 0.22009176629808017, "PLAIN-207": 0.4230293752959408, "PLAIN-217": 0.13305201013572898, "PLAIN-227": 0.11004588314904008, "PLAIN-238": 0.0, "PLAIN-248": 0.38113435412202, "PLAIN-259": 0.3589542101716347, "PLAIN-270": 0.31488013066763093, "PLAIN-280": 0.39673781793804724, "PLAIN-291": 0.43231813227099475, "PLAIN-307": 0.9674679834891693, "PLAIN-320": 0.0, "PLAIN-332": 0.6131471927654584, "PLAIN-344": 0.4777009208974601, "PLAIN-358": 0.0, "PLAIN-371": 0.23981246656813146, "PLAIN-383": 0.16958010263680806, "PLAIN-395": 0.0, "PLAIN-407": 0.6387878864795979, "PLAIN-418": 0.8670870086853021, "PLAIN-430": 0.6174886499090687, "PLAIN-441": 0.777747077360283, "PLAIN-457": 0.0, "PLAIN-468": 0.4318845862516251, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 0.5174612499126158, "PLAIN-510": 0.0, "PLAIN-520": 0.0, "PLAIN-531": 0.5368103404171507, "PLAIN-541": 0.11706259313796354, "PLAIN-551": 0.7903864795495061, "PLAIN-561": 0.43735247915031, "PLAIN-571": 0.2162957183053307, "PLAIN-583": 0.22009176629808017, "PLAIN-593": 0.46927872602275644, "PLAIN-603": 0.0, "PLAIN-613": 0.11004588314904008, "PLAIN-623": 0.33013764944712026, "PLAIN-634": 0.424926013816671, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.4519792040377807, "PLAIN-711": 0.7605591431973849, "PLAIN-721": 1.0, "PLAIN-731": 0.6325414799409712, "PLAIN-741": 0.8701249883466593, "PLAIN-751": 0.0, "PLAIN-761": 0.15130120674940672, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.644824486427155, "PLAIN-806": 0.857980942822373, "PLAIN-817": 0.43067655807339306, "PLAIN-827": 0.7411902110103873, "PLAIN-838": 0.7767471075223497, "PLAIN-850": 0.6936634693435663, "PLAIN-872": 0.0, "PLAIN-882": 0.4690000933206748, "PLAIN-892": 0.3557772116892465, "PLAIN-902": 0.6366824387328317, "PLAIN-913": 0.7967610662472993, "PLAIN-924": 0.30523488393970116, "PLAIN-934": 0.702728403557943, "PLAIN-946": 0.22009176629808017, "PLAIN-956": 0.7312625444731047, "PLAIN-966": 0.0, "PLAIN-977": 0.46927872602275644, "PLAIN-987": 0.0, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.4263100803427562, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.07839826897867533, "PLAIN-1066": 0.0, "PLAIN-1088": 0.5531464700081437, "PLAIN-1098": 0.0, "PLAIN-1109": 0.762416491594312, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.22009176629808017, "PLAIN-1151": 0.7385567757549499, "PLAIN-1161": 0.0, "PLAIN-1172": 0.22009176629808017, "PLAIN-1183": 0.1731866333482261, "PLAIN-1193": 0.0, "PLAIN-1203": 0.48471198902551854, "PLAIN-1214": 0.0, "PLAIN-1225": 0.7411902110103873, "PLAIN-1236": 0.3391602052736161, "PLAIN-1249": 0.0, "PLAIN-1262": 0.4690000933206748, "PLAIN-1275": 0.727329844310522, "PLAIN-1288": 0.49828993591603105, "PLAIN-1299": 0.3589542101716347, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5842273861585908, "PLAIN-1331": 0.0, "PLAIN-1342": 0.5957616317188126, "PLAIN-1353": 0.0, "PLAIN-1363": 0.22009176629808017, "PLAIN-1374": 0.07839826897867533, "PLAIN-1387": 0.7819835854675755, "PLAIN-1398": 1.0, "PLAIN-1409": 0.5619107241149828, "PLAIN-1419": 0.4690000933206748, "PLAIN-1429": 0.0, "PLAIN-1441": 0.6325414799409712, "PLAIN-1453": 0.47674870152787435, "PLAIN-1463": 0.4576752747037683, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.30523488393970116, "PLAIN-1516": 0.0, "PLAIN-1527": 0.8482378089219645, "PLAIN-1537": 0.43295704183322337, "PLAIN-1547": 0.0, "PLAIN-1557": 0.2489083270225946, "PLAIN-1568": 1.0, "PLAIN-1579": 0.3391602052736161, "PLAIN-1590": 0.06943122193677727, "PLAIN-1601": 0.727329844310522, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.8643145546088337, "PLAIN-1645": 0.30260241349881345, "PLAIN-1656": 0.0, "PLAIN-1667": 0.8603818544462509, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 0.22009176629808017, "PLAIN-1731": 0.7653606369886217, "PLAIN-1741": 1.0, "PLAIN-1752": 0.0, "PLAIN-1762": 0.488675120609937, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 0.3589542101716347, "PLAIN-1837": 1.0, "PLAIN-1847": 0.22009176629808017, "PLAIN-1857": 0.6207622843987103, "PLAIN-1867": 0.27487633291429087, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.7917267193679839, "PLAIN-1919": 0.8630152897016883, "PLAIN-1929": 0.1736666713479918, "PLAIN-1940": 0.3589542101716347, "PLAIN-1950": 0.21726071285222986, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.9266360779006401, "PLAIN-1995": 0.0, "PLAIN-2009": 0.31442295467135667, "PLAIN-2019": 0.6364391809889587, "PLAIN-2030": 0.0, "PLAIN-2040": 0.5637884576902256, "PLAIN-2051": 0.6901935063354876, "PLAIN-2061": 0.933745776545611, "PLAIN-2071": 0.5103515512676448, "PLAIN-2081": 1.0, "PLAIN-2092": 0.6489315753318465, "PLAIN-2102": 0.8165901947515999, "PLAIN-2113": 0.0, "PLAIN-2124": 0.6906478832608419, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.644824486427155, "PLAIN-2167": 0.0, "PLAIN-2177": 0.46692474155501895, "PLAIN-2187": 0.23504554941448536, "PLAIN-2197": 0.8643145546088337, "PLAIN-2209": 0.0, "PLAIN-2220": 0.5526193270751085, "PLAIN-2230": 0.0, "PLAIN-2240": 0.43735247915031, "PLAIN-2250": 0.0, "PLAIN-2261": 0.7183627972686238, "PLAIN-2271": 0.0, "PLAIN-2281": 0.6309297535714575, "PLAIN-2291": 0.3391602052736161, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.4505655760587972, "PLAIN-2343": 0.0, "PLAIN-2354": 0.5531464700081437, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.31204907722178066, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.5115716521414854, "PLAIN-2440": 0.07336392209936005, "PLAIN-2450": 0.22009176629808017, "PLAIN-2460": 0.2873703978647362, "PLAIN-2470": 0.6923056707325975, "PLAIN-2480": 0.22025065699347168, "PLAIN-2490": 0.4104014996484018, "PLAIN-2500": 0.06760523069231983, "PLAIN-2510": 0.6320787427864673, "PLAIN-2520": 0.2085303067457527, "PLAIN-2530": 0.9652843890316114, "PLAIN-2540": 0.5907275541269412, "PLAIN-2550": 0.0, "PLAIN-2560": 0.9266360779006401, "PLAIN-2570": 0.4205032672422578, "PLAIN-2580": 0.3388812950446278, "PLAIN-2590": 0.17764036563972363, "PLAIN-2600": 0.6232826076301087, "PLAIN-2610": 0.3168370282761786, "PLAIN-2620": 0.5549841905774386, "PLAIN-2630": 0.8167922648045142, "PLAIN-2640": 0.5469224065306436, "PLAIN-2650": 0.6235744328990731, "PLAIN-2660": 0.31954962756492084, "PLAIN-2670": 0.35301686087887274, "PLAIN-2680": 0.4236154895010551, "PLAIN-2690": 0.5760823635418191, "PLAIN-2700": 0.28893989447363055, "PLAIN-2710": 0.6779733527921183, "PLAIN-2720": 0.2489083270225946, "PLAIN-2730": 0.6433495827340141, "PLAIN-2740": 0.6706555576145379, "PLAIN-2750": 0.5644487757556916, "PLAIN-2760": 0.2469526638494753, "PLAIN-2770": 0.4541431527378687, "PLAIN-2780": 0.4786771405648691, "PLAIN-2790": 0.8204437308808777, "PLAIN-2800": 0.15472892909388752, "PLAIN-2810": 0.054784966022752124, "PLAIN-2820": 0.15619484341784115, "PLAIN-2830": 0.6387878864795979, "PLAIN-2840": 0.0, "PLAIN-2850": 0.24014860122439108, "PLAIN-2860": 1.0, "PLAIN-2870": 0.08442111803551768, "PLAIN-2880": 0.0, "PLAIN-2890": 0.13956838328637605, "PLAIN-2900": 0.09382844813642036, "PLAIN-2910": 0.3239134396887189, "PLAIN-2920": 0.0, "PLAIN-2930": 0.43056322872474745, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.3273949503887395, "PLAIN-2970": 0.38062652754129855, "PLAIN-2981": 0.43122038713502525, "PLAIN-2991": 0.38685280723454163, "PLAIN-3001": 0.38685280723454163, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.7653606369886217, "PLAIN-3063": 0.0, "PLAIN-3074": 1.0, "PLAIN-3085": 0.5012658353418871, "PLAIN-3097": 0.4770382338730848, "PLAIN-3116": 0.0, "PLAIN-3131": 0.4599311598225878, "PLAIN-3141": 0.2996484034259831, "PLAIN-3151": 0.7722158236153559, "PLAIN-3161": 0.5396140508112093, "PLAIN-3171": 0.06978419164318803, "PLAIN-3181": 0.07839826897867533, "PLAIN-3191": 0.35185668051157637, "PLAIN-3201": 0.4209088765951527, "PLAIN-3211": 0.0, "PLAIN-3221": 0.0521442648465579, "PLAIN-3231": 0.0, "PLAIN-3241": 0.43056322872474745, "PLAIN-3251": 0.2155089134289123, "PLAIN-3261": 0.11591982604869, "PLAIN-3271": 0.0, "PLAIN-3281": 0.20436662668128655, "PLAIN-3292": 0.4734734392872383, "PLAIN-3302": 0.1599789509135779, "PLAIN-3312": 0.2763339555934507, "PLAIN-3322": 0.06943122193677727, "PLAIN-3332": 0.7601167308693849, "PLAIN-3342": 0.7653606369886217, "PLAIN-3352": 0.0, "PLAIN-3362": 0.42142110103371916, "PLAIN-3372": 0.227626228959616, "PLAIN-3382": 0.5889300098836854, "PLAIN-3392": 0.0, "PLAIN-3402": 0.0, "PLAIN-3412": 0.22461013156664808, "PLAIN-3422": 0.0, "PLAIN-3432": 0.21222636597291458, "PLAIN-3442": 0.14074267220463055, "PLAIN-3452": 0.39366440234814437, "PLAIN-3462": 0.5847192329112588, "PLAIN-3472": 0.08036018528064806}, "Recall@20": {"PLAIN-2": 0.4166666666666667, "PLAIN-12": 0.13333333333333333, "PLAIN-23": 0.06666666666666667, "PLAIN-33": 0.125, "PLAIN-44": 0.05172413793103448, "PLAIN-56": 0.2, "PLAIN-68": 0.07272727272727272, "PLAIN-78": 0.03225806451612903, "PLAIN-91": 0.15625, "PLAIN-102": 0.08333333333333333, "PLAIN-112": 0.08928571428571429, "PLAIN-123": 0.0196078431372549, "PLAIN-133": 0.027777777777777776, "PLAIN-143": 0.17647058823529413, "PLAIN-153": 0.23404255319148937, "PLAIN-165": 0.2222222222222222, "PLAIN-175": 0.10810810810810811, "PLAIN-186": 0.041666666666666664, "PLAIN-196": 0.014492753623188406, "PLAIN-207": 0.14545454545454545, "PLAIN-217": 0.16666666666666666, "PLAIN-227": 0.025, "PLAIN-238": 0.0, "PLAIN-248": 0.16129032258064516, "PLAIN-259": 0.21428571428571427, "PLAIN-270": 0.057692307692307696, "PLAIN-280": 0.25, "PLAIN-291": 0.3076923076923077, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.5, "PLAIN-344": 0.11475409836065574, "PLAIN-358": 0.3333333333333333, "PLAIN-371": 0.5, "PLAIN-383": 0.4, "PLAIN-395": 0.0, "PLAIN-407": 0.3333333333333333, "PLAIN-418": 0.6666666666666666, "PLAIN-430": 0.375, "PLAIN-441": 0.8, "PLAIN-457": 0.0, "PLAIN-468": 0.2, "PLAIN-478": 0.0, "PLAIN-488": 0.7333333333333333, "PLAIN-499": 0.11764705882352941, "PLAIN-510": 0.16666666666666666, "PLAIN-520": 0.0, "PLAIN-531": 0.03896103896103896, "PLAIN-541": 0.16666666666666666, "PLAIN-551": 1.0, "PLAIN-561": 0.12, "PLAIN-571": 0.42857142857142855, "PLAIN-583": 0.022727272727272728, "PLAIN-593": 0.3333333333333333, "PLAIN-603": 0.0, "PLAIN-613": 0.1, "PLAIN-623": 0.12195121951219512, "PLAIN-634": 0.14285714285714285, "PLAIN-645": 0.0, "PLAIN-660": 0.03368421052631579, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.375, "PLAIN-711": 0.21052631578947367, "PLAIN-721": 0.68, "PLAIN-731": 0.2037037037037037, "PLAIN-741": 0.47368421052631576, "PLAIN-751": 0.0, "PLAIN-761": 0.16666666666666666, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.5, "PLAIN-806": 0.13541666666666666, "PLAIN-817": 1.0, "PLAIN-827": 0.040983606557377046, "PLAIN-838": 0.6666666666666666, "PLAIN-850": 0.19298245614035087, "PLAIN-872": 0.0, "PLAIN-882": 0.2, "PLAIN-892": 0.06521739130434782, "PLAIN-902": 0.5, "PLAIN-913": 0.23529411764705882, "PLAIN-924": 0.0625, "PLAIN-934": 0.09900990099009901, "PLAIN-946": 0.03225806451612903, "PLAIN-956": 0.05825242718446602, "PLAIN-966": 0.0, "PLAIN-977": 0.3333333333333333, "PLAIN-987": 0.3333333333333333, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.1, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.010638297872340425, "PLAIN-1066": 0.0, "PLAIN-1088": 0.4, "PLAIN-1098": 0.0, "PLAIN-1109": 0.11224489795918367, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.0625, "PLAIN-1151": 0.06535947712418301, "PLAIN-1161": 0.0, "PLAIN-1172": 0.05263157894736842, "PLAIN-1183": 0.08, "PLAIN-1193": 0.05555555555555555, "PLAIN-1203": 0.3333333333333333, "PLAIN-1214": 0.058823529411764705, "PLAIN-1225": 0.3333333333333333, "PLAIN-1236": 0.2, "PLAIN-1249": 0.0, "PLAIN-1262": 0.3, "PLAIN-1275": 0.16363636363636364, "PLAIN-1288": 0.02040816326530612, "PLAIN-1299": 0.03278688524590164, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5, "PLAIN-1331": 0.0, "PLAIN-1342": 0.3125, "PLAIN-1353": 0.0, "PLAIN-1363": 0.09090909090909091, "PLAIN-1374": 0.07692307692307693, "PLAIN-1387": 0.5833333333333334, "PLAIN-1398": 0.1553398058252427, "PLAIN-1409": 0.02766798418972332, "PLAIN-1419": 0.06153846153846154, "PLAIN-1429": 0.0, "PLAIN-1441": 0.04838709677419355, "PLAIN-1453": 0.036734693877551024, "PLAIN-1463": 0.20833333333333334, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.13333333333333333, "PLAIN-1516": 0.0, "PLAIN-1527": 0.06930693069306931, "PLAIN-1537": 0.13333333333333333, "PLAIN-1547": 0.0, "PLAIN-1557": 0.07692307692307693, "PLAIN-1568": 1.0, "PLAIN-1579": 0.2, "PLAIN-1590": 0.05555555555555555, "PLAIN-1601": 0.0945945945945946, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.030162412993039442, "PLAIN-1645": 0.16666666666666666, "PLAIN-1656": 0.03571428571428571, "PLAIN-1667": 0.06918238993710692, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 0.9375, "PLAIN-1721": 0.0967741935483871, "PLAIN-1731": 0.6666666666666666, "PLAIN-1741": 0.04523809523809524, "PLAIN-1752": 1.0, "PLAIN-1762": 0.5, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 0.13793103448275862, "PLAIN-1817": 0.2222222222222222, "PLAIN-1837": 0.08673469387755102, "PLAIN-1847": 0.18181818181818182, "PLAIN-1857": 0.24242424242424243, "PLAIN-1867": 0.14285714285714285, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.023758099352051837, "PLAIN-1919": 0.3333333333333333, "PLAIN-1929": 0.3, "PLAIN-1940": 0.16666666666666666, "PLAIN-1950": 0.23076923076923078, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.28125, "PLAIN-1995": 0.09090909090909091, "PLAIN-2009": 0.375, "PLAIN-2019": 0.6666666666666666, "PLAIN-2030": 0.0, "PLAIN-2040": 0.05303030303030303, "PLAIN-2051": 0.030985915492957747, "PLAIN-2061": 0.04390243902439024, "PLAIN-2071": 0.15555555555555556, "PLAIN-2081": 1.0, "PLAIN-2092": 0.5, "PLAIN-2102": 0.030878859857482184, "PLAIN-2113": 0.0, "PLAIN-2124": 0.35294117647058826, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.5, "PLAIN-2167": 0.0, "PLAIN-2177": 0.20833333333333334, "PLAIN-2187": 0.2222222222222222, "PLAIN-2197": 0.1724137931034483, "PLAIN-2209": 0.0, "PLAIN-2220": 0.4444444444444444, "PLAIN-2230": 0.0, "PLAIN-2240": 0.36363636363636365, "PLAIN-2250": 0.0, "PLAIN-2261": 0.11666666666666667, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.4, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.09278350515463918, "PLAIN-2343": 0.0, "PLAIN-2354": 0.4, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.3684210526315789, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.13043478260869565, "PLAIN-2440": 0.047619047619047616, "PLAIN-2450": 0.07894736842105263, "PLAIN-2460": 0.0967741935483871, "PLAIN-2470": 0.1643835616438356, "PLAIN-2480": 0.16666666666666666, "PLAIN-2490": 0.13043478260869565, "PLAIN-2500": 0.42857142857142855, "PLAIN-2510": 0.21739130434782608, "PLAIN-2520": 0.0821917808219178, "PLAIN-2530": 0.20833333333333334, "PLAIN-2540": 0.38095238095238093, "PLAIN-2550": 0.04, "PLAIN-2560": 0.47619047619047616, "PLAIN-2570": 0.075, "PLAIN-2580": 0.08695652173913043, "PLAIN-2590": 0.07936507936507936, "PLAIN-2600": 0.4, "PLAIN-2610": 0.11475409836065574, "PLAIN-2620": 0.18181818181818182, "PLAIN-2630": 0.1875, "PLAIN-2640": 0.22916666666666666, "PLAIN-2650": 0.2413793103448276, "PLAIN-2660": 0.14814814814814814, "PLAIN-2670": 0.05714285714285714, "PLAIN-2680": 0.35714285714285715, "PLAIN-2690": 0.20930232558139536, "PLAIN-2700": 0.2, "PLAIN-2710": 0.23809523809523808, "PLAIN-2720": 0.05555555555555555, "PLAIN-2730": 0.20689655172413793, "PLAIN-2740": 0.12, "PLAIN-2750": 0.15254237288135594, "PLAIN-2760": 0.25, "PLAIN-2770": 0.21951219512195122, "PLAIN-2780": 0.3181818181818182, "PLAIN-2790": 0.15254237288135594, "PLAIN-2800": 0.08333333333333333, "PLAIN-2810": 0.125, "PLAIN-2820": 0.125, "PLAIN-2830": 0.3333333333333333, "PLAIN-2840": 0.0, "PLAIN-2850": 0.1111111111111111, "PLAIN-2860": 1.0, "PLAIN-2870": 0.2, "PLAIN-2880": 0.0, "PLAIN-2890": 0.034482758620689655, "PLAIN-2900": 0.08333333333333333, "PLAIN-2910": 0.06666666666666667, "PLAIN-2920": 0.0, "PLAIN-2930": 0.14285714285714285, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.5, "PLAIN-2970": 0.4444444444444444, "PLAIN-2981": 0.14285714285714285, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.6666666666666666, "PLAIN-3063": 0.25, "PLAIN-3074": 1.0, "PLAIN-3085": 1.0, "PLAIN-3097": 0.5, "PLAIN-3116": 0.0, "PLAIN-3131": 0.3333333333333333, "PLAIN-3141": 0.029411764705882353, "PLAIN-3151": 0.5, "PLAIN-3161": 0.13953488372093023, "PLAIN-3171": 0.029411764705882353, "PLAIN-3181": 0.037037037037037035, "PLAIN-3191": 0.1875, "PLAIN-3201": 0.09375, "PLAIN-3211": 0.0, "PLAIN-3221": 0.125, "PLAIN-3231": 0.25, "PLAIN-3241": 0.06896551724137931, "PLAIN-3251": 0.09523809523809523, "PLAIN-3261": 0.07692307692307693, "PLAIN-3271": 0.0, "PLAIN-3281": 0.038461538461538464, "PLAIN-3292": 0.7142857142857143, "PLAIN-3302": 0.08108108108108109, "PLAIN-3312": 0.5384615384615384, "PLAIN-3322": 0.02531645569620253, "PLAIN-3332": 0.6428571428571429, "PLAIN-3342": 0.6666666666666666, "PLAIN-3352": 0.02564102564102564, "PLAIN-3362": 0.18181818181818182, "PLAIN-3372": 0.07142857142857142, "PLAIN-3382": 0.5714285714285714, "PLAIN-3392": 0.13043478260869565, "PLAIN-3402": 0.07142857142857142, "PLAIN-3412": 0.061224489795918366, "PLAIN-3422": 0.0967741935483871, "PLAIN-3432": 0.09523809523809523, "PLAIN-3442": 0.034482758620689655, "PLAIN-3452": 0.29411764705882354, "PLAIN-3462": 0.1875, "PLAIN-3472": 0.058823529411764705}, "MRR@10": {"PLAIN-2": 1.0, "PLAIN-12": 0.3333333333333333, "PLAIN-23": 0.5, "PLAIN-33": 0.5, "PLAIN-44": 0.25, "PLAIN-56": 1.0, "PLAIN-68": 1.0, "PLAIN-78": 0.3333333333333333, "PLAIN-91": 1.0, "PLAIN-102": 1.0, "PLAIN-112": 1.0, "PLAIN-123": 0.5, "PLAIN-133": 0.3333333333333333, "PLAIN-143": 1.0, "PLAIN-153": 1.0, "PLAIN-165": 0.5, "PLAIN-175": 1.0, "PLAIN-186": 1.0, "PLAIN-196": 1.0, "PLAIN-207": 0.3333333333333333, "PLAIN-217": 0.125, "PLAIN-227": 0.3333333333333333, "PLAIN-238": 0.0, "PLAIN-248": 1.0, "PLAIN-259": 1.0, "PLAIN-270": 1.0, "PLAIN-280": 0.5, "PLAIN-291": 1.0, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 1.0, "PLAIN-344": 0.5, "PLAIN-358": 0.06666666666666667, "PLAIN-371": 0.5, "PLAIN-383": 0.3333333333333333, "PLAIN-395": 0.0, "PLAIN-407": 1.0, "PLAIN-418": 1.0, "PLAIN-430": 1.0, "PLAIN-441": 1.0, "PLAIN-457": 0.0, "PLAIN-468": 1.0, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 1.0, "PLAIN-510": 0.07692307692307693, "PLAIN-520": 0.0, "PLAIN-531": 0.5, "PLAIN-541": 0.2, "PLAIN-551": 1.0, "PLAIN-561": 1.0, "PLAIN-571": 0.25, "PLAIN-583": 1.0, "PLAIN-593": 1.0, "PLAIN-603": 0.0, "PLAIN-613": 0.3333333333333333, "PLAIN-623": 1.0, "PLAIN-634": 1.0, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 1.0, "PLAIN-711": 1.0, "PLAIN-721": 1.0, "PLAIN-731": 1.0, "PLAIN-741": 1.0, "PLAIN-751": 0.0, "PLAIN-761": 0.3333333333333333, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 1.0, "PLAIN-806": 1.0, "PLAIN-817": 0.25, "PLAIN-827": 1.0, "PLAIN-838": 1.0, "PLAIN-850": 1.0, "PLAIN-872": 0.0, "PLAIN-882": 1.0, "PLAIN-892": 1.0, "PLAIN-902": 1.0, "PLAIN-913": 1.0, "PLAIN-924": 1.0, "PLAIN-934": 1.0, "PLAIN-946": 1.0, "PLAIN-956": 1.0, "PLAIN-966": 0.0, "PLAIN-977": 1.0, "PLAIN-987": 0.07692307692307693, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.16666666666666666, "PLAIN-1066": 0.0, "PLAIN-1088": 1.0, "PLAIN-1098": 0.0, "PLAIN-1109": 1.0, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 1.0, "PLAIN-1151": 1.0, "PLAIN-1161": 0.0, "PLAIN-1172": 1.0, "PLAIN-1183": 0.25, "PLAIN-1193": 0.09090909090909091, "PLAIN-1203": 1.0, "PLAIN-1214": 0.06666666666666667, "PLAIN-1225": 1.0, "PLAIN-1236": 1.0, "PLAIN-1249": 0.0, "PLAIN-1262": 1.0, "PLAIN-1275": 1.0, "PLAIN-1288": 1.0, "PLAIN-1299": 1.0, "PLAIN-1309": 0.0, "PLAIN-1320": 1.0, "PLAIN-1331": 0.0, "PLAIN-1342": 1.0, "PLAIN-1353": 0.0, "PLAIN-1363": 1.0, "PLAIN-1374": 0.16666666666666666, "PLAIN-1387": 1.0, "PLAIN-1398": 1.0, "PLAIN-1409": 1.0, "PLAIN-1419": 1.0, "PLAIN-1429": 0.0, "PLAIN-1441": 1.0, "PLAIN-1453": 0.5, "PLAIN-1463": 1.0, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 1.0, "PLAIN-1516": 0.0, "PLAIN-1527": 1.0, "PLAIN-1537": 0.5, "PLAIN-1547": 0.0, "PLAIN-1557": 0.5, "PLAIN-1568": 1.0, "PLAIN-1579": 1.0, "PLAIN-1590": 0.125, "PLAIN-1601": 1.0, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 1.0, "PLAIN-1645": 1.0, "PLAIN-1656": 0.06666666666666667, "PLAIN-1667": 1.0, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 1.0, "PLAIN-1731": 1.0, "PLAIN-1741": 1.0, "PLAIN-1752": 0.05, "PLAIN-1762": 1.0, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 1.0, "PLAIN-1837": 1.0, "PLAIN-1847": 1.0, "PLAIN-1857": 1.0, "PLAIN-1867": 1.0, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 1.0, "PLAIN-1919": 1.0, "PLAIN-1929": 0.3333333333333333, "PLAIN-1940": 1.0, "PLAIN-1950": 0.5, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 1.0, "PLAIN-1995": 0.06666666666666667, "PLAIN-2009": 0.3333333333333333, "PLAIN-2019": 1.0, "PLAIN-2030": 0.0, "PLAIN-2040": 1.0, "PLAIN-2051": 1.0, "PLAIN-2061": 1.0, "PLAIN-2071": 1.0, "PLAIN-2081": 1.0, "PLAIN-2092": 1.0, "PLAIN-2102": 1.0, "PLAIN-2113": 0.0, "PLAIN-2124": 1.0, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 1.0, "PLAIN-2167": 0.0, "PLAIN-2177": 0.5, "PLAIN-2187": 1.0, "PLAIN-2197": 1.0, "PLAIN-2209": 0.0, "PLAIN-2220": 1.0, "PLAIN-2230": 0.0, "PLAIN-2240": 1.0, "PLAIN-2250": 0.0, "PLAIN-2261": 1.0, "PLAIN-2271": 0.0, "PLAIN-2281": 0.5, "PLAIN-2291": 1.0, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 1.0, "PLAIN-2343": 0.0, "PLAIN-2354": 1.0, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.5, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 1.0, "PLAIN-2440": 0.14285714285714285, "PLAIN-2450": 1.0, "PLAIN-2460": 0.5, "PLAIN-2470": 1.0, "PLAIN-2480": 0.5, "PLAIN-2490": 1.0, "PLAIN-2500": 0.16666666666666666, "PLAIN-2510": 0.5, "PLAIN-2520": 0.5, "PLAIN-2530": 1.0, "PLAIN-2540": 1.0, "PLAIN-2550": 0.08333333333333333, "PLAIN-2560": 1.0, "PLAIN-2570": 1.0, "PLAIN-2580": 1.0, "PLAIN-2590": 0.5, "PLAIN-2600": 1.0, "PLAIN-2610": 0.3333333333333333, "PLAIN-2620": 1.0, "PLAIN-2630": 1.0, "PLAIN-2640": 1.0, "PLAIN-2650": 1.0, "PLAIN-2660": 0.25, "PLAIN-2670": 1.0, "PLAIN-2680": 0.5, "PLAIN-2690": 1.0, "PLAIN-2700": 0.5, "PLAIN-2710": 1.0, "PLAIN-2720": 1.0, "PLAIN-2730": 1.0, "PLAIN-2740": 1.0, "PLAIN-2750": 1.0, "PLAIN-2760": 0.16666666666666666, "PLAIN-2770": 1.0, "PLAIN-2780": 1.0, "PLAIN-2790": 1.0, "PLAIN-2800": 0.25, "PLAIN-2810": 0.14285714285714285, "PLAIN-2820": 0.2, "PLAIN-2830": 1.0, "PLAIN-2840": 0.0, "PLAIN-2850": 0.5, "PLAIN-2860": 1.0, "PLAIN-2870": 0.14285714285714285, "PLAIN-2880": 0.0, "PLAIN-2890": 0.2, "PLAIN-2900": 0.14285714285714285, "PLAIN-2910": 1.0, "PLAIN-2920": 0.0, "PLAIN-2930": 1.0, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.25, "PLAIN-2970": 1.0, "PLAIN-2981": 1.0, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 1.0, "PLAIN-3063": 0.07692307692307693, "PLAIN-3074": 1.0, "PLAIN-3085": 0.25, "PLAIN-3097": 1.0, "PLAIN-3116": 0.0, "PLAIN-3131": 1.0, "PLAIN-3141": 1.0, "PLAIN-3151": 1.0, "PLAIN-3161": 1.0, "PLAIN-3171": 0.2, "PLAIN-3181": 0.16666666666666666, "PLAIN-3191": 1.0, "PLAIN-3201": 1.0, "PLAIN-3211": 0.0, "PLAIN-3221": 0.1, "PLAIN-3231": 0.07142857142857142, "PLAIN-3241": 1.0, "PLAIN-3251": 0.25, "PLAIN-3261": 0.2, "PLAIN-3271": 0.0, "PLAIN-3281": 0.5, "PLAIN-3292": 0.3333333333333333, "PLAIN-3302": 0.3333333333333333, "PLAIN-3312": 0.5, "PLAIN-3322": 0.125, "PLAIN-3332": 1.0, "PLAIN-3342": 1.0, "PLAIN-3352": 0.0625, "PLAIN-3362": 1.0, "PLAIN-3372": 0.5, "PLAIN-3382": 1.0, "PLAIN-3392": 0.07692307692307693, "PLAIN-3402": 0.07692307692307693, "PLAIN-3412": 1.0, "PLAIN-3422": 0.08333333333333333, "PLAIN-3432": 0.5, "PLAIN-3442": 1.0, "PLAIN-3452": 1.0, "PLAIN-3462": 0.5, "PLAIN-3472": 0.1111111111111111}}} diff --git a/evals/benchmark/results-adaptive/nfcorpus/ir-adm0.30.jsonl b/evals/benchmark/results-adaptive/nfcorpus/ir-adm0.30.jsonl new file mode 100644 index 000000000..65edeaef2 --- /dev/null +++ b/evals/benchmark/results-adaptive/nfcorpus/ir-adm0.30.jsonl @@ -0,0 +1 @@ +{"dataset": "nfcorpus", "run_tag": "adm0.30", "aggregated": {"nDCG@10": 0.3753, "Recall@20": 0.2153, "MRR@10": 0.5917}, "per_query": {"nDCG@10": {"PLAIN-2": 0.7495470737433306, "PLAIN-12": 0.16950960215087577, "PLAIN-23": 0.46459600558095715, "PLAIN-33": 0.2640886107812416, "PLAIN-44": 0.23888170027858516, "PLAIN-56": 0.814153485068656, "PLAIN-68": 0.47459215156136747, "PLAIN-78": 0.10206510770605066, "PLAIN-91": 0.6947482816557203, "PLAIN-102": 0.21594229312581256, "PLAIN-112": 0.627217471333092, "PLAIN-123": 0.08039138809008574, "PLAIN-133": 0.13347537859706934, "PLAIN-143": 0.6917350424665718, "PLAIN-153": 1.0, "PLAIN-165": 0.45365777747025504, "PLAIN-175": 0.42027556825478546, "PLAIN-186": 0.22009176629808017, "PLAIN-196": 0.22009176629808017, "PLAIN-207": 0.4230293752959408, "PLAIN-217": 0.13305201013572898, "PLAIN-227": 0.11004588314904008, "PLAIN-238": 0.0, "PLAIN-248": 0.38113435412202, "PLAIN-259": 0.3589542101716347, "PLAIN-270": 0.21222636597291458, "PLAIN-280": 0.4131279133289227, "PLAIN-291": 0.43231813227099475, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.6131471927654584, "PLAIN-344": 0.4777009208974601, "PLAIN-358": 0.0, "PLAIN-371": 0.23981246656813146, "PLAIN-383": 0.16958010263680806, "PLAIN-395": 0.0, "PLAIN-407": 0.6387878864795979, "PLAIN-418": 0.8670870086853021, "PLAIN-430": 0.6174886499090687, "PLAIN-441": 0.777747077360283, "PLAIN-457": 0.0, "PLAIN-468": 0.4318845862516251, "PLAIN-478": 0.0, "PLAIN-488": 0.9363792118010483, "PLAIN-499": 0.5174612499126158, "PLAIN-510": 0.0, "PLAIN-520": 0.0, "PLAIN-531": 0.5368103404171507, "PLAIN-541": 0.11706259313796354, "PLAIN-551": 0.7903864795495061, "PLAIN-561": 0.43735247915031, "PLAIN-571": 0.22471947398559278, "PLAIN-583": 0.22009176629808017, "PLAIN-593": 0.46927872602275644, "PLAIN-603": 0.0, "PLAIN-613": 0.08514311764162098, "PLAIN-623": 0.33013764944712026, "PLAIN-634": 0.424926013816671, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.4772656487866461, "PLAIN-711": 0.7605591431973849, "PLAIN-721": 1.0, "PLAIN-731": 0.642186726668901, "PLAIN-741": 0.8701249883466593, "PLAIN-751": 0.0, "PLAIN-761": 0.15130120674940672, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.644824486427155, "PLAIN-806": 0.857980942822373, "PLAIN-817": 0.38685280723454163, "PLAIN-827": 0.7564477297898766, "PLAIN-838": 0.7767471075223497, "PLAIN-850": 0.6936634693435663, "PLAIN-872": 0.0, "PLAIN-882": 0.4690000933206748, "PLAIN-892": 0.3557772116892465, "PLAIN-902": 0.6366824387328317, "PLAIN-913": 0.800693766409882, "PLAIN-924": 0.30523488393970116, "PLAIN-934": 0.7289115290270202, "PLAIN-946": 0.22009176629808017, "PLAIN-956": 0.6500332220485792, "PLAIN-966": 0.0, "PLAIN-977": 0.46927872602275644, "PLAIN-987": 0.0, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.48993086854170786, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.07839826897867533, "PLAIN-1066": 0.0, "PLAIN-1088": 0.5531464700081437, "PLAIN-1098": 0.0, "PLAIN-1109": 0.6780494608112012, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.22009176629808017, "PLAIN-1151": 0.7000949683025056, "PLAIN-1161": 0.0, "PLAIN-1172": 0.22009176629808017, "PLAIN-1183": 0.1731866333482261, "PLAIN-1193": 0.0, "PLAIN-1203": 0.5036067026046991, "PLAIN-1214": 0.0, "PLAIN-1225": 0.6874708847784536, "PLAIN-1236": 0.3391602052736161, "PLAIN-1249": 0.0, "PLAIN-1262": 0.4537425745411855, "PLAIN-1275": 0.7007731679885374, "PLAIN-1288": 0.49828993591603105, "PLAIN-1299": 0.3589542101716347, "PLAIN-1309": 0.0, "PLAIN-1320": 0.6238470455881188, "PLAIN-1331": 0.0, "PLAIN-1342": 0.5861163849908828, "PLAIN-1353": 0.0, "PLAIN-1363": 0.22009176629808017, "PLAIN-1374": 0.07839826897867533, "PLAIN-1387": 0.7819835854675755, "PLAIN-1398": 1.0, "PLAIN-1409": 0.48068140169045714, "PLAIN-1419": 0.4690000933206748, "PLAIN-1429": 0.0, "PLAIN-1441": 0.627507133061656, "PLAIN-1453": 0.47938213678331176, "PLAIN-1463": 0.46270962158308354, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.33013764944712026, "PLAIN-1516": 0.0, "PLAIN-1527": 0.8482378089219645, "PLAIN-1537": 0.42331179510529354, "PLAIN-1547": 0.0, "PLAIN-1557": 0.2489083270225946, "PLAIN-1568": 1.0, "PLAIN-1579": 0.3391602052736161, "PLAIN-1590": 0.08514311764162098, "PLAIN-1601": 0.800693766409882, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.8643145546088337, "PLAIN-1645": 0.30260241349881345, "PLAIN-1656": 0.0, "PLAIN-1667": 0.8572048559638628, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 0.28952298823485745, "PLAIN-1731": 0.7653606369886217, "PLAIN-1741": 1.0, "PLAIN-1752": 0.0, "PLAIN-1762": 0.4856486183364634, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 0.568003577662161, "PLAIN-1837": 1.0, "PLAIN-1847": 0.22009176629808017, "PLAIN-1857": 0.627507133061656, "PLAIN-1867": 0.27487633291429087, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.7917267193679839, "PLAIN-1919": 0.866947989864271, "PLAIN-1929": 0.1736666713479918, "PLAIN-1940": 0.3589542101716347, "PLAIN-1950": 0.3222722491219547, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.9363792118010483, "PLAIN-1995": 0.0, "PLAIN-2009": 0.31442295467135667, "PLAIN-2019": 0.6364391809889587, "PLAIN-2030": 0.0, "PLAIN-2040": 0.5637884576902256, "PLAIN-2051": 0.6870165078530993, "PLAIN-2061": 0.933745776545611, "PLAIN-2071": 0.5771493379489847, "PLAIN-2081": 1.0, "PLAIN-2092": 0.6489315753318465, "PLAIN-2102": 0.8318477135310893, "PLAIN-2113": 0.0, "PLAIN-2124": 0.6906478832608419, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.644824486427155, "PLAIN-2167": 0.0, "PLAIN-2177": 0.46692474155501895, "PLAIN-2187": 0.3091941646995599, "PLAIN-2197": 0.8603818544462509, "PLAIN-2209": 0.0, "PLAIN-2220": 0.5526193270751085, "PLAIN-2230": 0.0, "PLAIN-2240": 0.43735247915031, "PLAIN-2250": 0.0, "PLAIN-2261": 0.715550648768261, "PLAIN-2271": 0.0, "PLAIN-2281": 0.6309297535714575, "PLAIN-2291": 0.3391602052736161, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.44988737637276544, "PLAIN-2343": 0.0, "PLAIN-2354": 0.5531464700081437, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.31204907722178066, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.4832964623025934, "PLAIN-2440": 0.07336392209936005, "PLAIN-2450": 0.28952298823485745, "PLAIN-2460": 0.2873703978647362, "PLAIN-2470": 0.6923056707325975, "PLAIN-2480": 0.214167553296389, "PLAIN-2490": 0.4104014996484018, "PLAIN-2500": 0.06760523069231983, "PLAIN-2510": 0.7093753650484099, "PLAIN-2520": 0.2085303067457527, "PLAIN-2530": 0.9633180389503199, "PLAIN-2540": 0.5907275541269412, "PLAIN-2550": 0.0, "PLAIN-2560": 0.866947989864271, "PLAIN-2570": 0.4205032672422578, "PLAIN-2580": 0.3941769500918012, "PLAIN-2590": 0.1581173338315241, "PLAIN-2600": 0.6542721934360559, "PLAIN-2610": 0.3052695562492681, "PLAIN-2620": 0.5549841905774386, "PLAIN-2630": 0.7917267193679839, "PLAIN-2640": 0.5519034797982907, "PLAIN-2650": 0.5352543167750639, "PLAIN-2660": 0.31954962756492084, "PLAIN-2670": 0.37456050428247883, "PLAIN-2680": 0.43478330062838466, "PLAIN-2690": 0.5760823635418191, "PLAIN-2700": 0.28893989447363055, "PLAIN-2710": 0.6665181205254729, "PLAIN-2720": 0.2489083270225946, "PLAIN-2730": 0.6338426833522841, "PLAIN-2740": 0.6854074373766073, "PLAIN-2750": 0.5946665900227162, "PLAIN-2760": 0.2469526638494753, "PLAIN-2770": 0.415769594467977, "PLAIN-2780": 0.4769783244417837, "PLAIN-2790": 0.8217429957880231, "PLAIN-2800": 0.15472892909388752, "PLAIN-2810": 0.05184819766823351, "PLAIN-2820": 0.15619484341784115, "PLAIN-2830": 0.6387878864795979, "PLAIN-2840": 0.0, "PLAIN-2850": 0.24014860122439108, "PLAIN-2860": 1.0, "PLAIN-2870": 0.08442111803551768, "PLAIN-2880": 0.0, "PLAIN-2890": 0.15537907391531905, "PLAIN-2900": 0.09382844813642036, "PLAIN-2910": 0.3239134396887189, "PLAIN-2920": 0.0, "PLAIN-2930": 0.415081693065718, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.3273949503887395, "PLAIN-2970": 0.38062652754129855, "PLAIN-2981": 0.43122038713502525, "PLAIN-2991": 0.38685280723454163, "PLAIN-3001": 0.38685280723454163, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.7653606369886217, "PLAIN-3063": 0.0, "PLAIN-3074": 1.0, "PLAIN-3085": 0.5012658353418871, "PLAIN-3097": 0.4770382338730848, "PLAIN-3116": 0.0, "PLAIN-3131": 0.4599311598225878, "PLAIN-3141": 0.2996484034259831, "PLAIN-3151": 0.7255652432906947, "PLAIN-3161": 0.4906128388857768, "PLAIN-3171": 0.06978419164318803, "PLAIN-3181": 0.07839826897867533, "PLAIN-3191": 0.35185668051157637, "PLAIN-3201": 0.37212364421416444, "PLAIN-3211": 0.0, "PLAIN-3221": 0.0, "PLAIN-3231": 0.0, "PLAIN-3241": 0.43056322872474745, "PLAIN-3251": 0.2155089134289123, "PLAIN-3261": 0.14982420171299154, "PLAIN-3271": 0.0, "PLAIN-3281": 0.20436662668128655, "PLAIN-3292": 0.4689564279260148, "PLAIN-3302": 0.1599789509135779, "PLAIN-3312": 0.3170759870637039, "PLAIN-3322": 0.06943122193677727, "PLAIN-3332": 0.8909491779135381, "PLAIN-3342": 0.7653606369886217, "PLAIN-3352": 0.0, "PLAIN-3362": 0.4634153650075699, "PLAIN-3372": 0.15537907391531905, "PLAIN-3382": 0.4484467219534551, "PLAIN-3392": 0.0, "PLAIN-3402": 0.0, "PLAIN-3412": 0.23170768250378496, "PLAIN-3422": 0.0, "PLAIN-3432": 0.2934556883974402, "PLAIN-3442": 0.07037133610231527, "PLAIN-3452": 0.5240044007925084, "PLAIN-3462": 0.6067216003536936, "PLAIN-3472": 0.08036018528064806}, "Recall@20": {"PLAIN-2": 0.4166666666666667, "PLAIN-12": 0.13333333333333333, "PLAIN-23": 0.06666666666666667, "PLAIN-33": 0.125, "PLAIN-44": 0.05172413793103448, "PLAIN-56": 0.2, "PLAIN-68": 0.07272727272727272, "PLAIN-78": 0.03225806451612903, "PLAIN-91": 0.15625, "PLAIN-102": 0.08333333333333333, "PLAIN-112": 0.08928571428571429, "PLAIN-123": 0.0196078431372549, "PLAIN-133": 0.027777777777777776, "PLAIN-143": 0.17647058823529413, "PLAIN-153": 0.23404255319148937, "PLAIN-165": 0.2222222222222222, "PLAIN-175": 0.10810810810810811, "PLAIN-186": 0.041666666666666664, "PLAIN-196": 0.014492753623188406, "PLAIN-207": 0.14545454545454545, "PLAIN-217": 0.16666666666666666, "PLAIN-227": 0.025, "PLAIN-238": 0.0, "PLAIN-248": 0.16129032258064516, "PLAIN-259": 0.21428571428571427, "PLAIN-270": 0.057692307692307696, "PLAIN-280": 0.25, "PLAIN-291": 0.3076923076923077, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.5, "PLAIN-344": 0.11475409836065574, "PLAIN-358": 0.3333333333333333, "PLAIN-371": 0.5, "PLAIN-383": 0.4, "PLAIN-395": 0.0, "PLAIN-407": 0.3333333333333333, "PLAIN-418": 0.6666666666666666, "PLAIN-430": 0.375, "PLAIN-441": 0.8, "PLAIN-457": 0.0, "PLAIN-468": 0.2, "PLAIN-478": 0.0, "PLAIN-488": 0.7333333333333333, "PLAIN-499": 0.11764705882352941, "PLAIN-510": 0.16666666666666666, "PLAIN-520": 0.0, "PLAIN-531": 0.03571428571428571, "PLAIN-541": 0.16666666666666666, "PLAIN-551": 1.0, "PLAIN-561": 0.12, "PLAIN-571": 0.42857142857142855, "PLAIN-583": 0.022727272727272728, "PLAIN-593": 0.3333333333333333, "PLAIN-603": 0.0, "PLAIN-613": 0.1, "PLAIN-623": 0.12195121951219512, "PLAIN-634": 0.14285714285714285, "PLAIN-645": 0.0, "PLAIN-660": 0.03368421052631579, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.375, "PLAIN-711": 0.21052631578947367, "PLAIN-721": 0.68, "PLAIN-731": 0.2037037037037037, "PLAIN-741": 0.47368421052631576, "PLAIN-751": 0.0, "PLAIN-761": 0.16666666666666666, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.5, "PLAIN-806": 0.13541666666666666, "PLAIN-817": 1.0, "PLAIN-827": 0.040983606557377046, "PLAIN-838": 0.6666666666666666, "PLAIN-850": 0.19298245614035087, "PLAIN-872": 0.0, "PLAIN-882": 0.2, "PLAIN-892": 0.06521739130434782, "PLAIN-902": 0.5, "PLAIN-913": 0.23529411764705882, "PLAIN-924": 0.0625, "PLAIN-934": 0.09900990099009901, "PLAIN-946": 0.0967741935483871, "PLAIN-956": 0.05825242718446602, "PLAIN-966": 0.0, "PLAIN-977": 0.3333333333333333, "PLAIN-987": 0.3333333333333333, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.1, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.010638297872340425, "PLAIN-1066": 0.0, "PLAIN-1088": 0.4, "PLAIN-1098": 0.0, "PLAIN-1109": 0.11224489795918367, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.0625, "PLAIN-1151": 0.06535947712418301, "PLAIN-1161": 0.0, "PLAIN-1172": 0.05263157894736842, "PLAIN-1183": 0.08, "PLAIN-1193": 0.05555555555555555, "PLAIN-1203": 0.3333333333333333, "PLAIN-1214": 0.058823529411764705, "PLAIN-1225": 0.3333333333333333, "PLAIN-1236": 0.2, "PLAIN-1249": 0.0, "PLAIN-1262": 0.3, "PLAIN-1275": 0.16363636363636364, "PLAIN-1288": 0.02040816326530612, "PLAIN-1299": 0.03278688524590164, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5, "PLAIN-1331": 0.0, "PLAIN-1342": 0.3125, "PLAIN-1353": 0.0, "PLAIN-1363": 0.09090909090909091, "PLAIN-1374": 0.07692307692307693, "PLAIN-1387": 0.5833333333333334, "PLAIN-1398": 0.1553398058252427, "PLAIN-1409": 0.023715415019762844, "PLAIN-1419": 0.06153846153846154, "PLAIN-1429": 0.0, "PLAIN-1441": 0.04838709677419355, "PLAIN-1453": 0.0326530612244898, "PLAIN-1463": 0.20833333333333334, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.13333333333333333, "PLAIN-1516": 0.0, "PLAIN-1527": 0.06930693069306931, "PLAIN-1537": 0.13333333333333333, "PLAIN-1547": 0.0, "PLAIN-1557": 0.07692307692307693, "PLAIN-1568": 1.0, "PLAIN-1579": 0.2, "PLAIN-1590": 0.05555555555555555, "PLAIN-1601": 0.0945945945945946, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.030162412993039442, "PLAIN-1645": 0.16666666666666666, "PLAIN-1656": 0.03571428571428571, "PLAIN-1667": 0.06918238993710692, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 0.9375, "PLAIN-1721": 0.16129032258064516, "PLAIN-1731": 0.6666666666666666, "PLAIN-1741": 0.04523809523809524, "PLAIN-1752": 1.0, "PLAIN-1762": 0.5, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 0.13793103448275862, "PLAIN-1817": 0.2222222222222222, "PLAIN-1837": 0.08673469387755102, "PLAIN-1847": 0.18181818181818182, "PLAIN-1857": 0.24242424242424243, "PLAIN-1867": 0.14285714285714285, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.023758099352051837, "PLAIN-1919": 0.3333333333333333, "PLAIN-1929": 0.3, "PLAIN-1940": 0.16666666666666666, "PLAIN-1950": 0.23076923076923078, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.28125, "PLAIN-1995": 0.09090909090909091, "PLAIN-2009": 0.375, "PLAIN-2019": 0.6666666666666666, "PLAIN-2030": 0.16666666666666666, "PLAIN-2040": 0.05303030303030303, "PLAIN-2051": 0.030985915492957747, "PLAIN-2061": 0.04390243902439024, "PLAIN-2071": 0.15555555555555556, "PLAIN-2081": 1.0, "PLAIN-2092": 0.5, "PLAIN-2102": 0.030878859857482184, "PLAIN-2113": 0.0, "PLAIN-2124": 0.35294117647058826, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.5, "PLAIN-2167": 0.0, "PLAIN-2177": 0.20833333333333334, "PLAIN-2187": 0.2222222222222222, "PLAIN-2197": 0.1724137931034483, "PLAIN-2209": 0.0, "PLAIN-2220": 0.4444444444444444, "PLAIN-2230": 0.14285714285714285, "PLAIN-2240": 0.36363636363636365, "PLAIN-2250": 0.0, "PLAIN-2261": 0.11666666666666667, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.4, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.09278350515463918, "PLAIN-2343": 0.0, "PLAIN-2354": 0.4, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.3684210526315789, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.13043478260869565, "PLAIN-2440": 0.047619047619047616, "PLAIN-2450": 0.07894736842105263, "PLAIN-2460": 0.0967741935483871, "PLAIN-2470": 0.136986301369863, "PLAIN-2480": 0.16666666666666666, "PLAIN-2490": 0.17391304347826086, "PLAIN-2500": 0.42857142857142855, "PLAIN-2510": 0.21739130434782608, "PLAIN-2520": 0.0821917808219178, "PLAIN-2530": 0.20833333333333334, "PLAIN-2540": 0.38095238095238093, "PLAIN-2550": 0.04, "PLAIN-2560": 0.47619047619047616, "PLAIN-2570": 0.075, "PLAIN-2580": 0.08695652173913043, "PLAIN-2590": 0.07936507936507936, "PLAIN-2600": 0.4, "PLAIN-2610": 0.11475409836065574, "PLAIN-2620": 0.23636363636363636, "PLAIN-2630": 0.1875, "PLAIN-2640": 0.22916666666666666, "PLAIN-2650": 0.2413793103448276, "PLAIN-2660": 0.14814814814814814, "PLAIN-2670": 0.05714285714285714, "PLAIN-2680": 0.35714285714285715, "PLAIN-2690": 0.20930232558139536, "PLAIN-2700": 0.2, "PLAIN-2710": 0.23809523809523808, "PLAIN-2720": 0.05555555555555555, "PLAIN-2730": 0.20689655172413793, "PLAIN-2740": 0.12, "PLAIN-2750": 0.15254237288135594, "PLAIN-2760": 0.25, "PLAIN-2770": 0.21951219512195122, "PLAIN-2780": 0.3181818181818182, "PLAIN-2790": 0.15254237288135594, "PLAIN-2800": 0.08333333333333333, "PLAIN-2810": 0.125, "PLAIN-2820": 0.125, "PLAIN-2830": 0.3333333333333333, "PLAIN-2840": 0.0, "PLAIN-2850": 0.1111111111111111, "PLAIN-2860": 1.0, "PLAIN-2870": 0.2, "PLAIN-2880": 0.0, "PLAIN-2890": 0.034482758620689655, "PLAIN-2900": 0.08333333333333333, "PLAIN-2910": 0.06666666666666667, "PLAIN-2920": 0.0, "PLAIN-2930": 0.14285714285714285, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.5, "PLAIN-2970": 0.4444444444444444, "PLAIN-2981": 0.14285714285714285, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.6666666666666666, "PLAIN-3063": 0.25, "PLAIN-3074": 1.0, "PLAIN-3085": 1.0, "PLAIN-3097": 0.5, "PLAIN-3116": 0.0, "PLAIN-3131": 0.3333333333333333, "PLAIN-3141": 0.029411764705882353, "PLAIN-3151": 0.5, "PLAIN-3161": 0.13953488372093023, "PLAIN-3171": 0.029411764705882353, "PLAIN-3181": 0.037037037037037035, "PLAIN-3191": 0.1875, "PLAIN-3201": 0.09375, "PLAIN-3211": 0.0, "PLAIN-3221": 0.125, "PLAIN-3231": 0.25, "PLAIN-3241": 0.06896551724137931, "PLAIN-3251": 0.09523809523809523, "PLAIN-3261": 0.07692307692307693, "PLAIN-3271": 0.0, "PLAIN-3281": 0.038461538461538464, "PLAIN-3292": 0.7142857142857143, "PLAIN-3302": 0.08108108108108109, "PLAIN-3312": 0.5384615384615384, "PLAIN-3322": 0.02531645569620253, "PLAIN-3332": 0.6428571428571429, "PLAIN-3342": 0.6666666666666666, "PLAIN-3352": 0.0, "PLAIN-3362": 0.18181818181818182, "PLAIN-3372": 0.07142857142857142, "PLAIN-3382": 0.5714285714285714, "PLAIN-3392": 0.043478260869565216, "PLAIN-3402": 0.07142857142857142, "PLAIN-3412": 0.061224489795918366, "PLAIN-3422": 0.0967741935483871, "PLAIN-3432": 0.09523809523809523, "PLAIN-3442": 0.034482758620689655, "PLAIN-3452": 0.29411764705882354, "PLAIN-3462": 0.1875, "PLAIN-3472": 0.058823529411764705}, "MRR@10": {"PLAIN-2": 1.0, "PLAIN-12": 0.3333333333333333, "PLAIN-23": 0.5, "PLAIN-33": 0.5, "PLAIN-44": 0.25, "PLAIN-56": 1.0, "PLAIN-68": 1.0, "PLAIN-78": 0.3333333333333333, "PLAIN-91": 1.0, "PLAIN-102": 0.3333333333333333, "PLAIN-112": 1.0, "PLAIN-123": 0.5, "PLAIN-133": 0.3333333333333333, "PLAIN-143": 1.0, "PLAIN-153": 1.0, "PLAIN-165": 0.5, "PLAIN-175": 1.0, "PLAIN-186": 1.0, "PLAIN-196": 1.0, "PLAIN-207": 0.3333333333333333, "PLAIN-217": 0.125, "PLAIN-227": 0.3333333333333333, "PLAIN-238": 0.0, "PLAIN-248": 1.0, "PLAIN-259": 1.0, "PLAIN-270": 0.5, "PLAIN-280": 0.5, "PLAIN-291": 1.0, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 1.0, "PLAIN-344": 0.5, "PLAIN-358": 0.06666666666666667, "PLAIN-371": 0.5, "PLAIN-383": 0.3333333333333333, "PLAIN-395": 0.0, "PLAIN-407": 1.0, "PLAIN-418": 1.0, "PLAIN-430": 1.0, "PLAIN-441": 1.0, "PLAIN-457": 0.0, "PLAIN-468": 1.0, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 1.0, "PLAIN-510": 0.07692307692307693, "PLAIN-520": 0.0, "PLAIN-531": 0.5, "PLAIN-541": 0.2, "PLAIN-551": 1.0, "PLAIN-561": 1.0, "PLAIN-571": 0.25, "PLAIN-583": 1.0, "PLAIN-593": 1.0, "PLAIN-603": 0.0, "PLAIN-613": 0.2, "PLAIN-623": 1.0, "PLAIN-634": 1.0, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 1.0, "PLAIN-711": 1.0, "PLAIN-721": 1.0, "PLAIN-731": 1.0, "PLAIN-741": 1.0, "PLAIN-751": 0.0, "PLAIN-761": 0.3333333333333333, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 1.0, "PLAIN-806": 1.0, "PLAIN-817": 0.2, "PLAIN-827": 1.0, "PLAIN-838": 1.0, "PLAIN-850": 1.0, "PLAIN-872": 0.0, "PLAIN-882": 1.0, "PLAIN-892": 1.0, "PLAIN-902": 1.0, "PLAIN-913": 1.0, "PLAIN-924": 1.0, "PLAIN-934": 1.0, "PLAIN-946": 1.0, "PLAIN-956": 0.5, "PLAIN-966": 0.0, "PLAIN-977": 1.0, "PLAIN-987": 0.07692307692307693, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.16666666666666666, "PLAIN-1066": 0.0, "PLAIN-1088": 1.0, "PLAIN-1098": 0.0, "PLAIN-1109": 1.0, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 1.0, "PLAIN-1151": 1.0, "PLAIN-1161": 0.0, "PLAIN-1172": 1.0, "PLAIN-1183": 0.25, "PLAIN-1193": 0.09090909090909091, "PLAIN-1203": 1.0, "PLAIN-1214": 0.06666666666666667, "PLAIN-1225": 1.0, "PLAIN-1236": 1.0, "PLAIN-1249": 0.0, "PLAIN-1262": 1.0, "PLAIN-1275": 1.0, "PLAIN-1288": 1.0, "PLAIN-1299": 1.0, "PLAIN-1309": 0.0, "PLAIN-1320": 1.0, "PLAIN-1331": 0.0, "PLAIN-1342": 1.0, "PLAIN-1353": 0.0, "PLAIN-1363": 1.0, "PLAIN-1374": 0.16666666666666666, "PLAIN-1387": 1.0, "PLAIN-1398": 1.0, "PLAIN-1409": 0.5, "PLAIN-1419": 1.0, "PLAIN-1429": 0.0, "PLAIN-1441": 1.0, "PLAIN-1453": 0.5, "PLAIN-1463": 1.0, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 1.0, "PLAIN-1516": 0.0, "PLAIN-1527": 1.0, "PLAIN-1537": 0.5, "PLAIN-1547": 0.0, "PLAIN-1557": 0.5, "PLAIN-1568": 1.0, "PLAIN-1579": 1.0, "PLAIN-1590": 0.2, "PLAIN-1601": 1.0, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 1.0, "PLAIN-1645": 1.0, "PLAIN-1656": 0.06666666666666667, "PLAIN-1667": 1.0, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 1.0, "PLAIN-1731": 1.0, "PLAIN-1741": 1.0, "PLAIN-1752": 0.05, "PLAIN-1762": 1.0, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 1.0, "PLAIN-1837": 1.0, "PLAIN-1847": 1.0, "PLAIN-1857": 1.0, "PLAIN-1867": 1.0, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 1.0, "PLAIN-1919": 1.0, "PLAIN-1929": 0.3333333333333333, "PLAIN-1940": 1.0, "PLAIN-1950": 0.5, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 1.0, "PLAIN-1995": 0.05263157894736842, "PLAIN-2009": 0.3333333333333333, "PLAIN-2019": 1.0, "PLAIN-2030": 0.09090909090909091, "PLAIN-2040": 1.0, "PLAIN-2051": 1.0, "PLAIN-2061": 1.0, "PLAIN-2071": 1.0, "PLAIN-2081": 1.0, "PLAIN-2092": 1.0, "PLAIN-2102": 1.0, "PLAIN-2113": 0.0, "PLAIN-2124": 1.0, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 1.0, "PLAIN-2167": 0.0, "PLAIN-2177": 0.5, "PLAIN-2187": 1.0, "PLAIN-2197": 1.0, "PLAIN-2209": 0.0, "PLAIN-2220": 1.0, "PLAIN-2230": 0.0625, "PLAIN-2240": 1.0, "PLAIN-2250": 0.0, "PLAIN-2261": 1.0, "PLAIN-2271": 0.0, "PLAIN-2281": 0.5, "PLAIN-2291": 1.0, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 1.0, "PLAIN-2343": 0.0, "PLAIN-2354": 1.0, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.5, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 1.0, "PLAIN-2440": 0.14285714285714285, "PLAIN-2450": 1.0, "PLAIN-2460": 0.5, "PLAIN-2470": 1.0, "PLAIN-2480": 0.5, "PLAIN-2490": 1.0, "PLAIN-2500": 0.16666666666666666, "PLAIN-2510": 1.0, "PLAIN-2520": 0.5, "PLAIN-2530": 1.0, "PLAIN-2540": 1.0, "PLAIN-2550": 0.08333333333333333, "PLAIN-2560": 1.0, "PLAIN-2570": 1.0, "PLAIN-2580": 1.0, "PLAIN-2590": 0.3333333333333333, "PLAIN-2600": 1.0, "PLAIN-2610": 0.3333333333333333, "PLAIN-2620": 1.0, "PLAIN-2630": 1.0, "PLAIN-2640": 1.0, "PLAIN-2650": 1.0, "PLAIN-2660": 0.25, "PLAIN-2670": 1.0, "PLAIN-2680": 0.5, "PLAIN-2690": 1.0, "PLAIN-2700": 0.5, "PLAIN-2710": 1.0, "PLAIN-2720": 1.0, "PLAIN-2730": 1.0, "PLAIN-2740": 1.0, "PLAIN-2750": 1.0, "PLAIN-2760": 0.16666666666666666, "PLAIN-2770": 0.5, "PLAIN-2780": 1.0, "PLAIN-2790": 1.0, "PLAIN-2800": 0.25, "PLAIN-2810": 0.125, "PLAIN-2820": 0.2, "PLAIN-2830": 1.0, "PLAIN-2840": 0.0, "PLAIN-2850": 0.5, "PLAIN-2860": 1.0, "PLAIN-2870": 0.14285714285714285, "PLAIN-2880": 0.0, "PLAIN-2890": 0.25, "PLAIN-2900": 0.14285714285714285, "PLAIN-2910": 1.0, "PLAIN-2920": 0.0, "PLAIN-2930": 1.0, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.25, "PLAIN-2970": 1.0, "PLAIN-2981": 1.0, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 1.0, "PLAIN-3063": 0.07692307692307693, "PLAIN-3074": 1.0, "PLAIN-3085": 0.25, "PLAIN-3097": 1.0, "PLAIN-3116": 0.0, "PLAIN-3131": 1.0, "PLAIN-3141": 1.0, "PLAIN-3151": 1.0, "PLAIN-3161": 1.0, "PLAIN-3171": 0.2, "PLAIN-3181": 0.16666666666666666, "PLAIN-3191": 1.0, "PLAIN-3201": 0.5, "PLAIN-3211": 0.0, "PLAIN-3221": 0.08333333333333333, "PLAIN-3231": 0.07142857142857142, "PLAIN-3241": 1.0, "PLAIN-3251": 0.25, "PLAIN-3261": 0.3333333333333333, "PLAIN-3271": 0.0, "PLAIN-3281": 0.5, "PLAIN-3292": 0.3333333333333333, "PLAIN-3302": 0.3333333333333333, "PLAIN-3312": 0.5, "PLAIN-3322": 0.125, "PLAIN-3332": 1.0, "PLAIN-3342": 1.0, "PLAIN-3352": 0.0, "PLAIN-3362": 1.0, "PLAIN-3372": 0.25, "PLAIN-3382": 0.5, "PLAIN-3392": 0.07692307692307693, "PLAIN-3402": 0.07692307692307693, "PLAIN-3412": 1.0, "PLAIN-3422": 0.08333333333333333, "PLAIN-3432": 1.0, "PLAIN-3442": 0.3333333333333333, "PLAIN-3452": 1.0, "PLAIN-3462": 0.5, "PLAIN-3472": 0.1111111111111111}}} diff --git a/evals/benchmark/results-adaptive/nfcorpus/ir-adm0.50.jsonl b/evals/benchmark/results-adaptive/nfcorpus/ir-adm0.50.jsonl new file mode 100644 index 000000000..973bae8c6 --- /dev/null +++ b/evals/benchmark/results-adaptive/nfcorpus/ir-adm0.50.jsonl @@ -0,0 +1 @@ +{"dataset": "nfcorpus", "run_tag": "adm0.50", "aggregated": {"nDCG@10": 0.3745, "Recall@20": 0.2152, "MRR@10": 0.5887}, "per_query": {"nDCG@10": {"PLAIN-2": 0.7495470737433306, "PLAIN-12": 0.16950960215087577, "PLAIN-23": 0.46459600558095715, "PLAIN-33": 0.2640886107812416, "PLAIN-44": 0.23888170027858516, "PLAIN-56": 0.814153485068656, "PLAIN-68": 0.47459215156136747, "PLAIN-78": 0.1090956384204283, "PLAIN-91": 0.6947482816557203, "PLAIN-102": 0.21594229312581256, "PLAIN-112": 0.5903626645556106, "PLAIN-123": 0.08039138809008574, "PLAIN-133": 0.13347537859706934, "PLAIN-143": 0.6917350424665718, "PLAIN-153": 1.0, "PLAIN-165": 0.45365777747025504, "PLAIN-175": 0.42027556825478546, "PLAIN-186": 0.22009176629808017, "PLAIN-196": 0.22009176629808017, "PLAIN-207": 0.4230293752959408, "PLAIN-217": 0.13305201013572898, "PLAIN-227": 0.11004588314904008, "PLAIN-238": 0.0, "PLAIN-248": 0.38113435412202, "PLAIN-259": 0.3589542101716347, "PLAIN-270": 0.21222636597291458, "PLAIN-280": 0.4131279133289227, "PLAIN-291": 0.43231813227099475, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.6131471927654584, "PLAIN-344": 0.4602250224698888, "PLAIN-358": 0.0, "PLAIN-371": 0.23981246656813146, "PLAIN-383": 0.16958010263680806, "PLAIN-395": 0.0, "PLAIN-407": 0.6387878864795979, "PLAIN-418": 0.8670870086853021, "PLAIN-430": 0.6174886499090687, "PLAIN-441": 0.777747077360283, "PLAIN-457": 0.0, "PLAIN-468": 0.4318845862516251, "PLAIN-478": 0.0, "PLAIN-488": 0.9363792118010483, "PLAIN-499": 0.48864468918810133, "PLAIN-510": 0.0, "PLAIN-520": 0.0, "PLAIN-531": 0.5341769051617135, "PLAIN-541": 0.11706259313796354, "PLAIN-551": 0.7903864795495061, "PLAIN-561": 0.43735247915031, "PLAIN-571": 0.22471947398559278, "PLAIN-583": 0.22009176629808017, "PLAIN-593": 0.46927872602275644, "PLAIN-603": 0.0, "PLAIN-613": 0.08514311764162098, "PLAIN-623": 0.33013764944712026, "PLAIN-634": 0.41528076708874123, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.5214682725103094, "PLAIN-711": 0.7605591431973849, "PLAIN-721": 1.0, "PLAIN-731": 0.7058075148678526, "PLAIN-741": 0.866947989864271, "PLAIN-751": 0.0, "PLAIN-761": 0.15130120674940672, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.644824486427155, "PLAIN-806": 0.857980942822373, "PLAIN-817": 0.3562071871080222, "PLAIN-827": 0.7564477297898766, "PLAIN-838": 0.7767471075223497, "PLAIN-850": 0.6936634693435663, "PLAIN-872": 0.0, "PLAIN-882": 0.4690000933206748, "PLAIN-892": 0.5326208815196265, "PLAIN-902": 0.6366824387328317, "PLAIN-913": 0.8643145546088337, "PLAIN-924": 0.30523488393970116, "PLAIN-934": 0.7257345305446319, "PLAIN-946": 0.28371255449703187, "PLAIN-956": 0.6468562235661909, "PLAIN-966": 0.0, "PLAIN-977": 0.46927872602275644, "PLAIN-987": 0.0, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5063209639325833, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.07839826897867533, "PLAIN-1066": 0.0, "PLAIN-1088": 0.5531464700081437, "PLAIN-1098": 0.0, "PLAIN-1109": 0.6678262889110271, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.22009176629808017, "PLAIN-1151": 0.7000949683025056, "PLAIN-1161": 0.0, "PLAIN-1172": 0.22009176629808017, "PLAIN-1183": 0.1731866333482261, "PLAIN-1193": 0.0, "PLAIN-1203": 0.5036067026046991, "PLAIN-1214": 0.0, "PLAIN-1225": 0.6874708847784536, "PLAIN-1236": 0.3391602052736161, "PLAIN-1249": 0.0, "PLAIN-1262": 0.4537425745411855, "PLAIN-1275": 0.6274092458891772, "PLAIN-1288": 0.49828993591603105, "PLAIN-1299": 0.3589542101716347, "PLAIN-1309": 0.0, "PLAIN-1320": 0.6238470455881188, "PLAIN-1331": 0.0, "PLAIN-1342": 0.5861163849908828, "PLAIN-1353": 0.0, "PLAIN-1363": 0.22009176629808017, "PLAIN-1374": 0.07839826897867533, "PLAIN-1387": 0.7819835854675755, "PLAIN-1398": 1.0, "PLAIN-1409": 0.4170606134915054, "PLAIN-1419": 0.4690000933206748, "PLAIN-1429": 0.0, "PLAIN-1441": 0.627507133061656, "PLAIN-1453": 0.48068140169045714, "PLAIN-1463": 0.46270962158308354, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.33013764944712026, "PLAIN-1516": 0.0, "PLAIN-1527": 0.8482378089219645, "PLAIN-1537": 0.42331179510529354, "PLAIN-1547": 0.0, "PLAIN-1557": 0.2489083270225946, "PLAIN-1568": 1.0, "PLAIN-1579": 0.3391602052736161, "PLAIN-1590": 0.09478836436955078, "PLAIN-1601": 0.800693766409882, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.8643145546088337, "PLAIN-1645": 0.30260241349881345, "PLAIN-1656": 0.0, "PLAIN-1667": 0.8572048559638628, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 0.2984900352767555, "PLAIN-1731": 0.7653606369886217, "PLAIN-1741": 1.0, "PLAIN-1752": 0.0, "PLAIN-1762": 0.4856486183364634, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 0.568003577662161, "PLAIN-1837": 1.0, "PLAIN-1847": 0.22009176629808017, "PLAIN-1857": 0.627507133061656, "PLAIN-1867": 0.27487633291429087, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.7917267193679839, "PLAIN-1919": 0.866947989864271, "PLAIN-1929": 0.11004588314904008, "PLAIN-1940": 0.3589542101716347, "PLAIN-1950": 0.3222722491219547, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.9363792118010483, "PLAIN-1995": 0.0, "PLAIN-2009": 0.31442295467135667, "PLAIN-2019": 0.6364391809889587, "PLAIN-2030": 0.09546043308946733, "PLAIN-2040": 0.5637884576902256, "PLAIN-2051": 0.6870165078530993, "PLAIN-2061": 0.933745776545611, "PLAIN-2071": 0.5797827732044221, "PLAIN-2081": 1.0, "PLAIN-2092": 0.6489315753318465, "PLAIN-2102": 0.835780413693672, "PLAIN-2113": 0.0, "PLAIN-2124": 0.6906478832608419, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.644824486427155, "PLAIN-2167": 0.0, "PLAIN-2177": 0.45166722277552973, "PLAIN-2187": 0.3187704634136788, "PLAIN-2197": 0.7967610662472993, "PLAIN-2209": 0.0, "PLAIN-2220": 0.5526193270751085, "PLAIN-2230": 0.0, "PLAIN-2240": 0.5637884576902256, "PLAIN-2250": 0.0, "PLAIN-2261": 0.715550648768261, "PLAIN-2271": 0.0, "PLAIN-2281": 0.6309297535714575, "PLAIN-2291": 0.3391602052736161, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.44988737637276544, "PLAIN-2343": 0.0, "PLAIN-2354": 0.5531464700081437, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.31204907722178066, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.4832964623025934, "PLAIN-2440": 0.07336392209936005, "PLAIN-2450": 0.28952298823485745, "PLAIN-2460": 0.2873703978647362, "PLAIN-2470": 0.6135425409603973, "PLAIN-2480": 0.214167553296389, "PLAIN-2490": 0.4592049927394573, "PLAIN-2500": 0.06760523069231983, "PLAIN-2510": 0.7093753650484099, "PLAIN-2520": 0.2085303067457527, "PLAIN-2530": 0.9633180389503199, "PLAIN-2540": 0.5907275541269412, "PLAIN-2550": 0.0, "PLAIN-2560": 0.800693766409882, "PLAIN-2570": 0.4205032672422578, "PLAIN-2580": 0.3941769500918012, "PLAIN-2590": 0.1581173338315241, "PLAIN-2600": 0.6604400652351458, "PLAIN-2610": 0.3052695562492681, "PLAIN-2620": 0.5549841905774386, "PLAIN-2630": 0.7222954974312066, "PLAIN-2640": 0.5567466860572858, "PLAIN-2650": 0.5384313152574521, "PLAIN-2660": 0.31954962756492084, "PLAIN-2670": 0.37456050428247883, "PLAIN-2680": 0.43478330062838466, "PLAIN-2690": 0.5760823635418191, "PLAIN-2700": 0.28893989447363055, "PLAIN-2710": 0.6665181205254729, "PLAIN-2720": 0.2489083270225946, "PLAIN-2730": 0.5469722089928789, "PLAIN-2740": 0.6854074373766073, "PLAIN-2750": 0.5946665900227162, "PLAIN-2760": 0.2469526638494753, "PLAIN-2770": 0.415769594467977, "PLAIN-2780": 0.4769783244417837, "PLAIN-2790": 0.8251154201194959, "PLAIN-2800": 0.15472892909388752, "PLAIN-2810": 0.05184819766823351, "PLAIN-2820": 0.15619484341784115, "PLAIN-2830": 0.4030302838010049, "PLAIN-2840": 0.0, "PLAIN-2850": 0.24014860122439108, "PLAIN-2860": 1.0, "PLAIN-2870": 0.08442111803551768, "PLAIN-2880": 0.0, "PLAIN-2890": 0.15537907391531905, "PLAIN-2900": 0.09382844813642036, "PLAIN-2910": 0.3239134396887189, "PLAIN-2920": 0.0, "PLAIN-2930": 0.3607790370815594, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.3273949503887395, "PLAIN-2970": 0.38062652754129855, "PLAIN-2981": 0.43122038713502525, "PLAIN-2991": 0.38685280723454163, "PLAIN-3001": 0.38685280723454163, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.7653606369886217, "PLAIN-3063": 0.0, "PLAIN-3074": 1.0, "PLAIN-3085": 0.5012658353418871, "PLAIN-3097": 0.4770382338730848, "PLAIN-3116": 0.0, "PLAIN-3131": 0.4599311598225878, "PLAIN-3141": 0.2996484034259831, "PLAIN-3151": 0.707059297977083, "PLAIN-3161": 0.4906128388857768, "PLAIN-3171": 0.06978419164318803, "PLAIN-3181": 0.07336392209936005, "PLAIN-3191": 0.35185668051157637, "PLAIN-3201": 0.37212364421416444, "PLAIN-3211": 0.0, "PLAIN-3221": 0.0, "PLAIN-3231": 0.0, "PLAIN-3241": 0.43056322872474745, "PLAIN-3251": 0.2155089134289123, "PLAIN-3261": 0.14982420171299154, "PLAIN-3271": 0.0, "PLAIN-3281": 0.20436662668128655, "PLAIN-3292": 0.4689564279260148, "PLAIN-3302": 0.18359730612299605, "PLAIN-3312": 0.3170759870637039, "PLAIN-3322": 0.06362078819895171, "PLAIN-3332": 0.8909491779135381, "PLAIN-3342": 0.7653606369886217, "PLAIN-3352": 0.0, "PLAIN-3362": 0.4634153650075699, "PLAIN-3372": 0.15537907391531905, "PLAIN-3382": 0.4596741192162093, "PLAIN-3392": 0.0, "PLAIN-3402": 0.0, "PLAIN-3412": 0.23170768250378496, "PLAIN-3422": 0.0, "PLAIN-3432": 0.2934556883974402, "PLAIN-3442": 0.07037133610231527, "PLAIN-3452": 0.5288361569833154, "PLAIN-3462": 0.6067216003536936, "PLAIN-3472": 0.08036018528064806}, "Recall@20": {"PLAIN-2": 0.4166666666666667, "PLAIN-12": 0.13333333333333333, "PLAIN-23": 0.08888888888888889, "PLAIN-33": 0.125, "PLAIN-44": 0.06896551724137931, "PLAIN-56": 0.2, "PLAIN-68": 0.07272727272727272, "PLAIN-78": 0.03225806451612903, "PLAIN-91": 0.15625, "PLAIN-102": 0.08333333333333333, "PLAIN-112": 0.08928571428571429, "PLAIN-123": 0.0196078431372549, "PLAIN-133": 0.027777777777777776, "PLAIN-143": 0.17647058823529413, "PLAIN-153": 0.23404255319148937, "PLAIN-165": 0.2222222222222222, "PLAIN-175": 0.10810810810810811, "PLAIN-186": 0.041666666666666664, "PLAIN-196": 0.014492753623188406, "PLAIN-207": 0.16363636363636364, "PLAIN-217": 0.1388888888888889, "PLAIN-227": 0.025, "PLAIN-238": 0.0, "PLAIN-248": 0.1935483870967742, "PLAIN-259": 0.21428571428571427, "PLAIN-270": 0.057692307692307696, "PLAIN-280": 0.25, "PLAIN-291": 0.3076923076923077, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.5, "PLAIN-344": 0.11475409836065574, "PLAIN-358": 0.0, "PLAIN-371": 0.5, "PLAIN-383": 0.4, "PLAIN-395": 0.0, "PLAIN-407": 0.3333333333333333, "PLAIN-418": 0.6666666666666666, "PLAIN-430": 0.375, "PLAIN-441": 0.8, "PLAIN-457": 0.0, "PLAIN-468": 0.2, "PLAIN-478": 0.0, "PLAIN-488": 0.7333333333333333, "PLAIN-499": 0.11764705882352941, "PLAIN-510": 0.16666666666666666, "PLAIN-520": 0.0, "PLAIN-531": 0.03571428571428571, "PLAIN-541": 0.16666666666666666, "PLAIN-551": 1.0, "PLAIN-561": 0.12, "PLAIN-571": 0.42857142857142855, "PLAIN-583": 0.022727272727272728, "PLAIN-593": 0.3333333333333333, "PLAIN-603": 0.0, "PLAIN-613": 0.1, "PLAIN-623": 0.12195121951219512, "PLAIN-634": 0.14285714285714285, "PLAIN-645": 0.0, "PLAIN-660": 0.03368421052631579, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.375, "PLAIN-711": 0.19298245614035087, "PLAIN-721": 0.68, "PLAIN-731": 0.2037037037037037, "PLAIN-741": 0.47368421052631576, "PLAIN-751": 0.0, "PLAIN-761": 0.16666666666666666, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.5, "PLAIN-806": 0.13541666666666666, "PLAIN-817": 1.0, "PLAIN-827": 0.04918032786885246, "PLAIN-838": 0.6666666666666666, "PLAIN-850": 0.17543859649122806, "PLAIN-872": 0.0, "PLAIN-882": 0.2, "PLAIN-892": 0.06521739130434782, "PLAIN-902": 0.5, "PLAIN-913": 0.23529411764705882, "PLAIN-924": 0.0625, "PLAIN-934": 0.10891089108910891, "PLAIN-946": 0.0967741935483871, "PLAIN-956": 0.05825242718446602, "PLAIN-966": 0.0, "PLAIN-977": 0.3333333333333333, "PLAIN-987": 0.6666666666666666, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.1, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.010638297872340425, "PLAIN-1066": 0.0, "PLAIN-1088": 0.4, "PLAIN-1098": 0.0, "PLAIN-1109": 0.11224489795918367, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.0625, "PLAIN-1151": 0.058823529411764705, "PLAIN-1161": 0.0, "PLAIN-1172": 0.05263157894736842, "PLAIN-1183": 0.08, "PLAIN-1193": 0.05555555555555555, "PLAIN-1203": 0.3333333333333333, "PLAIN-1214": 0.058823529411764705, "PLAIN-1225": 0.3333333333333333, "PLAIN-1236": 0.2, "PLAIN-1249": 0.0, "PLAIN-1262": 0.3, "PLAIN-1275": 0.16363636363636364, "PLAIN-1288": 0.03571428571428571, "PLAIN-1299": 0.03278688524590164, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5, "PLAIN-1331": 0.0, "PLAIN-1342": 0.3125, "PLAIN-1353": 0.0, "PLAIN-1363": 0.09090909090909091, "PLAIN-1374": 0.07692307692307693, "PLAIN-1387": 0.5833333333333334, "PLAIN-1398": 0.1650485436893204, "PLAIN-1409": 0.023715415019762844, "PLAIN-1419": 0.06153846153846154, "PLAIN-1429": 0.0, "PLAIN-1441": 0.04838709677419355, "PLAIN-1453": 0.0326530612244898, "PLAIN-1463": 0.20833333333333334, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.13333333333333333, "PLAIN-1516": 0.0, "PLAIN-1527": 0.06930693069306931, "PLAIN-1537": 0.13333333333333333, "PLAIN-1547": 0.0, "PLAIN-1557": 0.07692307692307693, "PLAIN-1568": 1.0, "PLAIN-1579": 0.2, "PLAIN-1590": 0.05555555555555555, "PLAIN-1601": 0.0945945945945946, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.030162412993039442, "PLAIN-1645": 0.16666666666666666, "PLAIN-1656": 0.03571428571428571, "PLAIN-1667": 0.06918238993710692, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 0.9375, "PLAIN-1721": 0.16129032258064516, "PLAIN-1731": 0.6666666666666666, "PLAIN-1741": 0.04523809523809524, "PLAIN-1752": 1.0, "PLAIN-1762": 0.5, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 0.13793103448275862, "PLAIN-1817": 0.2222222222222222, "PLAIN-1837": 0.08673469387755102, "PLAIN-1847": 0.18181818181818182, "PLAIN-1857": 0.24242424242424243, "PLAIN-1867": 0.14285714285714285, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.02591792656587473, "PLAIN-1919": 0.3333333333333333, "PLAIN-1929": 0.2, "PLAIN-1940": 0.16666666666666666, "PLAIN-1950": 0.23076923076923078, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.28125, "PLAIN-1995": 0.0, "PLAIN-2009": 0.375, "PLAIN-2019": 0.6666666666666666, "PLAIN-2030": 0.16666666666666666, "PLAIN-2040": 0.05303030303030303, "PLAIN-2051": 0.028169014084507043, "PLAIN-2061": 0.04390243902439024, "PLAIN-2071": 0.15555555555555556, "PLAIN-2081": 1.0, "PLAIN-2092": 0.5, "PLAIN-2102": 0.030878859857482184, "PLAIN-2113": 0.0, "PLAIN-2124": 0.35294117647058826, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.5, "PLAIN-2167": 0.0, "PLAIN-2177": 0.20833333333333334, "PLAIN-2187": 0.2222222222222222, "PLAIN-2197": 0.1724137931034483, "PLAIN-2209": 0.0, "PLAIN-2220": 0.4444444444444444, "PLAIN-2230": 0.14285714285714285, "PLAIN-2240": 0.36363636363636365, "PLAIN-2250": 0.0, "PLAIN-2261": 0.11666666666666667, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.4, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.09278350515463918, "PLAIN-2343": 0.0, "PLAIN-2354": 0.4, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.3157894736842105, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.13043478260869565, "PLAIN-2440": 0.047619047619047616, "PLAIN-2450": 0.07894736842105263, "PLAIN-2460": 0.0967741935483871, "PLAIN-2470": 0.136986301369863, "PLAIN-2480": 0.16666666666666666, "PLAIN-2490": 0.21739130434782608, "PLAIN-2500": 0.42857142857142855, "PLAIN-2510": 0.21739130434782608, "PLAIN-2520": 0.0821917808219178, "PLAIN-2530": 0.20833333333333334, "PLAIN-2540": 0.38095238095238093, "PLAIN-2550": 0.04, "PLAIN-2560": 0.47619047619047616, "PLAIN-2570": 0.075, "PLAIN-2580": 0.08695652173913043, "PLAIN-2590": 0.07936507936507936, "PLAIN-2600": 0.4, "PLAIN-2610": 0.09836065573770492, "PLAIN-2620": 0.2545454545454545, "PLAIN-2630": 0.1875, "PLAIN-2640": 0.25, "PLAIN-2650": 0.2413793103448276, "PLAIN-2660": 0.18518518518518517, "PLAIN-2670": 0.05714285714285714, "PLAIN-2680": 0.35714285714285715, "PLAIN-2690": 0.20930232558139536, "PLAIN-2700": 0.2, "PLAIN-2710": 0.23809523809523808, "PLAIN-2720": 0.05555555555555555, "PLAIN-2730": 0.20689655172413793, "PLAIN-2740": 0.12, "PLAIN-2750": 0.15254237288135594, "PLAIN-2760": 0.25, "PLAIN-2770": 0.21951219512195122, "PLAIN-2780": 0.3181818181818182, "PLAIN-2790": 0.15254237288135594, "PLAIN-2800": 0.08333333333333333, "PLAIN-2810": 0.125, "PLAIN-2820": 0.125, "PLAIN-2830": 0.3333333333333333, "PLAIN-2840": 0.0, "PLAIN-2850": 0.1111111111111111, "PLAIN-2860": 1.0, "PLAIN-2870": 0.2, "PLAIN-2880": 0.0, "PLAIN-2890": 0.034482758620689655, "PLAIN-2900": 0.08333333333333333, "PLAIN-2910": 0.06666666666666667, "PLAIN-2920": 0.0, "PLAIN-2930": 0.14285714285714285, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.5, "PLAIN-2970": 0.4444444444444444, "PLAIN-2981": 0.14285714285714285, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.6666666666666666, "PLAIN-3063": 0.25, "PLAIN-3074": 1.0, "PLAIN-3085": 1.0, "PLAIN-3097": 0.5, "PLAIN-3116": 0.0, "PLAIN-3131": 0.3333333333333333, "PLAIN-3141": 0.029411764705882353, "PLAIN-3151": 0.5, "PLAIN-3161": 0.13953488372093023, "PLAIN-3171": 0.08823529411764706, "PLAIN-3181": 0.037037037037037035, "PLAIN-3191": 0.1875, "PLAIN-3201": 0.09375, "PLAIN-3211": 0.0, "PLAIN-3221": 0.125, "PLAIN-3231": 0.25, "PLAIN-3241": 0.06896551724137931, "PLAIN-3251": 0.09523809523809523, "PLAIN-3261": 0.07692307692307693, "PLAIN-3271": 0.0, "PLAIN-3281": 0.038461538461538464, "PLAIN-3292": 0.7142857142857143, "PLAIN-3302": 0.08108108108108109, "PLAIN-3312": 0.5384615384615384, "PLAIN-3322": 0.012658227848101266, "PLAIN-3332": 0.6428571428571429, "PLAIN-3342": 0.6666666666666666, "PLAIN-3352": 0.0, "PLAIN-3362": 0.18181818181818182, "PLAIN-3372": 0.07142857142857142, "PLAIN-3382": 0.5714285714285714, "PLAIN-3392": 0.043478260869565216, "PLAIN-3402": 0.07142857142857142, "PLAIN-3412": 0.061224489795918366, "PLAIN-3422": 0.0967741935483871, "PLAIN-3432": 0.09523809523809523, "PLAIN-3442": 0.034482758620689655, "PLAIN-3452": 0.29411764705882354, "PLAIN-3462": 0.1875, "PLAIN-3472": 0.058823529411764705}, "MRR@10": {"PLAIN-2": 1.0, "PLAIN-12": 0.3333333333333333, "PLAIN-23": 0.5, "PLAIN-33": 0.5, "PLAIN-44": 0.25, "PLAIN-56": 1.0, "PLAIN-68": 1.0, "PLAIN-78": 0.3333333333333333, "PLAIN-91": 1.0, "PLAIN-102": 0.3333333333333333, "PLAIN-112": 1.0, "PLAIN-123": 0.5, "PLAIN-133": 0.3333333333333333, "PLAIN-143": 1.0, "PLAIN-153": 1.0, "PLAIN-165": 0.5, "PLAIN-175": 1.0, "PLAIN-186": 1.0, "PLAIN-196": 1.0, "PLAIN-207": 0.3333333333333333, "PLAIN-217": 0.125, "PLAIN-227": 0.3333333333333333, "PLAIN-238": 0.0, "PLAIN-248": 1.0, "PLAIN-259": 1.0, "PLAIN-270": 0.5, "PLAIN-280": 0.5, "PLAIN-291": 1.0, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 1.0, "PLAIN-344": 0.5, "PLAIN-358": 0.0, "PLAIN-371": 0.5, "PLAIN-383": 0.3333333333333333, "PLAIN-395": 0.0, "PLAIN-407": 1.0, "PLAIN-418": 1.0, "PLAIN-430": 1.0, "PLAIN-441": 1.0, "PLAIN-457": 0.0, "PLAIN-468": 1.0, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 1.0, "PLAIN-510": 0.07142857142857142, "PLAIN-520": 0.0, "PLAIN-531": 0.5, "PLAIN-541": 0.2, "PLAIN-551": 1.0, "PLAIN-561": 1.0, "PLAIN-571": 0.25, "PLAIN-583": 1.0, "PLAIN-593": 1.0, "PLAIN-603": 0.0, "PLAIN-613": 0.2, "PLAIN-623": 1.0, "PLAIN-634": 1.0, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 1.0, "PLAIN-711": 1.0, "PLAIN-721": 1.0, "PLAIN-731": 1.0, "PLAIN-741": 1.0, "PLAIN-751": 0.0, "PLAIN-761": 0.3333333333333333, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 1.0, "PLAIN-806": 1.0, "PLAIN-817": 0.16666666666666666, "PLAIN-827": 1.0, "PLAIN-838": 1.0, "PLAIN-850": 1.0, "PLAIN-872": 0.0, "PLAIN-882": 1.0, "PLAIN-892": 1.0, "PLAIN-902": 1.0, "PLAIN-913": 1.0, "PLAIN-924": 1.0, "PLAIN-934": 1.0, "PLAIN-946": 1.0, "PLAIN-956": 0.5, "PLAIN-966": 0.0, "PLAIN-977": 1.0, "PLAIN-987": 0.07692307692307693, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.16666666666666666, "PLAIN-1066": 0.0, "PLAIN-1088": 1.0, "PLAIN-1098": 0.0, "PLAIN-1109": 1.0, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 1.0, "PLAIN-1151": 1.0, "PLAIN-1161": 0.0, "PLAIN-1172": 1.0, "PLAIN-1183": 0.25, "PLAIN-1193": 0.09090909090909091, "PLAIN-1203": 1.0, "PLAIN-1214": 0.06666666666666667, "PLAIN-1225": 1.0, "PLAIN-1236": 1.0, "PLAIN-1249": 0.0, "PLAIN-1262": 1.0, "PLAIN-1275": 1.0, "PLAIN-1288": 1.0, "PLAIN-1299": 1.0, "PLAIN-1309": 0.0, "PLAIN-1320": 1.0, "PLAIN-1331": 0.0, "PLAIN-1342": 1.0, "PLAIN-1353": 0.0, "PLAIN-1363": 1.0, "PLAIN-1374": 0.16666666666666666, "PLAIN-1387": 1.0, "PLAIN-1398": 1.0, "PLAIN-1409": 0.5, "PLAIN-1419": 1.0, "PLAIN-1429": 0.0, "PLAIN-1441": 1.0, "PLAIN-1453": 0.5, "PLAIN-1463": 1.0, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 1.0, "PLAIN-1516": 0.0, "PLAIN-1527": 1.0, "PLAIN-1537": 0.5, "PLAIN-1547": 0.0, "PLAIN-1557": 0.5, "PLAIN-1568": 1.0, "PLAIN-1579": 1.0, "PLAIN-1590": 0.25, "PLAIN-1601": 1.0, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 1.0, "PLAIN-1645": 1.0, "PLAIN-1656": 0.06666666666666667, "PLAIN-1667": 1.0, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 1.0, "PLAIN-1731": 1.0, "PLAIN-1741": 1.0, "PLAIN-1752": 0.05, "PLAIN-1762": 1.0, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 1.0, "PLAIN-1837": 1.0, "PLAIN-1847": 1.0, "PLAIN-1857": 1.0, "PLAIN-1867": 1.0, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 1.0, "PLAIN-1919": 1.0, "PLAIN-1929": 0.3333333333333333, "PLAIN-1940": 1.0, "PLAIN-1950": 0.5, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 1.0, "PLAIN-1995": 0.0, "PLAIN-2009": 0.3333333333333333, "PLAIN-2019": 1.0, "PLAIN-2030": 0.125, "PLAIN-2040": 1.0, "PLAIN-2051": 1.0, "PLAIN-2061": 1.0, "PLAIN-2071": 1.0, "PLAIN-2081": 1.0, "PLAIN-2092": 1.0, "PLAIN-2102": 1.0, "PLAIN-2113": 0.0, "PLAIN-2124": 1.0, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 1.0, "PLAIN-2167": 0.0, "PLAIN-2177": 0.5, "PLAIN-2187": 1.0, "PLAIN-2197": 1.0, "PLAIN-2209": 0.0, "PLAIN-2220": 1.0, "PLAIN-2230": 0.07692307692307693, "PLAIN-2240": 1.0, "PLAIN-2250": 0.0, "PLAIN-2261": 1.0, "PLAIN-2271": 0.0, "PLAIN-2281": 0.5, "PLAIN-2291": 1.0, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 1.0, "PLAIN-2343": 0.0, "PLAIN-2354": 1.0, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.5, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 1.0, "PLAIN-2440": 0.14285714285714285, "PLAIN-2450": 1.0, "PLAIN-2460": 0.5, "PLAIN-2470": 1.0, "PLAIN-2480": 0.5, "PLAIN-2490": 1.0, "PLAIN-2500": 0.16666666666666666, "PLAIN-2510": 1.0, "PLAIN-2520": 0.5, "PLAIN-2530": 1.0, "PLAIN-2540": 1.0, "PLAIN-2550": 0.07692307692307693, "PLAIN-2560": 1.0, "PLAIN-2570": 1.0, "PLAIN-2580": 1.0, "PLAIN-2590": 0.3333333333333333, "PLAIN-2600": 1.0, "PLAIN-2610": 0.3333333333333333, "PLAIN-2620": 1.0, "PLAIN-2630": 1.0, "PLAIN-2640": 1.0, "PLAIN-2650": 1.0, "PLAIN-2660": 0.25, "PLAIN-2670": 1.0, "PLAIN-2680": 0.5, "PLAIN-2690": 1.0, "PLAIN-2700": 0.5, "PLAIN-2710": 1.0, "PLAIN-2720": 1.0, "PLAIN-2730": 0.5, "PLAIN-2740": 1.0, "PLAIN-2750": 1.0, "PLAIN-2760": 0.16666666666666666, "PLAIN-2770": 0.5, "PLAIN-2780": 1.0, "PLAIN-2790": 1.0, "PLAIN-2800": 0.25, "PLAIN-2810": 0.125, "PLAIN-2820": 0.2, "PLAIN-2830": 0.5, "PLAIN-2840": 0.0, "PLAIN-2850": 0.5, "PLAIN-2860": 1.0, "PLAIN-2870": 0.14285714285714285, "PLAIN-2880": 0.0, "PLAIN-2890": 0.25, "PLAIN-2900": 0.14285714285714285, "PLAIN-2910": 1.0, "PLAIN-2920": 0.0, "PLAIN-2930": 1.0, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.25, "PLAIN-2970": 1.0, "PLAIN-2981": 1.0, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 1.0, "PLAIN-3063": 0.07692307692307693, "PLAIN-3074": 1.0, "PLAIN-3085": 0.25, "PLAIN-3097": 1.0, "PLAIN-3116": 0.0, "PLAIN-3131": 1.0, "PLAIN-3141": 1.0, "PLAIN-3151": 1.0, "PLAIN-3161": 1.0, "PLAIN-3171": 0.2, "PLAIN-3181": 0.14285714285714285, "PLAIN-3191": 1.0, "PLAIN-3201": 0.5, "PLAIN-3211": 0.0, "PLAIN-3221": 0.08333333333333333, "PLAIN-3231": 0.07142857142857142, "PLAIN-3241": 1.0, "PLAIN-3251": 0.25, "PLAIN-3261": 0.3333333333333333, "PLAIN-3271": 0.0, "PLAIN-3281": 0.5, "PLAIN-3292": 0.3333333333333333, "PLAIN-3302": 0.5, "PLAIN-3312": 0.5, "PLAIN-3322": 0.1, "PLAIN-3332": 1.0, "PLAIN-3342": 1.0, "PLAIN-3352": 0.0, "PLAIN-3362": 1.0, "PLAIN-3372": 0.25, "PLAIN-3382": 0.5, "PLAIN-3392": 0.06666666666666667, "PLAIN-3402": 0.07692307692307693, "PLAIN-3412": 1.0, "PLAIN-3422": 0.08333333333333333, "PLAIN-3432": 1.0, "PLAIN-3442": 0.3333333333333333, "PLAIN-3452": 1.0, "PLAIN-3462": 0.5, "PLAIN-3472": 0.1111111111111111}}} diff --git a/evals/benchmark/results-adaptive/nfcorpus/run-adm0.15.trec.gz b/evals/benchmark/results-adaptive/nfcorpus/run-adm0.15.trec.gz new file mode 100644 index 000000000..cedb36271 Binary files /dev/null and b/evals/benchmark/results-adaptive/nfcorpus/run-adm0.15.trec.gz differ diff --git a/evals/benchmark/results-adaptive/nfcorpus/stats-adm0.15-vs-dense.jsonl b/evals/benchmark/results-adaptive/nfcorpus/stats-adm0.15-vs-dense.jsonl new file mode 100644 index 000000000..5be72eabd --- /dev/null +++ b/evals/benchmark/results-adaptive/nfcorpus/stats-adm0.15-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "adm0.15", "arm_b": "dense"} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "nDCG@10", "n": 322, "mean_a": 0.37507950372235543, "mean_b": 0.36660768485593975, "mean_diff": -0.008471818866415706, "ci_low": -0.016166010439983513, "ci_high": -0.0007040087395521865, "p_value": 0.0332966703329667, "significant": true} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "Recall@20", "n": 322, "mean_a": 0.214096312157869, "mean_b": 0.2135261475395606, "mean_diff": -0.0005701646183084199, "ci_low": -0.0014853947990737068, "ci_high": 2.535175560907591e-05, "p_value": 0.24677532246775322, "significant": false} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "MRR@10", "n": 322, "mean_a": 0.603003083010847, "mean_b": 0.5918453065831581, "mean_diff": -0.011157776427688742, "ci_low": -0.037030675849695306, "ci_high": 0.013630450165776251, "p_value": 0.3885611438856114, "significant": false} diff --git a/evals/benchmark/results-adaptive/nfcorpus/stats-adm0.30-vs-dense.jsonl b/evals/benchmark/results-adaptive/nfcorpus/stats-adm0.30-vs-dense.jsonl new file mode 100644 index 000000000..3435afab5 --- /dev/null +++ b/evals/benchmark/results-adaptive/nfcorpus/stats-adm0.30-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "adm0.30", "arm_b": "dense"} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "nDCG@10", "n": 322, "mean_a": 0.3753092086467289, "mean_b": 0.36660768485593975, "mean_diff": -0.008701523790789157, "ci_low": -0.01705590052967067, "ci_high": -0.00021396123575186536, "p_value": 0.042695730426957304, "significant": true} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "Recall@20", "n": 322, "mean_a": 0.2152929084941406, "mean_b": 0.2135261475395606, "mean_diff": -0.0017667609545799782, "ci_low": -0.003770827067363018, "ci_high": -3.629642821803704e-05, "p_value": 0.06309369063093691, "significant": true} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "MRR@10", "n": 322, "mean_a": 0.5917215618731637, "mean_b": 0.5918453065831581, "mean_diff": 0.0001237447099945421, "ci_low": -0.027058454036428866, "ci_high": 0.02664162761986745, "p_value": 0.9926007399260074, "significant": false} diff --git a/evals/benchmark/results-adaptive/nfcorpus/stats-adm0.50-vs-dense.jsonl b/evals/benchmark/results-adaptive/nfcorpus/stats-adm0.50-vs-dense.jsonl new file mode 100644 index 000000000..737336bb7 --- /dev/null +++ b/evals/benchmark/results-adaptive/nfcorpus/stats-adm0.50-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "adm0.50", "arm_b": "dense"} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "nDCG@10", "n": 322, "mean_a": 0.3745430024862999, "mean_b": 0.36660768485593975, "mean_diff": -0.007935317630360133, "ci_low": -0.016730637847947268, "ci_high": 0.0008698032157170701, "p_value": 0.08299170082991701, "significant": false} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "Recall@20", "n": 322, "mean_a": 0.21519576307636726, "mean_b": 0.2135261475395606, "mean_diff": -0.0016696155368066692, "ci_low": -0.0053732193298111994, "ci_high": 0.00199579038719068, "p_value": 0.38906109389061094, "significant": false} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "MRR@10", "n": 322, "mean_a": 0.5887450972078301, "mean_b": 0.5918453065831581, "mean_diff": 0.003100209375328119, "ci_low": -0.024121624934558803, "ci_high": 0.029901906947029343, "p_value": 0.8174182581741826, "significant": false} diff --git a/evals/benchmark/results-adaptive/scifact/ir-adm0.15.jsonl b/evals/benchmark/results-adaptive/scifact/ir-adm0.15.jsonl new file mode 100644 index 000000000..d831cfe12 --- /dev/null +++ b/evals/benchmark/results-adaptive/scifact/ir-adm0.15.jsonl @@ -0,0 +1 @@ +{"dataset": "scifact", "run_tag": "adm0.15", "aggregated": {"nDCG@10": 0.7186, "Recall@20": 0.8667, "MRR@10": 0.6867}, "per_query": {"nDCG@10": {"1": 0.0, "3": 0.5, "5": 1.0, "13": 0.0, "36": 0.2640681225725909, "42": 1.0, "48": 0.5, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.23719771276929622, "72": 0.6309297535714575, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 0.8850504602968671, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 0.6309297535714575, "163": 1.0, "171": 1.0, "179": 0.9267582364714126, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.5, "213": 0.3333333333333333, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.6309297535714575, "233": 1.0, "236": 1.0, "237": 0.6309297535714575, "238": 0.5, "239": 0.43067655807339306, "248": 1.0, "249": 1.0, "261": 0.5912352048230277, "268": 1.0, "269": 1.0, "274": 1.0, "275": 0.7039180890341347, "279": 1.0, "294": 0.0, "295": 0.6309297535714575, "298": 1.0, "300": 0.0, "303": 0.0, "312": 0.0, "314": 0.3562071871080222, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.6509209298071326, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.3562071871080222, "385": 0.6934264036172708, "386": 1.0, "388": 0.6309297535714575, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.6309297535714575, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3065735963827292, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.6309297535714575, "508": 0.3010299956639812, "513": 0.6309297535714575, "514": 1.0, "516": 1.0, "517": 0.3333333333333333, "521": 1.0, "525": 1.0, "527": 0.6309297535714575, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.3333333333333333, "536": 1.0, "539": 1.0, "540": 0.38685280723454163, "544": 0.38685280723454163, "549": 1.0, "551": 0.38685280723454163, "552": 1.0, "554": 0.3010299956639812, "560": 0.0, "569": 0.38685280723454163, "575": 1.0, "577": 0.6309297535714575, "578": 1.0, "587": 0.6309297535714575, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.5, "613": 1.0, "619": 0.0, "623": 0.6309297535714575, "628": 1.0, "636": 1.0, "637": 0.6309297535714575, "641": 0.8315546295836225, "644": 1.0, "649": 0.6309297535714575, "659": 0.0, "660": 0.0, "674": 0.6309297535714575, "684": 0.6309297535714575, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.38685280723454163, "728": 0.8772153153380493, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.5, "756": 0.6309297535714575, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.3333333333333333, "784": 1.0, "785": 1.0, "793": 0.43067655807339306, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.43067655807339306, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.5, "847": 1.0, "852": 0.6309297535714575, "859": 0.6309297535714575, "870": 0.0, "873": 0.7870111744655099, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.6309297535714575, "1012": 1.0, "1014": 1.0, "1019": 0.6309297535714575, "1020": 0.6309297535714575, "1021": 0.6309297535714575, "1024": 0.5, "1029": 0.9134015924715543, "1041": 0.430624116386567, "1049": 0.3333333333333333, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 0.5, "1100": 0.31546487678572877, "1104": 1.0, "1107": 0.6309297535714575, "1110": 0.2890648263178879, "1121": 1.0, "1130": 1.0, "1132": 0.20438239758848611, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.38685280723454163, "1185": 1.0, "1187": 0.6309297535714575, "1191": 0.38685280723454163, "1194": 1.0, "1196": 0.43067655807339306, "1197": 0.38685280723454163, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 0.38685280723454163, "1225": 1.0, "1226": 0.3010299956639812, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 0.8278123145308894, "1278": 0.6309297535714575, "1279": 0.0, "1280": 0.3333333333333333, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.6309297535714575, "1320": 0.6309297535714575, "1332": 0.0, "1335": 0.6309297535714575, "1336": 0.6309297535714575, "1337": 1.0, "1339": 1.0, "1344": 0.6309297535714575, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 0.6309297535714575, "1370": 1.0, "1379": 0.7095272044910244, "1382": 0.5, "1385": 1.0, "1389": 1.0, "1395": 0.5}, "Recall@20": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 1.0, "42": 1.0, "48": 1.0, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.5, "72": 1.0, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 1.0, "213": 1.0, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 1.0, "233": 1.0, "236": 1.0, "237": 1.0, "238": 1.0, "239": 1.0, "248": 1.0, "249": 1.0, "261": 1.0, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 1.0, "303": 0.0, "312": 0.0, "314": 1.0, "324": 0.0, "327": 1.0, "338": 1.0, "343": 1.0, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 1.0, "385": 1.0, "386": 1.0, "388": 1.0, "399": 1.0, "410": 1.0, "411": 1.0, "415": 1.0, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.5, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 1.0, "508": 1.0, "513": 1.0, "514": 1.0, "516": 1.0, "517": 1.0, "521": 1.0, "525": 1.0, "527": 1.0, "528": 1.0, "532": 1.0, "533": 1.0, "535": 1.0, "536": 1.0, "539": 1.0, "540": 1.0, "544": 1.0, "549": 1.0, "551": 1.0, "552": 1.0, "554": 1.0, "560": 0.0, "569": 1.0, "575": 1.0, "577": 1.0, "578": 1.0, "587": 1.0, "589": 1.0, "593": 1.0, "597": 1.0, "598": 1.0, "613": 1.0, "619": 0.0, "623": 1.0, "628": 1.0, "636": 1.0, "637": 1.0, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.0, "660": 1.0, "674": 1.0, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 1.0, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 1.0, "756": 1.0, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 1.0, "784": 1.0, "785": 1.0, "793": 1.0, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 1.0, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 1.0, "847": 1.0, "852": 1.0, "859": 1.0, "870": 1.0, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 1.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 1.0, "1012": 1.0, "1014": 1.0, "1019": 1.0, "1020": 1.0, "1021": 1.0, "1024": 1.0, "1029": 1.0, "1041": 1.0, "1049": 1.0, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 1.0, "1100": 1.0, "1104": 1.0, "1107": 1.0, "1110": 1.0, "1121": 1.0, "1130": 1.0, "1132": 1.0, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 1.0, "1185": 1.0, "1187": 1.0, "1191": 1.0, "1194": 1.0, "1196": 1.0, "1197": 1.0, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 1.0, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 1.0, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 1.0, "1279": 0.0, "1280": 1.0, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 1.0, "1320": 1.0, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 1.0, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 1.0, "1370": 1.0, "1379": 1.0, "1382": 1.0, "1385": 1.0, "1389": 1.0, "1395": 1.0}, "MRR@10": {"1": 0.0, "3": 0.3333333333333333, "5": 1.0, "13": 0.0, "36": 0.25, "42": 1.0, "48": 0.3333333333333333, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.2, "72": 0.5, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 0.5, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.3333333333333333, "213": 0.14285714285714285, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.5, "233": 1.0, "236": 1.0, "237": 0.5, "238": 0.3333333333333333, "239": 0.25, "248": 1.0, "249": 1.0, "261": 0.5, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 0.5, "298": 1.0, "300": 0.07692307692307693, "303": 0.0, "312": 0.0, "314": 0.16666666666666666, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.5, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.16666666666666666, "385": 0.5, "386": 1.0, "388": 0.5, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.5, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3333333333333333, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.5, "508": 0.1111111111111111, "513": 0.5, "514": 1.0, "516": 1.0, "517": 0.14285714285714285, "521": 1.0, "525": 1.0, "527": 0.5, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.14285714285714285, "536": 1.0, "539": 1.0, "540": 0.5, "544": 0.2, "549": 1.0, "551": 0.2, "552": 1.0, "554": 0.1111111111111111, "560": 0.0, "569": 0.2, "575": 1.0, "577": 0.5, "578": 1.0, "587": 0.5, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.3333333333333333, "613": 1.0, "619": 0.0, "623": 0.5, "628": 1.0, "636": 1.0, "637": 0.5, "641": 1.0, "644": 1.0, "649": 0.5, "659": 0.0, "660": 0.07692307692307693, "674": 0.5, "684": 0.5, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.2, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.3333333333333333, "756": 0.5, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.14285714285714285, "784": 1.0, "785": 1.0, "793": 0.25, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.25, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.3333333333333333, "847": 1.0, "852": 0.5, "859": 0.5, "870": 0.07142857142857142, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0625, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.5, "1012": 1.0, "1014": 1.0, "1019": 0.5, "1020": 0.5, "1021": 0.5, "1024": 0.3333333333333333, "1029": 1.0, "1041": 0.2, "1049": 0.14285714285714285, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 0.3333333333333333, "1100": 0.125, "1104": 1.0, "1107": 0.5, "1110": 0.1, "1121": 1.0, "1130": 1.0, "1132": 0.14285714285714285, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.2, "1185": 1.0, "1187": 0.5, "1191": 0.2, "1194": 1.0, "1196": 0.25, "1197": 0.2, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.07692307692307693, "1216": 1.0, "1221": 0.2, "1225": 1.0, "1226": 0.1111111111111111, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 0.5, "1279": 0.0, "1280": 0.14285714285714285, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.5, "1320": 0.5, "1332": 0.0, "1335": 0.5, "1336": 0.5, "1337": 1.0, "1339": 1.0, "1344": 0.5, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 0.5, "1370": 1.0, "1379": 1.0, "1382": 0.3333333333333333, "1385": 1.0, "1389": 1.0, "1395": 0.3333333333333333}}} diff --git a/evals/benchmark/results-adaptive/scifact/ir-adm0.30.jsonl b/evals/benchmark/results-adaptive/scifact/ir-adm0.30.jsonl new file mode 100644 index 000000000..ceb56f915 --- /dev/null +++ b/evals/benchmark/results-adaptive/scifact/ir-adm0.30.jsonl @@ -0,0 +1 @@ +{"dataset": "scifact", "run_tag": "adm0.30", "aggregated": {"nDCG@10": 0.7233, "Recall@20": 0.8867, "MRR@10": 0.6937}, "per_query": {"nDCG@10": {"1": 0.0, "3": 0.6309297535714575, "5": 1.0, "13": 0.0, "36": 0.2640681225725909, "42": 1.0, "48": 0.6309297535714575, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.2640681225725909, "72": 0.6309297535714575, "75": 1.0, "94": 0.2890648263178879, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 0.8712322899646365, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 0.7095272044910244, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.5, "213": 0.38685280723454163, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.6309297535714575, "233": 1.0, "236": 1.0, "237": 0.6309297535714575, "238": 0.5, "239": 0.38685280723454163, "248": 1.0, "249": 1.0, "261": 0.46845052016107697, "268": 1.0, "269": 1.0, "274": 1.0, "275": 0.6508205185601091, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 0.5, "303": 0.0, "312": 0.0, "314": 0.5, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.6509209298071326, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.3562071871080222, "385": 0.6934264036172708, "386": 1.0, "388": 0.6309297535714575, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.5, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3065735963827292, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.6309297535714575, "508": 0.3010299956639812, "513": 0.6309297535714575, "514": 1.0, "516": 1.0, "517": 0.3333333333333333, "521": 1.0, "525": 1.0, "527": 0.6309297535714575, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.3333333333333333, "536": 1.0, "539": 1.0, "540": 0.38685280723454163, "544": 0.38685280723454163, "549": 1.0, "551": 0.38685280723454163, "552": 1.0, "554": 0.31546487678572877, "560": 0.0, "569": 0.6309297535714575, "575": 1.0, "577": 0.6309297535714575, "578": 1.0, "587": 0.6309297535714575, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.5, "613": 1.0, "619": 0.0, "623": 0.0, "628": 1.0, "636": 1.0, "637": 0.6309297535714575, "641": 0.8065735963827293, "644": 1.0, "649": 1.0, "659": 0.0, "660": 0.31546487678572877, "674": 0.6309297535714575, "684": 0.6309297535714575, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.43067655807339306, "728": 0.8772153153380493, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.6309297535714575, "756": 0.5, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.3333333333333333, "784": 1.0, "785": 1.0, "793": 0.6309297535714575, "800": 0.38685280723454163, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.43067655807339306, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.6309297535714575, "847": 1.0, "852": 1.0, "859": 0.6309297535714575, "870": 0.0, "873": 0.7870111744655099, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.6309297535714575, "1012": 1.0, "1014": 1.0, "1019": 0.6309297535714575, "1020": 0.6309297535714575, "1021": 0.6309297535714575, "1024": 0.6309297535714575, "1029": 0.7653606369886217, "1041": 0.39780880120575696, "1049": 0.3333333333333333, "1062": 1.0, "1086": 1.0, "1088": 0.38685280723454163, "1089": 1.0, "1099": 0.5, "1100": 0.3562071871080222, "1104": 1.0, "1107": 0.6309297535714575, "1110": 0.2890648263178879, "1121": 1.0, "1130": 1.0, "1132": 0.21840743681816419, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.3562071871080222, "1185": 1.0, "1187": 1.0, "1191": 0.38685280723454163, "1194": 1.0, "1196": 0.43067655807339306, "1197": 0.43067655807339306, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 0.38685280723454163, "1225": 1.0, "1226": 0.3010299956639812, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 0.8278123145308894, "1278": 0.6309297535714575, "1279": 0.0, "1280": 0.31546487678572877, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.6309297535714575, "1320": 0.6309297535714575, "1332": 0.0, "1335": 0.6309297535714575, "1336": 0.6309297535714575, "1337": 1.0, "1339": 1.0, "1344": 0.6309297535714575, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 0.43067655807339306, "1370": 1.0, "1379": 0.7095272044910244, "1382": 0.5, "1385": 1.0, "1389": 1.0, "1395": 0.5}, "Recall@20": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 1.0, "42": 1.0, "48": 1.0, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 1.0, "72": 1.0, "75": 1.0, "94": 1.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 1.0, "208": 1.0, "212": 1.0, "213": 1.0, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 1.0, "233": 1.0, "236": 1.0, "237": 1.0, "238": 1.0, "239": 1.0, "248": 1.0, "249": 1.0, "261": 1.0, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 1.0, "303": 0.0, "312": 0.0, "314": 1.0, "324": 1.0, "327": 1.0, "338": 1.0, "343": 1.0, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 1.0, "385": 1.0, "386": 1.0, "388": 1.0, "399": 1.0, "410": 1.0, "411": 1.0, "415": 1.0, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.5, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 1.0, "508": 1.0, "513": 1.0, "514": 1.0, "516": 1.0, "517": 1.0, "521": 1.0, "525": 1.0, "527": 1.0, "528": 1.0, "532": 1.0, "533": 1.0, "535": 1.0, "536": 1.0, "539": 1.0, "540": 1.0, "544": 1.0, "549": 1.0, "551": 1.0, "552": 1.0, "554": 1.0, "560": 0.0, "569": 1.0, "575": 1.0, "577": 1.0, "578": 1.0, "587": 1.0, "589": 1.0, "593": 1.0, "597": 1.0, "598": 1.0, "613": 1.0, "619": 0.5, "623": 1.0, "628": 1.0, "636": 1.0, "637": 1.0, "641": 1.0, "644": 1.0, "649": 1.0, "659": 1.0, "660": 1.0, "674": 1.0, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 1.0, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 1.0, "756": 1.0, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 1.0, "784": 1.0, "785": 1.0, "793": 1.0, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 1.0, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 1.0, "847": 1.0, "852": 1.0, "859": 1.0, "870": 1.0, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 1.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 1.0, "1012": 1.0, "1014": 1.0, "1019": 1.0, "1020": 1.0, "1021": 1.0, "1024": 1.0, "1029": 1.0, "1041": 1.0, "1049": 1.0, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 1.0, "1100": 1.0, "1104": 1.0, "1107": 1.0, "1110": 1.0, "1121": 1.0, "1130": 1.0, "1132": 1.0, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 1.0, "1185": 1.0, "1187": 1.0, "1191": 1.0, "1194": 1.0, "1196": 1.0, "1197": 1.0, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 1.0, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 1.0, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 1.0, "1279": 0.0, "1280": 1.0, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 1.0, "1320": 1.0, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 1.0, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 1.0, "1368": 1.0, "1370": 1.0, "1379": 1.0, "1382": 1.0, "1385": 1.0, "1389": 1.0, "1395": 1.0}, "MRR@10": {"1": 0.0, "3": 0.5, "5": 1.0, "13": 0.0, "36": 0.25, "42": 1.0, "48": 0.5, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.25, "72": 0.5, "75": 1.0, "94": 0.1, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0625, "208": 1.0, "212": 0.3333333333333333, "213": 0.2, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.5, "233": 1.0, "236": 1.0, "237": 0.5, "238": 0.3333333333333333, "239": 0.2, "248": 1.0, "249": 1.0, "261": 0.25, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 0.3333333333333333, "303": 0.0, "312": 0.0, "314": 0.3333333333333333, "324": 0.09090909090909091, "327": 1.0, "338": 1.0, "343": 0.5, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.16666666666666666, "385": 0.5, "386": 1.0, "388": 0.5, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.3333333333333333, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3333333333333333, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.5, "508": 0.1111111111111111, "513": 0.5, "514": 1.0, "516": 1.0, "517": 0.14285714285714285, "521": 1.0, "525": 1.0, "527": 0.5, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.14285714285714285, "536": 1.0, "539": 1.0, "540": 0.5, "544": 0.2, "549": 1.0, "551": 0.2, "552": 1.0, "554": 0.125, "560": 0.0, "569": 0.5, "575": 1.0, "577": 0.5, "578": 1.0, "587": 0.5, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.3333333333333333, "613": 1.0, "619": 0.0625, "623": 0.08333333333333333, "628": 1.0, "636": 1.0, "637": 0.5, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.08333333333333333, "660": 0.125, "674": 0.5, "684": 0.5, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.25, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.5, "756": 0.3333333333333333, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.14285714285714285, "784": 1.0, "785": 1.0, "793": 0.5, "800": 0.2, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.25, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.5, "847": 1.0, "852": 1.0, "859": 0.5, "870": 0.07142857142857142, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0625, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.5, "1012": 1.0, "1014": 1.0, "1019": 0.5, "1020": 0.5, "1021": 0.5, "1024": 0.5, "1029": 1.0, "1041": 0.14285714285714285, "1049": 0.14285714285714285, "1062": 1.0, "1086": 1.0, "1088": 0.2, "1089": 1.0, "1099": 0.3333333333333333, "1100": 0.16666666666666666, "1104": 1.0, "1107": 0.5, "1110": 0.1, "1121": 1.0, "1130": 1.0, "1132": 0.16666666666666666, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.16666666666666666, "1185": 1.0, "1187": 1.0, "1191": 0.2, "1194": 1.0, "1196": 0.25, "1197": 0.25, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.07692307692307693, "1216": 1.0, "1221": 0.2, "1225": 1.0, "1226": 0.1111111111111111, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 0.5, "1279": 0.0, "1280": 0.125, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.5, "1320": 0.5, "1332": 0.0, "1335": 0.5, "1336": 0.5, "1337": 1.0, "1339": 1.0, "1344": 0.5, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0625, "1368": 0.25, "1370": 1.0, "1379": 1.0, "1382": 0.3333333333333333, "1385": 1.0, "1389": 1.0, "1395": 0.3333333333333333}}} diff --git a/evals/benchmark/results-adaptive/scifact/ir-adm0.50.jsonl b/evals/benchmark/results-adaptive/scifact/ir-adm0.50.jsonl new file mode 100644 index 000000000..a893b329c --- /dev/null +++ b/evals/benchmark/results-adaptive/scifact/ir-adm0.50.jsonl @@ -0,0 +1 @@ +{"dataset": "scifact", "run_tag": "adm0.50", "aggregated": {"nDCG@10": 0.7275, "Recall@20": 0.8917, "MRR@10": 0.6966}, "per_query": {"nDCG@10": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 0.3065735963827292, "42": 1.0, "48": 0.6309297535714575, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.2640681225725909, "72": 0.6309297535714575, "75": 1.0, "94": 0.3333333333333333, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 0.8712322899646365, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 0.7095272044910244, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.5, "213": 0.43067655807339306, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.6309297535714575, "233": 1.0, "236": 1.0, "237": 1.0, "238": 0.5, "239": 0.38685280723454163, "248": 1.0, "249": 1.0, "261": 0.46845052016107697, "268": 1.0, "269": 1.0, "274": 1.0, "275": 0.6364391809889587, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 0.5, "303": 0.0, "312": 0.0, "314": 0.5, "324": 0.3010299956639812, "327": 1.0, "338": 1.0, "343": 0.6509209298071326, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.3562071871080222, "385": 0.6934264036172708, "386": 1.0, "388": 0.6309297535714575, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.5, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3065735963827292, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.6309297535714575, "508": 0.3010299956639812, "513": 0.6309297535714575, "514": 1.0, "516": 1.0, "517": 0.3333333333333333, "521": 1.0, "525": 1.0, "527": 0.6309297535714575, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.3333333333333333, "536": 1.0, "539": 1.0, "540": 0.38685280723454163, "544": 0.38685280723454163, "549": 1.0, "551": 0.38685280723454163, "552": 1.0, "554": 0.3333333333333333, "560": 0.0, "569": 0.6309297535714575, "575": 1.0, "577": 0.6309297535714575, "578": 1.0, "587": 0.6309297535714575, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.43067655807339306, "613": 1.0, "619": 0.0, "623": 0.0, "628": 1.0, "636": 1.0, "637": 0.6309297535714575, "641": 0.8065735963827293, "644": 1.0, "649": 1.0, "659": 0.31546487678572877, "660": 0.38685280723454163, "674": 0.6309297535714575, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.43067655807339306, "728": 0.6934264036172708, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.6309297535714575, "756": 0.5, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.3333333333333333, "784": 1.0, "785": 1.0, "793": 0.6309297535714575, "800": 0.38685280723454163, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.43067655807339306, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.6309297535714575, "847": 1.0, "852": 1.0, "859": 0.6309297535714575, "870": 0.0, "873": 0.7870111744655099, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.6309297535714575, "1012": 1.0, "1014": 1.0, "1019": 0.6309297535714575, "1020": 0.6309297535714575, "1021": 0.6309297535714575, "1024": 0.6309297535714575, "1029": 0.7653606369886217, "1041": 0.41183384043543503, "1049": 0.3333333333333333, "1062": 1.0, "1086": 1.0, "1088": 0.38685280723454163, "1089": 1.0, "1099": 0.5, "1100": 0.3562071871080222, "1104": 1.0, "1107": 0.6309297535714575, "1110": 0.0, "1121": 1.0, "1130": 1.0, "1132": 0.21840743681816419, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.3562071871080222, "1185": 1.0, "1187": 1.0, "1191": 0.38685280723454163, "1194": 1.0, "1196": 0.43067655807339306, "1197": 0.43067655807339306, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 0.38685280723454163, "1225": 1.0, "1226": 0.3010299956639812, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 0.8278123145308894, "1278": 0.6309297535714575, "1279": 0.0, "1280": 0.31546487678572877, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.6309297535714575, "1320": 0.6309297535714575, "1332": 0.0, "1335": 0.6309297535714575, "1336": 0.6309297535714575, "1337": 1.0, "1339": 1.0, "1344": 0.6309297535714575, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.2890648263178879, "1368": 0.38685280723454163, "1370": 0.6309297535714575, "1379": 0.6886342695939197, "1382": 0.5, "1385": 1.0, "1389": 1.0, "1395": 0.5}, "Recall@20": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 1.0, "42": 1.0, "48": 1.0, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 1.0, "72": 1.0, "75": 1.0, "94": 1.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 1.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 1.0, "208": 1.0, "212": 1.0, "213": 1.0, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 1.0, "233": 1.0, "236": 1.0, "237": 1.0, "238": 1.0, "239": 1.0, "248": 1.0, "249": 1.0, "261": 1.0, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 1.0, "295": 1.0, "298": 1.0, "300": 1.0, "303": 0.0, "312": 0.0, "314": 1.0, "324": 1.0, "327": 1.0, "338": 1.0, "343": 1.0, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 1.0, "385": 1.0, "386": 1.0, "388": 1.0, "399": 1.0, "410": 1.0, "411": 1.0, "415": 1.0, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.5, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 1.0, "508": 1.0, "513": 1.0, "514": 1.0, "516": 1.0, "517": 1.0, "521": 1.0, "525": 1.0, "527": 1.0, "528": 1.0, "532": 1.0, "533": 1.0, "535": 1.0, "536": 1.0, "539": 1.0, "540": 0.5, "544": 1.0, "549": 1.0, "551": 1.0, "552": 1.0, "554": 1.0, "560": 0.0, "569": 1.0, "575": 1.0, "577": 1.0, "578": 1.0, "587": 1.0, "589": 1.0, "593": 1.0, "597": 1.0, "598": 1.0, "613": 1.0, "619": 0.5, "623": 1.0, "628": 1.0, "636": 1.0, "637": 1.0, "641": 1.0, "644": 1.0, "649": 1.0, "659": 1.0, "660": 1.0, "674": 1.0, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 1.0, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 1.0, "756": 1.0, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 1.0, "784": 1.0, "785": 1.0, "793": 1.0, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 1.0, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 1.0, "847": 1.0, "852": 1.0, "859": 1.0, "870": 1.0, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 1.0, "1012": 1.0, "1014": 1.0, "1019": 1.0, "1020": 1.0, "1021": 1.0, "1024": 1.0, "1029": 1.0, "1041": 1.0, "1049": 1.0, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 1.0, "1100": 1.0, "1104": 1.0, "1107": 1.0, "1110": 0.0, "1121": 1.0, "1130": 1.0, "1132": 1.0, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 1.0, "1185": 1.0, "1187": 1.0, "1191": 1.0, "1194": 1.0, "1196": 1.0, "1197": 1.0, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 1.0, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 1.0, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 1.0, "1279": 1.0, "1280": 1.0, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 1.0, "1320": 1.0, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 1.0, "1352": 1.0, "1359": 1.0, "1362": 1.0, "1363": 1.0, "1368": 1.0, "1370": 1.0, "1379": 1.0, "1382": 1.0, "1385": 1.0, "1389": 1.0, "1395": 1.0}, "MRR@10": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 0.3333333333333333, "42": 1.0, "48": 0.5, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.25, "72": 0.5, "75": 1.0, "94": 0.14285714285714285, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.07692307692307693, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.08333333333333333, "208": 1.0, "212": 0.3333333333333333, "213": 0.25, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.5, "233": 1.0, "236": 1.0, "237": 1.0, "238": 0.3333333333333333, "239": 0.2, "248": 1.0, "249": 1.0, "261": 0.25, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.05263157894736842, "295": 1.0, "298": 1.0, "300": 0.3333333333333333, "303": 0.0, "312": 0.0, "314": 0.3333333333333333, "324": 0.1111111111111111, "327": 1.0, "338": 1.0, "343": 0.5, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.16666666666666666, "385": 0.5, "386": 1.0, "388": 0.5, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.3333333333333333, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3333333333333333, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.5, "508": 0.1111111111111111, "513": 0.5, "514": 1.0, "516": 1.0, "517": 0.14285714285714285, "521": 1.0, "525": 1.0, "527": 0.5, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.14285714285714285, "536": 1.0, "539": 1.0, "540": 0.5, "544": 0.2, "549": 1.0, "551": 0.2, "552": 1.0, "554": 0.14285714285714285, "560": 0.0, "569": 0.5, "575": 1.0, "577": 0.5, "578": 1.0, "587": 0.5, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.25, "613": 1.0, "619": 0.0625, "623": 0.08333333333333333, "628": 1.0, "636": 1.0, "637": 0.5, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.125, "660": 0.2, "674": 0.5, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.25, "728": 0.5, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.5, "756": 0.3333333333333333, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.14285714285714285, "784": 1.0, "785": 1.0, "793": 0.5, "800": 0.2, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.25, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.5, "847": 1.0, "852": 1.0, "859": 0.5, "870": 0.07142857142857142, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.5, "1012": 1.0, "1014": 1.0, "1019": 0.5, "1020": 0.5, "1021": 0.5, "1024": 0.5, "1029": 1.0, "1041": 0.16666666666666666, "1049": 0.14285714285714285, "1062": 1.0, "1086": 1.0, "1088": 0.2, "1089": 1.0, "1099": 0.3333333333333333, "1100": 0.16666666666666666, "1104": 1.0, "1107": 0.5, "1110": 0.0, "1121": 1.0, "1130": 1.0, "1132": 0.16666666666666666, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.16666666666666666, "1185": 1.0, "1187": 1.0, "1191": 0.2, "1194": 1.0, "1196": 0.25, "1197": 0.25, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.07142857142857142, "1216": 1.0, "1221": 0.2, "1225": 1.0, "1226": 0.1111111111111111, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 0.5, "1279": 0.0625, "1280": 0.125, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.5, "1320": 0.5, "1332": 0.0, "1335": 0.5, "1336": 0.5, "1337": 1.0, "1339": 1.0, "1344": 0.5, "1352": 1.0, "1359": 1.0, "1362": 0.08333333333333333, "1363": 0.1, "1368": 0.2, "1370": 0.5, "1379": 1.0, "1382": 0.3333333333333333, "1385": 1.0, "1389": 1.0, "1395": 0.3333333333333333}}} diff --git a/evals/benchmark/results-adaptive/scifact/run-adm0.15.trec.gz b/evals/benchmark/results-adaptive/scifact/run-adm0.15.trec.gz new file mode 100644 index 000000000..8351af0c0 Binary files /dev/null and b/evals/benchmark/results-adaptive/scifact/run-adm0.15.trec.gz differ diff --git a/evals/benchmark/results-adaptive/scifact/stats-adm0.15-vs-dense.jsonl b/evals/benchmark/results-adaptive/scifact/stats-adm0.15-vs-dense.jsonl new file mode 100644 index 000000000..f21b045ba --- /dev/null +++ b/evals/benchmark/results-adaptive/scifact/stats-adm0.15-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "adm0.15", "arm_b": "dense"} +{"row_type": "ir", "dataset": "scifact", "metric": "nDCG@10", "n": 300, "mean_a": 0.7186192497673477, "mean_b": 0.6981162821923903, "mean_diff": -0.02050296757495748, "ci_low": -0.03670825373269546, "ci_high": -0.005773315508087141, "p_value": 0.008599140085991401, "significant": true} +{"row_type": "ir", "dataset": "scifact", "metric": "Recall@20", "n": 300, "mean_a": 0.8666666666666667, "mean_b": 0.8633333333333333, "mean_diff": -0.0033333333333333335, "ci_low": -0.01, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "scifact", "metric": "MRR@10", "n": 300, "mean_a": 0.6866878815628815, "mean_b": 0.6647890581640581, "mean_diff": -0.0218988233988234, "ci_low": -0.04104473211973213, "ci_high": -0.003594316100566109, "p_value": 0.0225977402259774, "significant": true} diff --git a/evals/benchmark/results-adaptive/scifact/stats-adm0.30-vs-dense.jsonl b/evals/benchmark/results-adaptive/scifact/stats-adm0.30-vs-dense.jsonl new file mode 100644 index 000000000..7463cb292 --- /dev/null +++ b/evals/benchmark/results-adaptive/scifact/stats-adm0.30-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "adm0.30", "arm_b": "dense"} +{"row_type": "ir", "dataset": "scifact", "metric": "nDCG@10", "n": 300, "mean_a": 0.7233040792450709, "mean_b": 0.6981162821923903, "mean_diff": -0.02518779705268061, "ci_low": -0.04409942699131636, "ci_high": -0.00725354443545278, "p_value": 0.008099190080991902, "significant": true} +{"row_type": "ir", "dataset": "scifact", "metric": "Recall@20", "n": 300, "mean_a": 0.8866666666666667, "mean_b": 0.8633333333333333, "mean_diff": -0.023333333333333334, "ci_low": -0.04, "ci_high": -0.008333333333333333, "p_value": 0.006899310068993101, "significant": true} +{"row_type": "ir", "dataset": "scifact", "metric": "MRR@10", "n": 300, "mean_a": 0.6936692289192289, "mean_b": 0.6647890581640581, "mean_diff": -0.028880170755170756, "ci_low": -0.05118670276482777, "ci_high": -0.007652527102527105, "p_value": 0.0100989901009899, "significant": true} diff --git a/evals/benchmark/results-adaptive/scifact/stats-adm0.50-vs-dense.jsonl b/evals/benchmark/results-adaptive/scifact/stats-adm0.50-vs-dense.jsonl new file mode 100644 index 000000000..bd47dcdfc --- /dev/null +++ b/evals/benchmark/results-adaptive/scifact/stats-adm0.50-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "adm0.50", "arm_b": "dense"} +{"row_type": "ir", "dataset": "scifact", "metric": "nDCG@10", "n": 300, "mean_a": 0.727491759732547, "mean_b": 0.6981162821923903, "mean_diff": -0.02937547754015673, "ci_low": -0.04934259246796051, "ci_high": -0.009909930265102123, "p_value": 0.0034996500349965005, "significant": true} +{"row_type": "ir", "dataset": "scifact", "metric": "Recall@20", "n": 300, "mean_a": 0.8916666666666667, "mean_b": 0.8633333333333333, "mean_diff": -0.028333333333333332, "ci_low": -0.051666666666666666, "ci_high": -0.005, "p_value": 0.024697530246975304, "significant": true} +{"row_type": "ir", "dataset": "scifact", "metric": "MRR@10", "n": 300, "mean_a": 0.6966262932973459, "mean_b": 0.6647890581640581, "mean_diff": -0.03183723513328777, "ci_low": -0.05561938407596302, "ci_high": -0.008638796565276842, "p_value": 0.007699230076992301, "significant": true} diff --git a/evals/benchmark/results/fiqa/ir-dense.jsonl b/evals/benchmark/results/fiqa/ir-dense.jsonl new file mode 100644 index 000000000..d1ccf288b --- /dev/null +++ b/evals/benchmark/results/fiqa/ir-dense.jsonl @@ -0,0 +1 @@ +{"dataset": "fiqa", "run_tag": "dense", "aggregated": {"nDCG@10": 0.4598, "Recall@20": 0.6186, "MRR@10": 0.5448}, "per_query": {"nDCG@10": {"8": 0.6131471927654584, "15": 1.0, "18": 0.6309297535714575, "26": 0.3065735963827292, "34": 0.0, "42": 0.46927872602275644, "56": 0.6309297535714575, "68": 0.0, "89": 0.32228364030009987, "90": 1.0, "94": 1.0, "98": 0.6131471927654584, "104": 0.6131471927654584, "106": 0.6309297535714575, "109": 1.0, "475": 1.0, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 0.0, "585": 0.38685280723454163, "588": 0.44130740935663865, "594": 0.6131471927654584, "603": 0.0, "604": 0.34870224750355494, "620": 0.10209739512291163, "622": 0.6131471927654584, "659": 0.5263304097820353, "672": 0.18457569677956817, "684": 0.0, "687": 0.21840743681816419, "689": 0.0, "691": 0.43067655807339306, "699": 0.6309297535714575, "701": 0.13565197343244778, "715": 0.0, "721": 1.0, "744": 0.23463936301137822, "750": 0.8315546295836225, "753": 0.38685280723454163, "766": 0.20210734650054757, "776": 0.4690000933206748, "810": 0.43067655807339306, "813": 0.23719771276929622, "849": 0.2890648263178879, "852": 0.7653606369886217, "853": 0.21840743681816419, "858": 0.0, "859": 0.6131471927654584, "864": 0.7149394488115768, "879": 0.6309297535714575, "885": 0.23719771276929622, "895": 1.0, "904": 1.0, "928": 0.6131471927654584, "929": 1.0, "932": 0.23719771276929622, "939": 1.0, "945": 0.6309297535714575, "957": 0.6713860725233041, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.3065735963827292, "1150": 0.0, "1157": 0.3065735963827292, "1159": 0.0, "1198": 0.6309297535714575, "1230": 0.0, "1281": 1.0, "1284": 1.0, "1297": 0.5, "1306": 0.0, "1309": 0.5205067333228022, "1310": 1.0, "1321": 0.0, "1322": 0.3903800499921017, "1391": 0.20210734650054757, "1393": 0.5493640533376876, "1415": 0.9178287402913377, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 0.6309297535714575, "1530": 0.7461976723150332, "1670": 0.7653606369886217, "1676": 0.4693608375942211, "1736": 0.5087403079104241, "1748": 0.43734888382569426, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.4034698846650846, "1824": 0.43067655807339306, "1826": 0.9828920819566879, "1832": 0.3562071871080222, "1871": 0.7246262544989281, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 0.43067655807339306, "1994": 0.0, "2010": 0.9469024295259745, "2051": 0.0, "2070": 0.0, "2075": 0.5716213154129578, "2076": 1.0, "2088": 0.0, "2108": 0.20210734650054757, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 0.7526136409516659, "2204": 0.8984972075197749, "2264": 0.18457569677956817, "2296": 0.38334277998463445, "2306": 0.38685280723454163, "2316": 0.8772153153380493, "2318": 0.6309297535714575, "2330": 0.6131471927654584, "2334": 0.46927872602275644, "2348": 0.37185395737611554, "2376": 0.9830168297915423, "2383": 0.6131471927654584, "2384": 1.0, "2385": 0.6131471927654584, "2388": 1.0, "2395": 0.5, "2398": 0.10699313236726378, "2399": 1.0, "2400": 0.6049306994552042, "2407": 0.15101961822780524, "2416": 0.8838242945899706, "2423": 0.5422491080636984, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 0.5, "2513": 0.0, "2516": 0.6366824387328317, "2549": 0.6131471927654584, "2551": 1.0, "2568": 0.3615901614084106, "2579": 0.22485174766247445, "2580": 0.46927872602275644, "2587": 1.0, "2589": 0.7653606369886217, "2590": 0.46927872602275644, "2593": 0.9217868789962071, "2598": 0.9197207891481876, "2648": 1.0, "2676": 1.0, "2685": 0.35079429740281753, "2695": 1.0, "2713": 0.0, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 0.8315546295836225, "2790": 0.49959580689988364, "2801": 0.6131471927654584, "2856": 0.5716190970674814, "2857": 0.4202024810692814, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.3562071871080222, "2923": 0.7972541056822071, "2964": 0.0, "2968": 1.0, "2994": 0.46927872602275644, "3006": 0.26836379988737025, "3008": 0.3903800499921017, "3014": 1.0, "3033": 0.6131471927654584, "3039": 0.3333333333333333, "3049": 0.6366824387328317, "3051": 1.0, "3067": 0.0, "3085": 0.21840743681816419, "3091": 0.8936633189697701, "3103": 1.0, "3115": 0.4329261794091841, "3125": 1.0, "3148": 0.4703652827859579, "3149": 0.0, "3177": 0.14126697285982898, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.7039180890341347, "3264": 0.8048099750039491, "3357": 0.6309297535714575, "3369": 0.09109240322345806, "3394": 0.787702056960637, "3404": 0.3932591258696538, "3405": 1.0, "3446": 0.46704774694449785, "3451": 0.07945707943276742, "3453": 0.6240505200038379, "3480": 1.0, "3490": 0.3333333333333333, "3500": 0.5889837127672154, "3503": 0.6891341765546151, "3512": 1.0, "3528": 1.0, "3530": 0.6508205185601091, "3534": 0.0, "3566": 0.8194270280062366, "3569": 0.0, "3594": 0.13905617951077562, "3612": 0.11751610475642714, "3615": 0.5912352048230277, "3625": 0.46927872602275644, "3682": 0.38685280723454163, "3683": 0.7021799053685303, "3694": 0.6131471927654584, "3724": 0.0, "3735": 0.20210734650054757, "3759": 0.0, "3767": 0.45221360703148816, "3771": 0.4935232796777481, "3781": 0.8772153153380493, "3789": 0.787702056960637, "3791": 0.431733884398313, "3801": 0.0, "3822": 0.0, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.6489315753318465, "3932": 0.0, "3934": 0.11751610475642714, "3995": 0.0, "4007": 0.6364391809889587, "4011": 0.9674679834891693, "4019": 0.4412576003965278, "4031": 0.0, "4037": 0.21840743681816419, "4047": 0.8048099750039491, "4071": 0.0, "4084": 0.9197207891481876, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 0.6131471927654584, "4125": 0.0, "4142": 0.5143371794949736, "4153": 0.38685280723454163, "4179": 0.0, "4188": 0.0, "4205": 0.36926780146674987, "4233": 0.8539316501572937, "4265": 0.7751482523375255, "4286": 0.6131471927654584, "4289": 0.6131471927654584, "4306": 0.7246262544989281, "4312": 0.6370339733570289, "4335": 0.8318724637288826, "4339": 1.0, "4394": 0.0, "4409": 0.31488013066763093, "4411": 0.46927872602275644, "4414": 0.9197207891481876, "4415": 0.0, "4433": 0.21840743681816419, "4447": 0.31546487678572877, "4464": 1.0, "4465": 0.23463936301137822, "4484": 1.0, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 0.48553456068968903, "4523": 0.21398626473452756, "4539": 0.3065735963827292, "4571": 0.7653606369886217, "4600": 1.0, "4605": 0.6773676006644395, "4615": 0.5889556853481906, "4640": 0.0, "4641": 0.5531464700081437, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.7598336331031966, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.3065735963827292, "4813": 1.0, "4823": 0.46927872602275644, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 0.7653606369886217, "4846": 0.0, "4863": 0.6508205185601091, "4865": 0.0, "4920": 1.0, "4942": 0.3333333333333333, "4946": 1.0, "4955": 0.0, "4962": 0.6131471927654584, "4968": 0.38685280723454163, "4981": 0.6508205185601091, "4999": 0.5855700749881525, "5021": 0.0, "5030": 0.0, "5045": 0.31546487678572877, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.3333333333333333, "5080": 0.0, "5083": 0.6131471927654584, "5085": 1.0, "5086": 1.0, "5090": 0.6131471927654584, "5125": 1.0, "5134": 0.6131471927654584, "5150": 0.0, "5155": 0.5, "5172": 0.18457569677956817, "5178": 0.5087403079104241, "5185": 0.0, "5196": 0.46927872602275644, "5206": 0.0, "5228": 0.5, "5231": 0.0, "5241": 0.44615333764087994, "5254": 0.6131471927654584, "5255": 1.0, "5264": 0.7227265726449519, "5271": 0.38268031849431083, "5331": 1.0, "5343": 0.3333333333333333, "5347": 1.0, "5356": 0.14606834984270645, "5369": 0.0, "5374": 0.31546487678572877, "5380": 1.0, "5402": 0.0, "5410": 0.46927872602275644, "5422": 0.35914753008966527, "5427": 0.38685280723454163, "5460": 0.0, "5464": 0.2960819109658652, "5503": 0.6131471927654584, "5505": 1.0, "5511": 0.1610425878239398, "5534": 0.38685280723454163, "5549": 0.0, "5585": 0.6309297535714575, "5592": 1.0, "5616": 0.43067655807339306, "5620": 0.0, "5646": 0.9674679834891693, "5653": 1.0, "5683": 0.6934264036172708, "5710": 0.0, "5741": 0.0, "5763": 0.13565197343244778, "5782": 0.4371994911049732, "5790": 0.0, "5808": 0.5, "5853": 0.1821658288910461, "5862": 0.32531670832675136, "5888": 0.19342640361727081, "5903": 0.0, "5906": 0.39564672360221187, "5940": 0.0, "5951": 0.43426773552878023, "5970": 0.2960819109658652, "5981": 1.0, "5993": 0.22009176629808017, "6002": 0.8630152897016883, "6004": 0.0, "6005": 0.40350157154648025, "6009": 0.31546487678572877, "6041": 0.6366824387328317, "6080": 0.6131471927654584, "6110": 0.3475406804124244, "6121": 0.6131471927654584, "6122": 0.46927872602275644, "6131": 0.6235744328990731, "6133": 0.38685280723454163, "6142": 0.6131471927654584, "6146": 1.0, "6199": 0.3764290720714305, "6219": 0.6934264036172708, "6221": 0.0, "6252": 0.0, "6262": 0.7653606369886217, "6278": 0.36181498356361597, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.3010299956639812, "6467": 0.0, "6468": 0.6131471927654584, "6479": 1.0, "6525": 0.4632423659320675, "6554": 0.17723928678404774, "6562": 0.123151194370365, "6611": 0.6131471927654584, "6612": 0.46927872602275644, "6625": 0.31546487678572877, "6629": 0.0, "6635": 0.6366824387328317, "6644": 0.0, "6647": 0.7653606369886217, "6668": 0.8772153153380493, "6679": 0.5307212739772434, "6683": 0.5249810332008933, "6713": 0.6131471927654584, "6715": 0.3065735963827292, "6746": 0.6131471927654584, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.5413996682199069, "6807": 1.0, "6814": 0.0, "6832": 0.6052602440527057, "6835": 0.38685280723454163, "6849": 0.3333333333333333, "6862": 1.0, "6867": 0.2558209594125084, "6875": 0.0, "6890": 1.0, "6891": 0.6364391809889587, "6896": 0.0, "6901": 0.3065735963827292, "6907": 0.0, "6909": 0.0, "6959": 0.38685280723454163, "6985": 1.0, "7017": 0.0, "7068": 0.5706417189553201, "7071": 0.0, "7080": 0.6509209298071326, "7096": 0.6309297535714575, "7098": 0.38685280723454163, "7105": 0.5, "7109": 0.15642624200758548, "7124": 0.7246262544989281, "7141": 0.0, "7145": 0.20210734650054757, "7178": 1.0, "7188": 0.38685280723454163, "7205": 1.0, "7206": 0.0, "7218": 0.43067655807339306, "7221": 0.0, "7269": 0.0, "7279": 0.502373740777944, "7295": 0.0, "7311": 0.2890648263178879, "7326": 0.0, "7329": 0.38685280723454163, "7344": 0.6131471927654584, "7345": 0.15642624200758548, "7377": 1.0, "7431": 0.0, "7441": 0.18154179253735267, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.16958010263680806, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.20210734650054757, "7529": 0.6131471927654584, "7533": 0.17723928678404774, "7534": 0.46927872602275644, "7590": 0.9197207891481876, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 0.3903800499921017, "7674": 0.0, "7700": 0.13905617951077562, "7702": 0.6240505200038379, "7705": 0.6049306994552042, "7734": 0.6131471927654584, "7747": 0.17723928678404774, "7754": 0.6309297535714575, "7758": 1.0, "7801": 0.8315546295836225, "7803": 0.0, "7823": 0.7039180890341347, "7876": 0.5, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.30260241349881345, "7928": 0.0, "7936": 0.6508205185601091, "7992": 0.23719771276929622, "8002": 0.6257049680303419, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.3333333333333333, "8040": 1.0, "8072": 0.617319681505689, "8079": 0.8451850618939637, "8102": 0.13012668333070054, "8116": 0.5, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 0.6131471927654584, "8275": 0.0, "8296": 0.9197207891481876, "8332": 0.9469024295259745, "8351": 0.6131471927654584, "8378": 0.16716045496620227, "8456": 0.617319681505689, "8475": 0.8104616303452685, "8507": 0.0, "8512": 0.6309297535714575, "8513": 0.0, "8532": 0.7988614740430418, "8537": 0.5, "8539": 0.11284514134893527, "8544": 0.6131471927654584, "8592": 0.20210734650054757, "8632": 0.3065735963827292, "8635": 0.0, "8702": 0.6713860725233041, "8779": 0.448643819352159, "8789": 0.7653606369886217, "8795": 1.0, "8832": 1.0, "8834": 0.3980628465882808, "8855": 0.0, "8874": 0.0, "8934": 0.8772153153380493, "8937": 0.6309297535714575, "8947": 0.6131471927654584, "8959": 0.3010299956639812, "8970": 0.6934264036172708, "8974": 0.35079429740281753, "8982": 0.0, "9060": 0.21840743681816419, "9088": 0.0, "9108": 0.0, "9115": 0.6653497124326151, "9126": 0.0, "9164": 0.1480409554829326, "9174": 0.0, "9188": 0.6131471927654584, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.38685280723454163, "9329": 0.8065735963827293, "9332": 0.43067655807339306, "9381": 0.6934264036172708, "9385": 1.0, "9391": 0.75369761125927, "9403": 0.0, "9481": 0.5706417189553201, "9487": 0.38685280723454163, "9548": 0.43067655807339306, "9556": 1.0, "9565": 1.0, "9598": 0.5, "9617": 0.6364391809889587, "9633": 0.6131471927654584, "9643": 0.5, "9644": 0.43067655807339306, "9646": 0.5109559939712153, "9668": 0.5307212739772434, "9701": 0.0, "9733": 0.23463936301137822, "9735": 0.0, "9737": 0.7653606369886217, "9771": 0.0, "9808": 0.14126697285982898, "9824": 1.0, "9871": 0.0, "9882": 0.0, "9925": 0.8175295903539445, "9929": 0.8065735963827293, "9961": 0.0, "9979": 0.5078961547485289, "10034": 0.6131471927654584, "10039": 1.0, "10109": 0.5306972923267992, "10122": 0.0, "10136": 0.38268031849431083, "10137": 0.38685280723454163, "10152": 1.0, "10183": 0.3065735963827292, "10213": 0.15642624200758548, "10246": 0.0, "10267": 0.0, "10414": 0.9197207891481876, "10447": 0.16716045496620227, "10462": 0.0, "10482": 0.0, "10497": 0.494639655562801, "10526": 0.2640681225725909, "10547": 0.0, "10558": 0.0, "10596": 0.9318171001351273, "10601": 0.5, "10628": 0.3562071871080222, "10639": 0.3120255505658846, "10645": 0.5109559939712153, "10674": 0.8048099750039491, "10710": 0.5, "10734": 0.7751482523375255, "10792": 0.8503449055347546, "10808": 0.5374517914100283, "10809": 0.4441228664487979, "10812": 0.7365896932159578, "10827": 0.3391602052736161, "10845": 1.0, "10912": 0.3010299956639812, "10932": 1.0, "10975": 0.0, "10979": 0.5855700749881525, "10994": 0.5, "11039": 0.38391202314388534, "11054": 1.0, "11088": 0.5}, "Recall@20": {"8": 0.5, "15": 1.0, "18": 1.0, "26": 0.5, "34": 0.0, "42": 0.3333333333333333, "56": 1.0, "68": 0.0, "89": 0.5, "90": 1.0, "94": 1.0, "98": 0.5, "104": 0.5, "106": 1.0, "109": 1.0, "475": 1.0, "503": 1.0, "504": 0.3333333333333333, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 1.0, "585": 0.5, "588": 1.0, "594": 1.0, "603": 0.0, "604": 1.0, "620": 0.2, "622": 0.5, "659": 0.7, "672": 0.5, "684": 1.0, "687": 0.5, "689": 0.0, "691": 1.0, "699": 1.0, "701": 0.3333333333333333, "715": 0.0, "721": 1.0, "744": 0.3333333333333333, "750": 1.0, "753": 0.5, "766": 0.3333333333333333, "776": 0.25, "810": 1.0, "813": 1.0, "849": 1.0, "852": 0.6666666666666666, "853": 0.5, "858": 0.0, "859": 0.5, "864": 0.6666666666666666, "879": 1.0, "885": 0.5, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.5, "939": 1.0, "945": 1.0, "957": 0.6666666666666666, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.5, "1150": 0.0, "1157": 0.5, "1159": 0.0, "1198": 1.0, "1230": 1.0, "1281": 1.0, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.75, "1310": 1.0, "1321": 0.0, "1322": 0.25, "1391": 0.6666666666666666, "1393": 0.5, "1415": 1.0, "1416": 1.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 1.0, "1670": 0.6666666666666666, "1676": 1.0, "1736": 0.4, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 1.0, "1826": 1.0, "1832": 1.0, "1871": 0.75, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.4, "2010": 1.0, "2051": 1.0, "2070": 0.6666666666666666, "2075": 0.6666666666666666, "2076": 1.0, "2088": 0.0, "2108": 0.3333333333333333, "2118": 0.0, "2154": 0.0, "2181": 0.5, "2183": 0.8333333333333334, "2204": 1.0, "2264": 0.5, "2296": 0.2222222222222222, "2306": 0.5, "2316": 1.0, "2318": 1.0, "2330": 0.5, "2334": 0.6666666666666666, "2348": 0.26666666666666666, "2376": 1.0, "2383": 0.5, "2384": 1.0, "2385": 0.5, "2388": 1.0, "2395": 1.0, "2398": 0.2, "2399": 1.0, "2400": 0.6666666666666666, "2407": 0.5, "2416": 1.0, "2423": 0.8571428571428571, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.3333333333333333, "2472": 0.0, "2486": 1.0, "2498": 1.0, "2513": 1.0, "2516": 0.75, "2549": 0.5, "2551": 1.0, "2568": 0.2857142857142857, "2579": 0.5, "2580": 0.3333333333333333, "2587": 1.0, "2589": 0.6666666666666666, "2590": 0.6666666666666666, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 1.0, "2685": 0.25, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 1.0, "2790": 0.42857142857142855, "2801": 1.0, "2856": 0.75, "2857": 0.75, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 1.0, "2923": 0.8, "2964": 0.5, "2968": 1.0, "2994": 0.3333333333333333, "3006": 0.5, "3008": 0.25, "3014": 1.0, "3033": 0.5, "3039": 1.0, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 1.0, "3091": 0.8571428571428571, "3103": 1.0, "3115": 0.5, "3125": 1.0, "3148": 0.6, "3149": 0.0, "3177": 0.6666666666666666, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.6666666666666666, "3264": 0.75, "3357": 1.0, "3369": 0.16666666666666666, "3394": 0.75, "3404": 0.42857142857142855, "3405": 1.0, "3446": 0.8, "3451": 0.42857142857142855, "3453": 1.0, "3480": 1.0, "3490": 1.0, "3500": 0.5, "3503": 0.7142857142857143, "3512": 1.0, "3528": 1.0, "3530": 0.6666666666666666, "3534": 0.0, "3566": 1.0, "3569": 1.0, "3594": 0.25, "3612": 0.75, "3615": 1.0, "3625": 0.3333333333333333, "3682": 1.0, "3683": 0.6666666666666666, "3694": 0.5, "3724": 0.0, "3735": 1.0, "3759": 0.0, "3767": 0.4, "3771": 0.3333333333333333, "3781": 1.0, "3789": 0.75, "3791": 0.6666666666666666, "3801": 0.0, "3822": 0.25, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 0.6666666666666666, "4011": 1.0, "4019": 0.4, "4031": 0.0, "4037": 0.5, "4047": 0.75, "4071": 0.0, "4084": 1.0, "4102": 0.25, "4103": 0.0, "4105": 0.0, "4116": 0.5, "4125": 0.0, "4142": 0.75, "4153": 1.0, "4179": 1.0, "4188": 1.0, "4205": 0.6666666666666666, "4233": 0.8, "4265": 0.6666666666666666, "4286": 0.5, "4289": 0.5, "4306": 0.75, "4312": 0.8571428571428571, "4335": 0.75, "4339": 1.0, "4394": 0.0, "4409": 0.2, "4411": 0.6666666666666666, "4414": 1.0, "4415": 0.0, "4433": 0.5, "4447": 1.0, "4464": 1.0, "4465": 0.6666666666666666, "4484": 1.0, "4499": 0.3333333333333333, "4500": 0.5, "4504": 1.0, "4514": 0.5, "4523": 0.2, "4539": 0.5, "4571": 0.6666666666666666, "4600": 1.0, "4605": 0.7142857142857143, "4615": 0.6666666666666666, "4640": 0.0, "4641": 0.6, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 0.6666666666666666, "4846": 0.0, "4863": 1.0, "4865": 0.0, "4920": 1.0, "4942": 1.0, "4946": 1.0, "4955": 0.0, "4962": 0.5, "4968": 1.0, "4981": 0.6666666666666666, "4999": 0.75, "5021": 0.0, "5030": 0.0, "5045": 1.0, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 1.0, "5080": 0.0, "5083": 0.5, "5085": 1.0, "5086": 1.0, "5090": 0.5, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 1.0, "5178": 0.4, "5185": 0.0, "5196": 0.3333333333333333, "5206": 0.0, "5228": 1.0, "5231": 1.0, "5241": 0.4, "5254": 0.5, "5255": 1.0, "5264": 0.6, "5271": 1.0, "5331": 1.0, "5343": 1.0, "5347": 1.0, "5356": 0.4, "5369": 0.0, "5374": 1.0, "5380": 1.0, "5402": 0.5, "5410": 0.3333333333333333, "5422": 0.75, "5427": 0.5, "5460": 0.0, "5464": 0.3333333333333333, "5503": 0.5, "5505": 1.0, "5511": 0.3, "5534": 0.5, "5549": 0.0, "5585": 1.0, "5592": 1.0, "5616": 1.0, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.5, "5763": 0.6666666666666666, "5782": 0.4, "5790": 0.0, "5808": 1.0, "5853": 0.375, "5862": 0.5, "5888": 0.5, "5903": 0.0, "5906": 1.0, "5940": 0.0, "5951": 0.5555555555555556, "5970": 0.6666666666666666, "5981": 1.0, "5993": 0.2, "6002": 0.9090909090909091, "6004": 0.0, "6005": 0.38461538461538464, "6009": 1.0, "6041": 0.75, "6080": 0.5, "6110": 0.375, "6121": 0.5, "6122": 0.3333333333333333, "6131": 0.4166666666666667, "6133": 0.5, "6142": 0.5, "6146": 1.0, "6199": 0.5, "6219": 1.0, "6221": 0.125, "6252": 0.0, "6262": 0.6666666666666666, "6278": 1.0, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 1.0, "6467": 0.0, "6468": 0.5, "6479": 1.0, "6525": 0.6666666666666666, "6554": 0.5, "6562": 0.5, "6611": 1.0, "6612": 0.3333333333333333, "6625": 1.0, "6629": 0.0, "6635": 0.5, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 1.0, "6683": 1.0, "6713": 0.5, "6715": 1.0, "6746": 0.5, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.75, "6807": 1.0, "6814": 0.0, "6832": 1.0, "6835": 0.5, "6849": 1.0, "6862": 1.0, "6867": 0.42857142857142855, "6875": 0.0, "6890": 1.0, "6891": 0.6666666666666666, "6896": 0.0, "6901": 0.5, "6907": 0.0, "6909": 0.0, "6959": 1.0, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 1.0, "7109": 0.6666666666666666, "7124": 0.75, "7141": 0.5, "7145": 0.3333333333333333, "7178": 1.0, "7188": 1.0, "7205": 1.0, "7206": 0.0, "7218": 1.0, "7221": 0.0, "7269": 1.0, "7279": 0.75, "7295": 0.0, "7311": 1.0, "7326": 0.0, "7329": 1.0, "7344": 0.5, "7345": 0.6666666666666666, "7377": 1.0, "7431": 0.0, "7441": 0.3333333333333333, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.4, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 1.0, "7513": 0.6666666666666666, "7529": 0.5, "7533": 0.5, "7534": 0.3333333333333333, "7590": 1.0, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 0.5, "7674": 0.3333333333333333, "7700": 0.25, "7702": 1.0, "7705": 0.6666666666666666, "7734": 0.5, "7747": 0.5, "7754": 1.0, "7758": 1.0, "7801": 1.0, "7803": 0.5, "7823": 0.6666666666666666, "7876": 1.0, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.16666666666666666, "7928": 0.25, "7936": 0.6666666666666666, "7992": 1.0, "8002": 0.6666666666666666, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 1.0, "8040": 1.0, "8072": 0.6666666666666666, "8079": 1.0, "8102": 0.25, "8116": 1.0, "8121": 1.0, "8202": 0.0, "8230": 1.0, "8247": 0.0, "8271": 0.5, "8275": 1.0, "8296": 1.0, "8332": 1.0, "8351": 0.5, "8378": 0.3333333333333333, "8456": 0.6666666666666666, "8475": 0.8571428571428571, "8507": 0.3333333333333333, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 1.0, "8539": 0.25, "8544": 0.5, "8592": 0.6666666666666666, "8632": 1.0, "8635": 0.0, "8702": 0.6666666666666666, "8779": 1.0, "8789": 0.6666666666666666, "8795": 1.0, "8832": 1.0, "8834": 0.5, "8855": 0.0, "8874": 1.0, "8934": 1.0, "8937": 1.0, "8947": 0.5, "8959": 1.0, "8970": 1.0, "8974": 0.25, "8982": 0.0, "9060": 0.5, "9088": 0.0, "9108": 0.0, "9115": 1.0, "9126": 0.0, "9164": 0.3333333333333333, "9174": 0.2, "9188": 0.5, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.5, "9329": 1.0, "9332": 1.0, "9381": 1.0, "9385": 1.0, "9391": 0.75, "9403": 0.0, "9481": 1.0, "9487": 1.0, "9548": 1.0, "9556": 1.0, "9565": 1.0, "9598": 1.0, "9617": 0.6666666666666666, "9633": 0.5, "9643": 1.0, "9644": 1.0, "9646": 1.0, "9668": 1.0, "9701": 0.0, "9733": 0.3333333333333333, "9735": 1.0, "9737": 0.6666666666666666, "9771": 0.0, "9808": 0.3333333333333333, "9824": 1.0, "9871": 0.25, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 0.0, "9979": 0.5, "10034": 0.5, "10039": 1.0, "10109": 0.42857142857142855, "10122": 0.5, "10136": 0.6666666666666666, "10137": 1.0, "10152": 1.0, "10183": 1.0, "10213": 0.3333333333333333, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.3333333333333333, "10462": 0.0, "10482": 0.0, "10497": 0.7, "10526": 0.5, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 1.0, "10639": 0.4, "10645": 1.0, "10674": 1.0, "10710": 1.0, "10734": 1.0, "10792": 1.0, "10808": 1.0, "10809": 0.6666666666666666, "10812": 1.0, "10827": 0.2, "10845": 1.0, "10912": 1.0, "10932": 1.0, "10975": 0.0, "10979": 0.75, "10994": 1.0, "11039": 0.5, "11054": 1.0, "11088": 1.0}, "MRR@10": {"8": 1.0, "15": 1.0, "18": 0.5, "26": 0.3333333333333333, "34": 0.0, "42": 1.0, "56": 0.5, "68": 0.0, "89": 0.25, "90": 1.0, "94": 1.0, "98": 1.0, "104": 1.0, "106": 0.5, "109": 1.0, "475": 1.0, "503": 0.08333333333333333, "504": 0.05555555555555555, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 0.06666666666666667, "585": 0.5, "588": 0.25, "594": 1.0, "603": 0.0, "604": 0.2, "620": 0.1111111111111111, "622": 1.0, "659": 1.0, "672": 0.1111111111111111, "684": 0.0625, "687": 0.16666666666666666, "689": 0.0, "691": 0.25, "699": 0.5, "701": 0.1, "715": 0.0, "721": 1.0, "744": 0.3333333333333333, "750": 1.0, "753": 0.5, "766": 0.25, "776": 1.0, "810": 0.25, "813": 0.2, "849": 0.1, "852": 1.0, "853": 0.16666666666666666, "858": 0.0, "859": 1.0, "864": 1.0, "879": 0.5, "885": 0.2, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.2, "939": 1.0, "945": 0.5, "957": 1.0, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.3333333333333333, "1150": 0.0, "1157": 0.3333333333333333, "1159": 0.0, "1198": 0.5, "1230": 0.07692307692307693, "1281": 1.0, "1284": 1.0, "1297": 0.3333333333333333, "1306": 0.0, "1309": 1.0, "1310": 1.0, "1321": 0.0, "1322": 1.0, "1391": 0.25, "1393": 1.0, "1415": 1.0, "1416": 0.058823529411764705, "1441": 1.0, "1451": 1.0, "1469": 0.5, "1530": 1.0, "1670": 1.0, "1676": 0.3333333333333333, "1736": 1.0, "1748": 0.5, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 1.0, "1824": 0.25, "1826": 1.0, "1832": 0.16666666666666666, "1871": 1.0, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 0.25, "1994": 0.058823529411764705, "2010": 1.0, "2051": 0.07692307692307693, "2070": 0.09090909090909091, "2075": 1.0, "2076": 1.0, "2088": 0.0, "2108": 0.25, "2118": 0.0, "2154": 0.0, "2181": 0.058823529411764705, "2183": 1.0, "2204": 1.0, "2264": 0.1111111111111111, "2296": 1.0, "2306": 0.5, "2316": 1.0, "2318": 0.5, "2330": 1.0, "2334": 1.0, "2348": 1.0, "2376": 1.0, "2383": 1.0, "2384": 1.0, "2385": 1.0, "2388": 1.0, "2395": 0.3333333333333333, "2398": 0.125, "2399": 1.0, "2400": 1.0, "2407": 0.2, "2416": 1.0, "2423": 1.0, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.06666666666666667, "2472": 0.0, "2486": 1.0, "2498": 0.3333333333333333, "2513": 0.0625, "2516": 1.0, "2549": 1.0, "2551": 1.0, "2568": 1.0, "2579": 0.2, "2580": 1.0, "2587": 1.0, "2589": 1.0, "2590": 1.0, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 1.0, "2685": 1.0, "2695": 1.0, "2713": 0.08333333333333333, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 1.0, "2790": 1.0, "2801": 1.0, "2856": 0.5, "2857": 0.2, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.16666666666666666, "2923": 1.0, "2964": 0.09090909090909091, "2968": 1.0, "2994": 1.0, "3006": 0.3333333333333333, "3008": 1.0, "3014": 1.0, "3033": 1.0, "3039": 0.14285714285714285, "3049": 1.0, "3051": 1.0, "3067": 0.0, "3085": 0.16666666666666666, "3091": 1.0, "3103": 1.0, "3115": 1.0, "3125": 1.0, "3148": 1.0, "3149": 0.0, "3177": 0.1111111111111111, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 1.0, "3264": 1.0, "3357": 0.5, "3369": 0.1111111111111111, "3394": 1.0, "3404": 1.0, "3405": 1.0, "3446": 0.5, "3451": 0.1, "3453": 0.5, "3480": 1.0, "3490": 0.14285714285714285, "3500": 1.0, "3503": 1.0, "3512": 1.0, "3528": 1.0, "3530": 1.0, "3534": 0.0, "3566": 1.0, "3569": 0.07692307692307693, "3594": 0.16666666666666666, "3612": 0.1111111111111111, "3615": 0.5, "3625": 1.0, "3682": 0.5, "3683": 1.0, "3694": 1.0, "3724": 0.0, "3735": 0.25, "3759": 0.0, "3767": 1.0, "3771": 1.0, "3781": 1.0, "3789": 1.0, "3791": 0.5, "3801": 0.0, "3822": 0.05263157894736842, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 1.0, "3932": 0.0, "3934": 0.1111111111111111, "3995": 0.0, "4007": 1.0, "4011": 1.0, "4019": 1.0, "4031": 0.0, "4037": 0.16666666666666666, "4047": 1.0, "4071": 0.0, "4084": 1.0, "4102": 0.06666666666666667, "4103": 0.0, "4105": 0.0, "4116": 1.0, "4125": 0.0, "4142": 0.3333333333333333, "4153": 0.2, "4179": 0.09090909090909091, "4188": 0.06666666666666667, "4205": 0.25, "4233": 1.0, "4265": 1.0, "4286": 1.0, "4289": 1.0, "4306": 1.0, "4312": 1.0, "4335": 1.0, "4339": 1.0, "4394": 0.0, "4409": 1.0, "4411": 1.0, "4414": 1.0, "4415": 0.0, "4433": 0.16666666666666666, "4447": 0.125, "4464": 1.0, "4465": 0.3333333333333333, "4484": 1.0, "4499": 0.05555555555555555, "4500": 0.07142857142857142, "4504": 1.0, "4514": 1.0, "4523": 0.5, "4539": 0.3333333333333333, "4571": 1.0, "4600": 1.0, "4605": 1.0, "4615": 1.0, "4640": 0.0, "4641": 1.0, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 1.0, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.3333333333333333, "4813": 1.0, "4823": 1.0, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 1.0, "4846": 0.0, "4863": 1.0, "4865": 0.0, "4920": 1.0, "4942": 0.14285714285714285, "4946": 1.0, "4955": 0.0, "4962": 1.0, "4968": 0.2, "4981": 1.0, "4999": 1.0, "5021": 0.0, "5030": 0.0, "5045": 0.125, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.14285714285714285, "5080": 0.0, "5083": 1.0, "5085": 1.0, "5086": 1.0, "5090": 1.0, "5125": 1.0, "5134": 1.0, "5150": 0.0, "5155": 0.3333333333333333, "5172": 0.1111111111111111, "5178": 1.0, "5185": 0.0, "5196": 1.0, "5206": 0.0, "5228": 0.3333333333333333, "5231": 0.058823529411764705, "5241": 1.0, "5254": 1.0, "5255": 1.0, "5264": 1.0, "5271": 0.3333333333333333, "5331": 1.0, "5343": 0.14285714285714285, "5347": 1.0, "5356": 0.25, "5369": 0.0, "5374": 0.125, "5380": 1.0, "5402": 0.0625, "5410": 1.0, "5422": 0.5, "5427": 0.5, "5460": 0.0, "5464": 0.5, "5503": 1.0, "5505": 1.0, "5511": 0.25, "5534": 0.5, "5549": 0.0, "5585": 0.5, "5592": 1.0, "5616": 0.25, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 0.5, "5710": 0.0, "5741": 0.07692307692307693, "5763": 0.1, "5782": 1.0, "5790": 0.0, "5808": 0.3333333333333333, "5853": 0.2, "5862": 0.3333333333333333, "5888": 0.125, "5903": 0.0, "5906": 0.16666666666666666, "5940": 0.0, "5951": 0.3333333333333333, "5970": 0.5, "5981": 1.0, "5993": 1.0, "6002": 1.0, "6004": 0.0, "6005": 1.0, "6009": 0.125, "6041": 1.0, "6080": 1.0, "6110": 0.5, "6121": 1.0, "6122": 1.0, "6131": 1.0, "6133": 0.5, "6142": 1.0, "6146": 1.0, "6199": 0.5, "6219": 0.5, "6221": 0.05263157894736842, "6252": 0.0, "6262": 1.0, "6278": 0.1111111111111111, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.1111111111111111, "6467": 0.0, "6468": 1.0, "6479": 1.0, "6525": 0.5, "6554": 0.1, "6562": 0.125, "6611": 1.0, "6612": 1.0, "6625": 0.125, "6629": 0.0, "6635": 1.0, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 0.5, "6683": 0.3333333333333333, "6713": 1.0, "6715": 0.3333333333333333, "6746": 1.0, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 1.0, "6807": 1.0, "6814": 0.0, "6832": 0.5, "6835": 0.5, "6849": 0.14285714285714285, "6862": 1.0, "6867": 0.3333333333333333, "6875": 0.0, "6890": 1.0, "6891": 1.0, "6896": 0.0, "6901": 0.3333333333333333, "6907": 0.0, "6909": 0.0, "6959": 0.5, "6985": 1.0, "7017": 0.0, "7068": 0.3333333333333333, "7071": 0.0, "7080": 0.5, "7096": 0.5, "7098": 0.5, "7105": 0.3333333333333333, "7109": 0.14285714285714285, "7124": 1.0, "7141": 0.0625, "7145": 0.25, "7178": 1.0, "7188": 0.2, "7205": 1.0, "7206": 0.0, "7218": 0.25, "7221": 0.0, "7269": 0.09090909090909091, "7279": 0.3333333333333333, "7295": 0.0, "7311": 0.1, "7326": 0.0, "7329": 0.5, "7344": 1.0, "7345": 0.14285714285714285, "7377": 1.0, "7431": 0.0, "7441": 0.2, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.3333333333333333, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 0.0625, "7513": 0.25, "7529": 1.0, "7533": 0.1, "7534": 1.0, "7590": 1.0, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 1.0, "7674": 0.0625, "7700": 0.16666666666666666, "7702": 0.5, "7705": 1.0, "7734": 1.0, "7747": 0.1, "7754": 0.5, "7758": 1.0, "7801": 1.0, "7803": 0.08333333333333333, "7823": 1.0, "7876": 0.3333333333333333, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 1.0, "7928": 0.05263157894736842, "7936": 1.0, "7992": 0.2, "8002": 1.0, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.14285714285714285, "8040": 1.0, "8072": 1.0, "8079": 1.0, "8102": 0.14285714285714285, "8116": 0.3333333333333333, "8121": 1.0, "8202": 0.0, "8230": 0.0625, "8247": 0.0, "8271": 1.0, "8275": 0.05555555555555555, "8296": 1.0, "8332": 1.0, "8351": 1.0, "8378": 0.16666666666666666, "8456": 1.0, "8475": 1.0, "8507": 0.09090909090909091, "8512": 0.5, "8513": 0.0, "8532": 1.0, "8537": 0.3333333333333333, "8539": 0.1, "8544": 1.0, "8592": 0.25, "8632": 0.3333333333333333, "8635": 0.0, "8702": 1.0, "8779": 0.25, "8789": 1.0, "8795": 1.0, "8832": 1.0, "8834": 1.0, "8855": 0.0, "8874": 0.07142857142857142, "8934": 1.0, "8937": 0.5, "8947": 1.0, "8959": 0.1111111111111111, "8970": 0.5, "8974": 1.0, "8982": 0.0, "9060": 0.16666666666666666, "9088": 0.0, "9108": 0.0, "9115": 0.5, "9126": 0.0, "9164": 0.125, "9174": 0.07692307692307693, "9188": 1.0, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.5, "9329": 1.0, "9332": 0.25, "9381": 0.5, "9385": 1.0, "9391": 1.0, "9403": 0.0, "9481": 0.3333333333333333, "9487": 0.2, "9548": 0.25, "9556": 1.0, "9565": 1.0, "9598": 0.3333333333333333, "9617": 1.0, "9633": 1.0, "9643": 0.3333333333333333, "9644": 0.25, "9646": 0.3333333333333333, "9668": 0.5, "9701": 0.0, "9733": 0.3333333333333333, "9735": 0.09090909090909091, "9737": 1.0, "9771": 0.0, "9808": 0.1111111111111111, "9824": 1.0, "9871": 0.05555555555555555, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 0.0, "9979": 1.0, "10034": 1.0, "10039": 1.0, "10109": 1.0, "10122": 0.07692307692307693, "10136": 0.3333333333333333, "10137": 0.2, "10152": 1.0, "10183": 0.3333333333333333, "10213": 0.14285714285714285, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.16666666666666666, "10462": 0.0, "10482": 0.0, "10497": 1.0, "10526": 0.25, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 0.3333333333333333, "10628": 0.16666666666666666, "10639": 0.5, "10645": 0.3333333333333333, "10674": 1.0, "10710": 0.3333333333333333, "10734": 1.0, "10792": 1.0, "10808": 0.3333333333333333, "10809": 0.5, "10812": 1.0, "10827": 1.0, "10845": 1.0, "10912": 0.1111111111111111, "10932": 1.0, "10975": 0.0, "10979": 1.0, "10994": 0.3333333333333333, "11039": 0.5, "11054": 1.0, "11088": 0.3333333333333333}}} diff --git a/evals/benchmark/results/fiqa/ir-w0.25.jsonl b/evals/benchmark/results/fiqa/ir-w0.25.jsonl new file mode 100644 index 000000000..51cdab0c1 --- /dev/null +++ b/evals/benchmark/results/fiqa/ir-w0.25.jsonl @@ -0,0 +1 @@ +{"dataset": "fiqa", "run_tag": "w0.25", "aggregated": {"nDCG@10": 0.4363, "Recall@20": 0.6186, "MRR@10": 0.5093}, "per_query": {"nDCG@10": {"8": 0.6131471927654584, "15": 0.3562071871080222, "18": 1.0, "26": 0.23719771276929622, "34": 0.0, "42": 0.46927872602275644, "56": 0.38685280723454163, "68": 0.0, "89": 0.286381299268402, "90": 0.6309297535714575, "94": 1.0, "98": 0.6131471927654584, "104": 0.6131471927654584, "106": 0.5, "109": 1.0, "475": 1.0, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.3562071871080222, "559": 0.0, "570": 0.0, "585": 0.2640681225725909, "588": 0.448643819352159, "594": 0.6131471927654584, "603": 0.0, "604": 0.3235866969737877, "620": 0.0, "622": 0.38685280723454163, "659": 0.44793214080335986, "672": 0.0, "684": 0.0, "687": 0.19342640361727081, "689": 0.0, "691": 0.6309297535714575, "699": 1.0, "701": 0.13565197343244778, "715": 0.0, "721": 1.0, "744": 0.18154179253735267, "750": 0.8503449055347546, "753": 0.38685280723454163, "766": 0.18154179253735267, "776": 0.3557772116892465, "810": 0.3333333333333333, "813": 0.20438239758848611, "849": 0.3333333333333333, "852": 0.7653606369886217, "853": 0.0, "858": 0.0, "859": 0.38685280723454163, "864": 0.7247145167543899, "879": 0.6309297535714575, "885": 0.19342640361727081, "895": 1.0, "904": 1.0, "928": 0.6131471927654584, "929": 1.0, "932": 0.3065735963827292, "939": 1.0, "945": 0.31546487678572877, "957": 0.7653606369886217, "988": 0.0, "1074": 0.6309297535714575, "1085": 0.0, "1090": 0.23719771276929622, "1150": 0.0, "1157": 0.6131471927654584, "1159": 0.0, "1198": 1.0, "1230": 0.0, "1281": 0.38685280723454163, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.7030861797445797, "1310": 1.0, "1321": 0.0, "1322": 0.24630238874073, "1391": 0.18154179253735267, "1393": 0.30260241349881345, "1415": 0.9438661545147249, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.431733884398313, "1670": 0.7653606369886217, "1676": 0.3853585682515056, "1736": 0.5531464700081437, "1748": 0.6049306994552042, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.286381299268402, "1824": 0.31546487678572877, "1826": 1.0, "1832": 0.5, "1871": 0.6984152163370878, "1877": 1.0, "1889": 0.0, "1915": 0.8772153153380493, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.21398626473452756, "2010": 0.8519590445170673, "2051": 0.0, "2070": 0.16716045496620227, "2075": 0.40129869165981646, "2076": 0.5706417189553201, "2088": 0.0, "2108": 0.16716045496620227, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 0.7526136409516659, "2204": 0.9830168297915423, "2264": 0.3065735963827292, "2296": 0.33627415762678553, "2306": 0.6131471927654584, "2316": 1.0, "2318": 0.31546487678572877, "2330": 0.6131471927654584, "2334": 0.46927872602275644, "2348": 0.43231813227099475, "2376": 0.9021484053648395, "2383": 0.6131471927654584, "2384": 1.0, "2385": 0.6131471927654584, "2388": 1.0, "2395": 1.0, "2398": 0.0, "2399": 1.0, "2400": 0.5307212739772434, "2407": 0.19519002499605084, "2416": 0.9058653015743079, "2423": 0.3576223542196109, "2443": 0.0, "2445": 1.0, "2460": 0.9469024295259745, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 0.43067655807339306, "2513": 0.0, "2516": 0.5585075862632192, "2549": 0.6131471927654584, "2551": 1.0, "2568": 0.25288473642100684, "2579": 0.28201326940239274, "2580": 0.46927872602275644, "2587": 1.0, "2589": 0.7653606369886217, "2590": 0.46927872602275644, "2593": 0.8854598815714874, "2598": 0.8503449055347546, "2648": 1.0, "2676": 0.6309297535714575, "2685": 0.35079429740281753, "2695": 1.0, "2713": 0.38685280723454163, "2724": 0.8315546295836225, "2737": 1.0, "2747": 0.5, "2749": 0.8772153153380493, "2790": 0.499028327865556, "2801": 0.6131471927654584, "2856": 0.7095272044910244, "2857": 0.5154852515822061, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.3010299956639812, "2923": 0.8207658584763088, "2964": 0.0, "2968": 0.8772153153380493, "2994": 0.46927872602275644, "3006": 0.25909036127391766, "3008": 0.24630238874073, "3014": 1.0, "3033": 0.6131471927654584, "3039": 0.38685280723454163, "3049": 0.24630238874073, "3051": 1.0, "3067": 0.0, "3085": 0.19342640361727081, "3091": 0.9020870746500321, "3103": 1.0, "3115": 0.4935232796777481, "3125": 1.0, "3148": 0.5531464700081437, "3149": 0.0, "3177": 0.15642624200758548, "3179": 0.8772153153380493, "3186": 1.0, "3189": 0.0, "3254": 0.6713860725233041, "3264": 0.8048099750039491, "3357": 1.0, "3369": 0.0, "3394": 0.5805485932475565, "3404": 0.3932591258696538, "3405": 0.2890648263178879, "3446": 0.46704774694449785, "3451": 0.07945707943276742, "3453": 0.6509209298071326, "3480": 1.0, "3490": 0.2890648263178879, "3500": 0.644824486427155, "3503": 0.6183313187362977, "3512": 0.8772153153380493, "3528": 0.6309297535714575, "3530": 0.4776237035032179, "3534": 0.0, "3566": 0.8194270280062366, "3569": 0.5, "3594": 0.123151194370365, "3612": 0.11751610475642714, "3615": 0.5, "3625": 0.20210734650054757, "3682": 0.21840743681816419, "3683": 0.6324677375730864, "3694": 0.6131471927654584, "3724": 0.0, "3735": 0.0, "3759": 0.0, "3767": 0.2716774977597197, "3771": 0.19632790425573848, "3781": 0.8772153153380493, "3789": 0.8048099750039491, "3791": 0.431733884398313, "3801": 0.0, "3822": 0.0, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5670429581667766, "3932": 0.0, "3934": 0.16812753627111746, "3995": 0.0, "4007": 0.7039180890341347, "4011": 0.9674679834891693, "4019": 0.31608365985743925, "4031": 0.0, "4037": 0.18457569677956817, "4047": 0.8318724637288826, "4071": 0.0, "4084": 0.5012658353418871, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 0.6131471927654584, "4125": 0.0, "4142": 0.39233405721184117, "4153": 0.31546487678572877, "4179": 0.6309297535714575, "4188": 0.0, "4205": 0.4525081529734507, "4233": 0.8687949224876582, "4265": 0.7751482523375255, "4286": 0.6131471927654584, "4289": 0.23719771276929622, "4306": 0.7598336331031966, "4312": 0.7237478018511487, "4335": 0.8318724637288826, "4339": 1.0, "4394": 0.0, "4409": 0.19518900079066107, "4411": 0.46927872602275644, "4414": 0.8315546295836225, "4415": 0.0, "4433": 0.0, "4447": 0.38685280723454163, "4464": 0.5, "4465": 0.2960819109658652, "4484": 0.6309297535714575, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 0.4034698846650846, "4523": 0.21398626473452756, "4539": 0.21840743681816419, "4571": 0.7653606369886217, "4600": 1.0, "4605": 0.5706291306523891, "4615": 0.4636246685132894, "4640": 0.0, "4641": 0.252016380211836, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75369761125927, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.6131471927654584, "4813": 1.0, "4823": 0.46927872602275644, "4827": 0.38685280723454163, "4837": 0.0, "4844": 1.0, "4845": 0.7653606369886217, "4846": 0.0, "4863": 0.5307212739772434, "4865": 0.0, "4920": 1.0, "4942": 0.31546487678572877, "4946": 1.0, "4955": 0.0, "4962": 0.6131471927654584, "4968": 0.6309297535714575, "4981": 0.7039180890341347, "4999": 0.5294362295028773, "5021": 0.0, "5030": 0.0, "5045": 0.3010299956639812, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.6309297535714575, "5080": 0.0, "5083": 0.23719771276929622, "5085": 1.0, "5086": 1.0, "5090": 0.2640681225725909, "5125": 1.0, "5134": 0.6131471927654584, "5150": 0.0, "5155": 1.0, "5172": 0.0, "5178": 0.252016380211836, "5185": 0.0, "5196": 0.2960819109658652, "5206": 0.0, "5228": 0.31546487678572877, "5231": 0.0, "5241": 0.44615333764087994, "5254": 0.6131471927654584, "5255": 1.0, "5264": 0.6843515475204855, "5271": 0.41618115554873086, "5331": 1.0, "5343": 0.3333333333333333, "5347": 1.0, "5356": 0.1208113026994942, "5369": 0.0, "5374": 0.3010299956639812, "5380": 1.0, "5402": 0.0, "5410": 0.2960819109658652, "5422": 0.3903800499921017, "5427": 0.2640681225725909, "5460": 0.0, "5464": 0.16716045496620227, "5503": 0.6131471927654584, "5505": 1.0, "5511": 0.1610425878239398, "5534": 0.38685280723454163, "5549": 0.0, "5585": 0.3333333333333333, "5592": 1.0, "5616": 0.3562071871080222, "5620": 0.0, "5646": 0.9217868789962071, "5653": 1.0, "5683": 0.6934264036172708, "5710": 0.0, "5741": 0.0, "5763": 0.18154179253735267, "5782": 0.4371994911049732, "5790": 0.0, "5808": 0.5, "5853": 0.21078558563971425, "5862": 0.29127873064148246, "5888": 0.23719771276929622, "5903": 0.0, "5906": 0.48381288316677695, "5940": 0.0, "5951": 0.5142521154617875, "5970": 0.23463936301137822, "5981": 0.9896062251871525, "5993": 0.30523488393970116, "6002": 0.7819835854675755, "6004": 0.0, "6005": 0.5388856921828065, "6009": 0.0, "6041": 0.7668091220635322, "6080": 0.6131471927654584, "6110": 0.32583242649464755, "6121": 0.38685280723454163, "6122": 0.46927872602275644, "6131": 0.5810820381115676, "6133": 0.6131471927654584, "6142": 0.6131471927654584, "6146": 0.8315546295836225, "6199": 0.5135312443624667, "6219": 0.6934264036172708, "6221": 0.0, "6252": 0.0, "6262": 0.7653606369886217, "6278": 0.20438239758848611, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.3562071871080222, "6467": 0.0, "6468": 0.6131471927654584, "6479": 1.0, "6525": 0.49818925746641285, "6554": 0.38685280723454163, "6562": 0.11751610475642714, "6611": 0.6131471927654584, "6612": 0.16716045496620227, "6625": 0.31546487678572877, "6629": 0.0, "6635": 0.6366824387328317, "6644": 0.0, "6647": 0.7653606369886217, "6668": 0.8503449055347546, "6679": 0.4632423659320675, "6683": 0.8772153153380493, "6713": 0.6131471927654584, "6715": 0.38685280723454163, "6746": 0.6131471927654584, "6787": 0.0, "6792": 0.0, "6800": 0.43067655807339306, "6803": 0.5205067333228022, "6807": 1.0, "6814": 0.0, "6832": 0.430624116386567, "6835": 0.38685280723454163, "6849": 0.38685280723454163, "6862": 1.0, "6867": 0.22471947398559278, "6875": 0.0, "6890": 0.5012658353418871, "6891": 0.617319681505689, "6896": 0.0, "6901": 0.21840743681816419, "6907": 0.0, "6909": 0.0, "6959": 0.38685280723454163, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 0.8772153153380493, "7096": 1.0, "7098": 0.6131471927654584, "7105": 0.38685280723454163, "7109": 0.23463936301137822, "7124": 0.6525874238732422, "7141": 0.0, "7145": 0.13565197343244778, "7178": 0.38685280723454163, "7188": 0.31546487678572877, "7205": 1.0, "7206": 0.0, "7218": 0.43067655807339306, "7221": 0.0, "7269": 0.0, "7279": 0.4693608375942211, "7295": 0.0, "7311": 0.0, "7326": 0.0, "7329": 0.5012658353418871, "7344": 0.6131471927654584, "7345": 0.15642624200758548, "7377": 1.0, "7431": 0.0, "7441": 0.23463936301137822, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.14606834984270645, "7467": 0.6309297535714575, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.16716045496620227, "7529": 0.6131471927654584, "7533": 0.0, "7534": 0.46927872602275644, "7590": 0.5802792108518124, "7592": 0.0, "7594": 0.3562071871080222, "7622": 0.0, "7633": 0.3903800499921017, "7674": 0.0, "7700": 0.123151194370365, "7702": 0.38685280723454163, "7705": 0.6508205185601091, "7734": 0.6131471927654584, "7747": 0.2640681225725909, "7754": 0.0, "7758": 0.6309297535714575, "7801": 1.0, "7803": 0.20438239758848611, "7823": 0.6105456988825855, "7876": 0.43067655807339306, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.19092086617893467, "7928": 0.19519002499605084, "7936": 0.4632423659320675, "7992": 0.38685280723454163, "8002": 0.6257049680303419, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.31546487678572877, "8040": 1.0, "8072": 0.6508205185601091, "8079": 0.7039180890341347, "8102": 0.11284514134893527, "8116": 0.45749452618986164, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 0.6131471927654584, "8275": 0.38685280723454163, "8296": 0.8772153153380493, "8332": 0.9674679834891693, "8351": 0.6131471927654584, "8378": 0.1480409554829326, "8456": 0.6049306994552042, "8475": 0.7957503936198023, "8507": 0.0, "8512": 1.0, "8513": 0.0, "8532": 0.9325210919548239, "8537": 0.3562071871080222, "8539": 0.0, "8544": 0.6131471927654584, "8592": 0.20210734650054757, "8632": 0.3065735963827292, "8635": 0.0, "8702": 0.7039180890341347, "8779": 0.0, "8789": 0.7653606369886217, "8795": 1.0, "8832": 1.0, "8834": 0.3980628465882808, "8855": 0.0, "8874": 0.2890648263178879, "8934": 0.8065735963827293, "8937": 0.3562071871080222, "8947": 0.23719771276929622, "8959": 0.38685280723454163, "8970": 0.6509209298071326, "8974": 0.41253177989255346, "8982": 0.0, "9060": 0.3065735963827292, "9088": 0.0, "9108": 0.0, "9115": 0.5256940434743352, "9126": 0.0, "9164": 0.15642624200758548, "9174": 0.0, "9188": 0.6131471927654584, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.20438239758848611, "9329": 0.9197207891481876, "9332": 0.43067655807339306, "9381": 0.6934264036172708, "9385": 0.3562071871080222, "9391": 0.5143371794949736, "9403": 0.0, "9481": 0.5706417189553201, "9487": 0.0, "9548": 0.5, "9556": 1.0, "9565": 1.0, "9598": 0.3562071871080222, "9617": 0.3152014104491349, "9633": 0.6131471927654584, "9643": 0.5, "9644": 0.5, "9646": 0.5249810332008933, "9668": 0.6508205185601091, "9701": 0.0, "9733": 0.18154179253735267, "9735": 0.3333333333333333, "9737": 0.7653606369886217, "9771": 0.0, "9808": 0.2960819109658652, "9824": 1.0, "9871": 0.24630238874073, "9882": 0.0, "9925": 0.8175295903539445, "9929": 0.7977228895450266, "9961": 0.0, "9979": 0.3903800499921017, "10034": 0.38685280723454163, "10039": 0.6309297535714575, "10109": 0.5039399436761999, "10122": 0.19519002499605084, "10136": 0.34337431936037655, "10137": 0.3333333333333333, "10152": 1.0, "10183": 0.3065735963827292, "10213": 0.46927872602275644, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.18154179253735267, "10462": 0.0, "10482": 0.0, "10497": 0.4818998405251556, "10526": 0.20438239758848611, "10547": 0.0, "10558": 0.0, "10596": 0.9427731067700877, "10601": 0.6309297535714575, "10628": 0.3010299956639812, "10639": 0.3120255505658846, "10645": 0.38685280723454163, "10674": 0.7095272044910244, "10710": 0.31546487678572877, "10734": 0.8696762340896295, "10792": 0.8315546295836225, "10808": 0.4652347214527331, "10809": 0.6049306994552042, "10812": 0.9493885684853097, "10827": 0.14606834984270645, "10845": 1.0, "10912": 0.0, "10932": 1.0, "10975": 0.0, "10979": 0.6366824387328317, "10994": 0.5, "11039": 0.3200466520981656, "11054": 1.0, "11088": 0.43067655807339306}, "Recall@20": {"8": 0.5, "15": 1.0, "18": 1.0, "26": 0.5, "34": 0.0, "42": 0.3333333333333333, "56": 1.0, "68": 0.0, "89": 0.5, "90": 1.0, "94": 1.0, "98": 0.5, "104": 0.5, "106": 1.0, "109": 1.0, "475": 1.0, "503": 1.0, "504": 0.3333333333333333, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 1.0, "585": 0.5, "588": 1.0, "594": 1.0, "603": 0.0, "604": 1.0, "620": 0.2, "622": 0.5, "659": 0.7, "672": 0.5, "684": 1.0, "687": 0.5, "689": 0.0, "691": 1.0, "699": 1.0, "701": 0.3333333333333333, "715": 0.0, "721": 1.0, "744": 0.3333333333333333, "750": 1.0, "753": 0.5, "766": 0.3333333333333333, "776": 0.25, "810": 1.0, "813": 1.0, "849": 1.0, "852": 0.6666666666666666, "853": 0.5, "858": 0.0, "859": 0.5, "864": 0.6666666666666666, "879": 1.0, "885": 0.5, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.5, "939": 1.0, "945": 1.0, "957": 0.6666666666666666, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.5, "1150": 0.0, "1157": 0.5, "1159": 0.0, "1198": 1.0, "1230": 1.0, "1281": 1.0, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.75, "1310": 1.0, "1321": 0.0, "1322": 0.25, "1391": 0.6666666666666666, "1393": 0.5, "1415": 1.0, "1416": 1.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 1.0, "1670": 0.6666666666666666, "1676": 1.0, "1736": 0.4, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 1.0, "1826": 1.0, "1832": 1.0, "1871": 0.75, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.4, "2010": 1.0, "2051": 1.0, "2070": 0.6666666666666666, "2075": 0.6666666666666666, "2076": 1.0, "2088": 0.0, "2108": 0.3333333333333333, "2118": 0.0, "2154": 0.0, "2181": 0.5, "2183": 0.8333333333333334, "2204": 1.0, "2264": 0.5, "2296": 0.2222222222222222, "2306": 0.5, "2316": 1.0, "2318": 1.0, "2330": 0.5, "2334": 0.6666666666666666, "2348": 0.26666666666666666, "2376": 1.0, "2383": 0.5, "2384": 1.0, "2385": 0.5, "2388": 1.0, "2395": 1.0, "2398": 0.2, "2399": 1.0, "2400": 0.6666666666666666, "2407": 0.5, "2416": 1.0, "2423": 0.8571428571428571, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.3333333333333333, "2472": 0.0, "2486": 1.0, "2498": 1.0, "2513": 1.0, "2516": 0.75, "2549": 0.5, "2551": 1.0, "2568": 0.2857142857142857, "2579": 0.5, "2580": 0.3333333333333333, "2587": 1.0, "2589": 0.6666666666666666, "2590": 0.6666666666666666, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 1.0, "2685": 0.25, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 1.0, "2790": 0.42857142857142855, "2801": 1.0, "2856": 0.75, "2857": 0.75, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 1.0, "2923": 0.8, "2964": 0.5, "2968": 1.0, "2994": 0.3333333333333333, "3006": 0.5, "3008": 0.25, "3014": 1.0, "3033": 0.5, "3039": 1.0, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 1.0, "3091": 0.8571428571428571, "3103": 1.0, "3115": 0.5, "3125": 1.0, "3148": 0.6, "3149": 0.0, "3177": 0.6666666666666666, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.6666666666666666, "3264": 0.75, "3357": 1.0, "3369": 0.16666666666666666, "3394": 0.75, "3404": 0.42857142857142855, "3405": 1.0, "3446": 0.8, "3451": 0.42857142857142855, "3453": 1.0, "3480": 1.0, "3490": 1.0, "3500": 0.5, "3503": 0.7142857142857143, "3512": 1.0, "3528": 1.0, "3530": 0.6666666666666666, "3534": 0.0, "3566": 1.0, "3569": 1.0, "3594": 0.25, "3612": 0.75, "3615": 1.0, "3625": 0.3333333333333333, "3682": 1.0, "3683": 0.6666666666666666, "3694": 0.5, "3724": 0.0, "3735": 1.0, "3759": 0.0, "3767": 0.4, "3771": 0.3333333333333333, "3781": 1.0, "3789": 0.75, "3791": 0.6666666666666666, "3801": 0.0, "3822": 0.25, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 0.6666666666666666, "4011": 1.0, "4019": 0.4, "4031": 0.0, "4037": 0.5, "4047": 0.75, "4071": 0.0, "4084": 1.0, "4102": 0.25, "4103": 0.0, "4105": 0.0, "4116": 0.5, "4125": 0.0, "4142": 0.75, "4153": 1.0, "4179": 1.0, "4188": 1.0, "4205": 0.6666666666666666, "4233": 0.8, "4265": 0.6666666666666666, "4286": 0.5, "4289": 0.5, "4306": 0.75, "4312": 0.8571428571428571, "4335": 0.75, "4339": 1.0, "4394": 0.0, "4409": 0.2, "4411": 0.6666666666666666, "4414": 1.0, "4415": 0.0, "4433": 0.5, "4447": 1.0, "4464": 1.0, "4465": 0.6666666666666666, "4484": 1.0, "4499": 0.3333333333333333, "4500": 0.5, "4504": 1.0, "4514": 0.5, "4523": 0.2, "4539": 0.5, "4571": 0.6666666666666666, "4600": 1.0, "4605": 0.7142857142857143, "4615": 0.6666666666666666, "4640": 0.0, "4641": 0.6, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 0.6666666666666666, "4846": 0.0, "4863": 1.0, "4865": 0.0, "4920": 1.0, "4942": 1.0, "4946": 1.0, "4955": 0.0, "4962": 0.5, "4968": 1.0, "4981": 0.6666666666666666, "4999": 0.75, "5021": 0.0, "5030": 0.0, "5045": 1.0, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 1.0, "5080": 0.0, "5083": 0.5, "5085": 1.0, "5086": 1.0, "5090": 0.5, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 1.0, "5178": 0.4, "5185": 0.0, "5196": 0.3333333333333333, "5206": 0.0, "5228": 1.0, "5231": 1.0, "5241": 0.4, "5254": 0.5, "5255": 1.0, "5264": 0.6, "5271": 1.0, "5331": 1.0, "5343": 1.0, "5347": 1.0, "5356": 0.4, "5369": 0.0, "5374": 1.0, "5380": 1.0, "5402": 0.5, "5410": 0.3333333333333333, "5422": 0.75, "5427": 0.5, "5460": 0.0, "5464": 0.3333333333333333, "5503": 0.5, "5505": 1.0, "5511": 0.3, "5534": 0.5, "5549": 0.0, "5585": 1.0, "5592": 1.0, "5616": 1.0, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.5, "5763": 0.6666666666666666, "5782": 0.4, "5790": 0.0, "5808": 1.0, "5853": 0.375, "5862": 0.5, "5888": 0.5, "5903": 0.0, "5906": 1.0, "5940": 0.0, "5951": 0.5555555555555556, "5970": 0.6666666666666666, "5981": 1.0, "5993": 0.2, "6002": 0.9090909090909091, "6004": 0.0, "6005": 0.38461538461538464, "6009": 1.0, "6041": 0.75, "6080": 0.5, "6110": 0.375, "6121": 0.5, "6122": 0.3333333333333333, "6131": 0.4166666666666667, "6133": 0.5, "6142": 0.5, "6146": 1.0, "6199": 0.5, "6219": 1.0, "6221": 0.125, "6252": 0.0, "6262": 0.6666666666666666, "6278": 1.0, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 1.0, "6467": 0.0, "6468": 0.5, "6479": 1.0, "6525": 0.6666666666666666, "6554": 0.5, "6562": 0.5, "6611": 1.0, "6612": 0.3333333333333333, "6625": 1.0, "6629": 0.0, "6635": 0.5, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 1.0, "6683": 1.0, "6713": 0.5, "6715": 1.0, "6746": 0.5, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.75, "6807": 1.0, "6814": 0.0, "6832": 1.0, "6835": 0.5, "6849": 1.0, "6862": 1.0, "6867": 0.42857142857142855, "6875": 0.0, "6890": 1.0, "6891": 0.6666666666666666, "6896": 0.0, "6901": 0.5, "6907": 0.0, "6909": 0.0, "6959": 1.0, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 1.0, "7109": 0.6666666666666666, "7124": 0.75, "7141": 0.5, "7145": 0.3333333333333333, "7178": 1.0, "7188": 1.0, "7205": 1.0, "7206": 0.0, "7218": 1.0, "7221": 0.0, "7269": 1.0, "7279": 0.75, "7295": 0.0, "7311": 1.0, "7326": 0.0, "7329": 1.0, "7344": 0.5, "7345": 0.6666666666666666, "7377": 1.0, "7431": 0.0, "7441": 0.3333333333333333, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.4, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 1.0, "7513": 0.6666666666666666, "7529": 0.5, "7533": 0.5, "7534": 0.3333333333333333, "7590": 1.0, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 0.5, "7674": 0.3333333333333333, "7700": 0.25, "7702": 1.0, "7705": 0.6666666666666666, "7734": 0.5, "7747": 0.5, "7754": 1.0, "7758": 1.0, "7801": 1.0, "7803": 0.5, "7823": 0.6666666666666666, "7876": 1.0, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.16666666666666666, "7928": 0.25, "7936": 0.6666666666666666, "7992": 1.0, "8002": 0.6666666666666666, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 1.0, "8040": 1.0, "8072": 0.6666666666666666, "8079": 1.0, "8102": 0.25, "8116": 1.0, "8121": 1.0, "8202": 0.0, "8230": 1.0, "8247": 0.0, "8271": 0.5, "8275": 1.0, "8296": 1.0, "8332": 1.0, "8351": 0.5, "8378": 0.3333333333333333, "8456": 0.6666666666666666, "8475": 0.8571428571428571, "8507": 0.3333333333333333, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 1.0, "8539": 0.25, "8544": 0.5, "8592": 0.6666666666666666, "8632": 1.0, "8635": 0.0, "8702": 0.6666666666666666, "8779": 1.0, "8789": 0.6666666666666666, "8795": 1.0, "8832": 1.0, "8834": 0.5, "8855": 0.0, "8874": 1.0, "8934": 1.0, "8937": 1.0, "8947": 0.5, "8959": 1.0, "8970": 1.0, "8974": 0.25, "8982": 0.0, "9060": 0.5, "9088": 0.0, "9108": 0.0, "9115": 1.0, "9126": 0.0, "9164": 0.3333333333333333, "9174": 0.2, "9188": 0.5, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.5, "9329": 1.0, "9332": 1.0, "9381": 1.0, "9385": 1.0, "9391": 0.75, "9403": 0.0, "9481": 1.0, "9487": 1.0, "9548": 1.0, "9556": 1.0, "9565": 1.0, "9598": 1.0, "9617": 0.6666666666666666, "9633": 0.5, "9643": 1.0, "9644": 1.0, "9646": 1.0, "9668": 1.0, "9701": 0.0, "9733": 0.3333333333333333, "9735": 1.0, "9737": 0.6666666666666666, "9771": 0.0, "9808": 0.3333333333333333, "9824": 1.0, "9871": 0.25, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 0.0, "9979": 0.5, "10034": 0.5, "10039": 1.0, "10109": 0.42857142857142855, "10122": 0.5, "10136": 0.6666666666666666, "10137": 1.0, "10152": 1.0, "10183": 1.0, "10213": 0.3333333333333333, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.3333333333333333, "10462": 0.0, "10482": 0.0, "10497": 0.7, "10526": 0.5, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 1.0, "10639": 0.4, "10645": 1.0, "10674": 1.0, "10710": 1.0, "10734": 1.0, "10792": 1.0, "10808": 1.0, "10809": 0.6666666666666666, "10812": 1.0, "10827": 0.2, "10845": 1.0, "10912": 1.0, "10932": 1.0, "10975": 0.0, "10979": 0.75, "10994": 1.0, "11039": 0.5, "11054": 1.0, "11088": 1.0}, "MRR@10": {"8": 1.0, "15": 0.16666666666666666, "18": 1.0, "26": 0.2, "34": 0.0, "42": 1.0, "56": 0.2, "68": 0.0, "89": 0.5, "90": 0.5, "94": 1.0, "98": 1.0, "104": 1.0, "106": 0.3333333333333333, "109": 1.0, "475": 1.0, "503": 0.08333333333333333, "504": 0.05555555555555555, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.16666666666666666, "559": 0.0, "570": 0.06666666666666667, "585": 0.25, "588": 0.25, "594": 1.0, "603": 0.0, "604": 0.16666666666666666, "620": 0.08333333333333333, "622": 0.5, "659": 1.0, "672": 0.07692307692307693, "684": 0.05555555555555555, "687": 0.125, "689": 0.0, "691": 0.5, "699": 1.0, "701": 0.1, "715": 0.0, "721": 1.0, "744": 0.2, "750": 1.0, "753": 0.5, "766": 0.2, "776": 1.0, "810": 0.14285714285714285, "813": 0.14285714285714285, "849": 0.14285714285714285, "852": 1.0, "853": 0.09090909090909091, "858": 0.0, "859": 0.5, "864": 1.0, "879": 0.5, "885": 0.125, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.3333333333333333, "939": 1.0, "945": 0.125, "957": 1.0, "988": 0.0, "1074": 0.5, "1085": 0.0, "1090": 0.2, "1150": 0.0, "1157": 1.0, "1159": 0.0, "1198": 1.0, "1230": 0.07142857142857142, "1281": 0.2, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 1.0, "1310": 1.0, "1321": 0.0, "1322": 0.5, "1391": 0.2, "1393": 1.0, "1415": 1.0, "1416": 0.058823529411764705, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.5, "1670": 1.0, "1676": 0.5, "1736": 1.0, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 0.125, "1826": 1.0, "1832": 0.3333333333333333, "1871": 1.0, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.5, "2010": 1.0, "2051": 0.07142857142857142, "2070": 0.16666666666666666, "2075": 0.5, "2076": 0.3333333333333333, "2088": 0.0, "2108": 0.16666666666666666, "2118": 0.0, "2154": 0.0, "2181": 0.05263157894736842, "2183": 1.0, "2204": 1.0, "2264": 0.3333333333333333, "2296": 1.0, "2306": 1.0, "2316": 1.0, "2318": 0.125, "2330": 1.0, "2334": 1.0, "2348": 1.0, "2376": 1.0, "2383": 1.0, "2384": 1.0, "2385": 1.0, "2388": 1.0, "2395": 1.0, "2398": 0.08333333333333333, "2399": 1.0, "2400": 0.5, "2407": 0.3333333333333333, "2416": 1.0, "2423": 1.0, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0625, "2472": 0.0, "2486": 1.0, "2498": 0.25, "2513": 0.0625, "2516": 1.0, "2549": 1.0, "2551": 1.0, "2568": 0.5, "2579": 0.5, "2580": 1.0, "2587": 1.0, "2589": 1.0, "2590": 1.0, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 0.5, "2685": 1.0, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 0.3333333333333333, "2749": 1.0, "2790": 1.0, "2801": 1.0, "2856": 1.0, "2857": 0.5, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.1111111111111111, "2923": 1.0, "2964": 0.07692307692307693, "2968": 1.0, "2994": 1.0, "3006": 0.3333333333333333, "3008": 0.5, "3014": 1.0, "3033": 1.0, "3039": 0.2, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 0.125, "3091": 1.0, "3103": 1.0, "3115": 1.0, "3125": 1.0, "3148": 1.0, "3149": 0.0, "3177": 0.14285714285714285, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 1.0, "3264": 1.0, "3357": 1.0, "3369": 0.09090909090909091, "3394": 0.5, "3404": 1.0, "3405": 0.1, "3446": 0.5, "3451": 0.1, "3453": 0.5, "3480": 1.0, "3490": 0.1, "3500": 1.0, "3503": 0.5, "3512": 1.0, "3528": 0.5, "3530": 0.5, "3534": 0.0, "3566": 1.0, "3569": 0.3333333333333333, "3594": 0.125, "3612": 0.1111111111111111, "3615": 0.3333333333333333, "3625": 0.25, "3682": 0.16666666666666666, "3683": 1.0, "3694": 1.0, "3724": 0.0, "3735": 0.09090909090909091, "3759": 0.0, "3767": 0.3333333333333333, "3771": 0.14285714285714285, "3781": 1.0, "3789": 1.0, "3791": 0.5, "3801": 0.0, "3822": 0.05263157894736842, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 1.0, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 1.0, "4011": 1.0, "4019": 0.5, "4031": 0.0, "4037": 0.1111111111111111, "4047": 1.0, "4071": 0.0, "4084": 0.25, "4102": 0.06666666666666667, "4103": 0.0, "4105": 0.0, "4116": 1.0, "4125": 0.0, "4142": 0.16666666666666666, "4153": 0.125, "4179": 0.5, "4188": 0.0625, "4205": 0.5, "4233": 1.0, "4265": 1.0, "4286": 1.0, "4289": 0.2, "4306": 1.0, "4312": 1.0, "4335": 1.0, "4339": 1.0, "4394": 0.0, "4409": 0.3333333333333333, "4411": 1.0, "4414": 1.0, "4415": 0.0, "4433": 0.09090909090909091, "4447": 0.2, "4464": 0.3333333333333333, "4465": 0.5, "4484": 0.5, "4499": 0.05555555555555555, "4500": 0.07142857142857142, "4504": 1.0, "4514": 1.0, "4523": 0.5, "4539": 0.16666666666666666, "4571": 1.0, "4600": 1.0, "4605": 1.0, "4615": 0.3333333333333333, "4640": 0.0, "4641": 0.2, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 1.0, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 0.2, "4837": 0.0, "4844": 1.0, "4845": 1.0, "4846": 0.0, "4863": 0.5, "4865": 0.0, "4920": 1.0, "4942": 0.125, "4946": 1.0, "4955": 0.0, "4962": 1.0, "4968": 0.5, "4981": 1.0, "4999": 1.0, "5021": 0.0, "5030": 0.0, "5045": 0.1111111111111111, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.5, "5080": 0.0, "5083": 0.2, "5085": 1.0, "5086": 1.0, "5090": 0.25, "5125": 1.0, "5134": 1.0, "5150": 0.0, "5155": 1.0, "5172": 0.09090909090909091, "5178": 0.2, "5185": 0.0, "5196": 0.5, "5206": 0.0, "5228": 0.125, "5231": 0.058823529411764705, "5241": 1.0, "5254": 1.0, "5255": 1.0, "5264": 1.0, "5271": 0.3333333333333333, "5331": 1.0, "5343": 0.14285714285714285, "5347": 1.0, "5356": 0.16666666666666666, "5369": 0.0, "5374": 0.1111111111111111, "5380": 1.0, "5402": 0.0625, "5410": 0.5, "5422": 1.0, "5427": 0.25, "5460": 0.0, "5464": 0.16666666666666666, "5503": 1.0, "5505": 1.0, "5511": 0.25, "5534": 0.5, "5549": 0.0, "5585": 0.14285714285714285, "5592": 1.0, "5616": 0.16666666666666666, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 0.5, "5710": 0.0, "5741": 0.0625, "5763": 0.2, "5782": 1.0, "5790": 0.0, "5808": 0.3333333333333333, "5853": 0.3333333333333333, "5862": 0.25, "5888": 0.2, "5903": 0.0, "5906": 0.3333333333333333, "5940": 0.0, "5951": 1.0, "5970": 0.3333333333333333, "5981": 1.0, "5993": 1.0, "6002": 1.0, "6004": 0.0, "6005": 1.0, "6009": 0.09090909090909091, "6041": 1.0, "6080": 1.0, "6110": 0.5, "6121": 0.5, "6122": 1.0, "6131": 1.0, "6133": 1.0, "6142": 1.0, "6146": 1.0, "6199": 1.0, "6219": 0.5, "6221": 0.05, "6252": 0.0, "6262": 1.0, "6278": 0.14285714285714285, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.16666666666666666, "6467": 0.0, "6468": 1.0, "6479": 1.0, "6525": 0.5, "6554": 0.5, "6562": 0.1111111111111111, "6611": 1.0, "6612": 0.16666666666666666, "6625": 0.125, "6629": 0.0, "6635": 1.0, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 0.5, "6683": 1.0, "6713": 1.0, "6715": 0.5, "6746": 1.0, "6787": 0.0, "6792": 0.0, "6800": 0.25, "6803": 1.0, "6807": 1.0, "6814": 0.0, "6832": 0.2, "6835": 0.5, "6849": 0.2, "6862": 1.0, "6867": 0.25, "6875": 0.0, "6890": 0.25, "6891": 1.0, "6896": 0.0, "6901": 0.16666666666666666, "6907": 0.0, "6909": 0.0, "6959": 0.5, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 0.2, "7109": 0.3333333333333333, "7124": 1.0, "7141": 0.0625, "7145": 0.1, "7178": 0.2, "7188": 0.125, "7205": 1.0, "7206": 0.0, "7218": 0.25, "7221": 0.0, "7269": 0.06666666666666667, "7279": 0.3333333333333333, "7295": 0.0, "7311": 0.09090909090909091, "7326": 0.0, "7329": 0.25, "7344": 1.0, "7345": 0.14285714285714285, "7377": 1.0, "7431": 0.0, "7441": 0.3333333333333333, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.25, "7467": 0.5, "7484": 1.0, "7509": 0.0, "7512": 0.0625, "7513": 0.16666666666666666, "7529": 1.0, "7533": 0.08333333333333333, "7534": 1.0, "7590": 0.5, "7592": 0.0, "7594": 0.16666666666666666, "7622": 0.0, "7633": 1.0, "7674": 0.0625, "7700": 0.125, "7702": 0.5, "7705": 1.0, "7734": 1.0, "7747": 0.25, "7754": 0.09090909090909091, "7758": 0.5, "7801": 1.0, "7803": 0.14285714285714285, "7823": 1.0, "7876": 0.25, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.5, "7928": 0.3333333333333333, "7936": 0.5, "7992": 0.5, "8002": 1.0, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.125, "8040": 1.0, "8072": 1.0, "8079": 1.0, "8102": 0.1, "8116": 0.25, "8121": 1.0, "8202": 0.0, "8230": 0.0625, "8247": 0.0, "8271": 1.0, "8275": 0.2, "8296": 1.0, "8332": 1.0, "8351": 1.0, "8378": 0.125, "8456": 1.0, "8475": 1.0, "8507": 0.09090909090909091, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 0.16666666666666666, "8539": 0.09090909090909091, "8544": 1.0, "8592": 0.25, "8632": 0.3333333333333333, "8635": 0.0, "8702": 1.0, "8779": 0.09090909090909091, "8789": 1.0, "8795": 1.0, "8832": 1.0, "8834": 1.0, "8855": 0.0, "8874": 0.1, "8934": 1.0, "8937": 0.16666666666666666, "8947": 0.2, "8959": 0.2, "8970": 0.5, "8974": 1.0, "8982": 0.0, "9060": 0.3333333333333333, "9088": 0.0, "9108": 0.0, "9115": 0.25, "9126": 0.0, "9164": 0.14285714285714285, "9174": 0.07142857142857142, "9188": 1.0, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.14285714285714285, "9329": 1.0, "9332": 0.25, "9381": 0.5, "9385": 0.16666666666666666, "9391": 0.3333333333333333, "9403": 0.0, "9481": 0.3333333333333333, "9487": 0.08333333333333333, "9548": 0.3333333333333333, "9556": 1.0, "9565": 1.0, "9598": 0.16666666666666666, "9617": 0.16666666666666666, "9633": 1.0, "9643": 0.3333333333333333, "9644": 0.3333333333333333, "9646": 0.3333333333333333, "9668": 1.0, "9701": 0.0, "9733": 0.2, "9735": 0.14285714285714285, "9737": 1.0, "9771": 0.0, "9808": 0.5, "9824": 1.0, "9871": 0.5, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 0.0, "9979": 1.0, "10034": 0.5, "10039": 0.5, "10109": 1.0, "10122": 0.3333333333333333, "10136": 0.25, "10137": 0.14285714285714285, "10152": 1.0, "10183": 0.3333333333333333, "10213": 1.0, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.2, "10462": 0.0, "10482": 0.0, "10497": 1.0, "10526": 0.14285714285714285, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 0.5, "10628": 0.1111111111111111, "10639": 0.5, "10645": 0.5, "10674": 1.0, "10710": 0.125, "10734": 1.0, "10792": 1.0, "10808": 0.2, "10809": 1.0, "10812": 1.0, "10827": 0.25, "10845": 1.0, "10912": 0.08333333333333333, "10932": 1.0, "10975": 0.0, "10979": 1.0, "10994": 0.3333333333333333, "11039": 0.5, "11054": 1.0, "11088": 0.25}}} diff --git a/evals/benchmark/results/fiqa/ir-w0.5.jsonl b/evals/benchmark/results/fiqa/ir-w0.5.jsonl new file mode 100644 index 000000000..dfbd9f999 --- /dev/null +++ b/evals/benchmark/results/fiqa/ir-w0.5.jsonl @@ -0,0 +1 @@ +{"dataset": "fiqa", "run_tag": "w0.5", "aggregated": {"nDCG@10": 0.4333, "Recall@20": 0.6186, "MRR@10": 0.5053}, "per_query": {"nDCG@10": {"8": 0.6131471927654584, "15": 0.3562071871080222, "18": 0.6309297535714575, "26": 0.23719771276929622, "34": 0.0, "42": 0.46927872602275644, "56": 0.3562071871080222, "68": 0.0, "89": 0.286381299268402, "90": 0.6309297535714575, "94": 1.0, "98": 0.6131471927654584, "104": 0.6131471927654584, "106": 0.5, "109": 1.0, "475": 1.0, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.3333333333333333, "559": 0.0, "570": 0.0, "585": 0.2640681225725909, "588": 0.448643819352159, "594": 0.6131471927654584, "603": 0.0, "604": 0.3235866969737877, "620": 0.0, "622": 0.38685280723454163, "659": 0.43828689407543014, "672": 0.0, "684": 0.0, "687": 0.19342640361727081, "689": 0.0, "691": 0.6309297535714575, "699": 1.0, "701": 0.13565197343244778, "715": 0.0, "721": 1.0, "744": 0.18154179253735267, "750": 0.8503449055347546, "753": 0.38685280723454163, "766": 0.18154179253735267, "776": 0.3557772116892465, "810": 0.3333333333333333, "813": 0.20438239758848611, "849": 0.3333333333333333, "852": 0.7653606369886217, "853": 0.0, "858": 0.0, "859": 0.38685280723454163, "864": 0.7247145167543899, "879": 0.6309297535714575, "885": 0.19342640361727081, "895": 1.0, "904": 1.0, "928": 0.6131471927654584, "929": 1.0, "932": 0.3065735963827292, "939": 0.5, "945": 0.3010299956639812, "957": 0.7653606369886217, "988": 0.0, "1074": 0.6309297535714575, "1085": 0.0, "1090": 0.23719771276929622, "1150": 0.0, "1157": 0.6131471927654584, "1159": 0.0, "1198": 1.0, "1230": 0.0, "1281": 0.38685280723454163, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.559008518493208, "1310": 1.0, "1321": 0.0, "1322": 0.24630238874073, "1391": 0.18154179253735267, "1393": 0.30260241349881345, "1415": 0.9709286432396583, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.431733884398313, "1670": 0.7653606369886217, "1676": 0.5101671483174705, "1736": 0.5531464700081437, "1748": 0.6049306994552042, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.286381299268402, "1824": 0.31546487678572877, "1826": 1.0, "1832": 0.5, "1871": 0.6713527276121545, "1877": 1.0, "1889": 0.0, "1915": 0.8772153153380493, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.21398626473452756, "2010": 0.8194270280062366, "2051": 0.0, "2070": 0.18154179253735267, "2075": 0.4880470105041527, "2076": 0.5706417189553201, "2088": 0.0, "2108": 0.16716045496620227, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 0.7526136409516659, "2204": 0.9830168297915423, "2264": 0.3065735963827292, "2296": 0.33627415762678553, "2306": 0.6131471927654584, "2316": 1.0, "2318": 0.31546487678572877, "2330": 0.6131471927654584, "2334": 0.46927872602275644, "2348": 0.43231813227099475, "2376": 0.8910635073822442, "2383": 0.6131471927654584, "2384": 0.6309297535714575, "2385": 0.6131471927654584, "2388": 1.0, "2395": 1.0, "2398": 0.0, "2399": 1.0, "2400": 0.5307212739772434, "2407": 0.24630238874073, "2416": 0.9058653015743079, "2423": 0.3576223542196109, "2443": 0.0, "2445": 1.0, "2460": 0.9469024295259745, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 0.43067655807339306, "2513": 0.0, "2516": 0.5585075862632192, "2549": 0.6131471927654584, "2551": 1.0, "2568": 0.25288473642100684, "2579": 0.28201326940239274, "2580": 0.46927872602275644, "2587": 1.0, "2589": 0.7653606369886217, "2590": 0.46927872602275644, "2593": 0.8854598815714874, "2598": 0.8503449055347546, "2648": 1.0, "2676": 0.5, "2685": 0.35079429740281753, "2695": 1.0, "2713": 0.38685280723454163, "2724": 0.8175295903539445, "2737": 1.0, "2747": 0.5, "2749": 0.8772153153380493, "2790": 0.47997295436377346, "2801": 0.6131471927654584, "2856": 0.7095272044910244, "2857": 0.6595629128335778, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.3010299956639812, "2923": 0.8207658584763088, "2964": 0.0, "2968": 0.8772153153380493, "2994": 0.46927872602275644, "3006": 0.25909036127391766, "3008": 0.24630238874073, "3014": 1.0, "3033": 0.38685280723454163, "3039": 0.38685280723454163, "3049": 0.24630238874073, "3051": 1.0, "3067": 0.0, "3085": 0.19342640361727081, "3091": 0.9020870746500321, "3103": 0.6309297535714575, "3115": 0.4935232796777481, "3125": 1.0, "3148": 0.5531464700081437, "3149": 0.0, "3177": 0.15642624200758548, "3179": 0.8503449055347546, "3186": 1.0, "3189": 0.0, "3254": 0.6713860725233041, "3264": 0.8048099750039491, "3357": 1.0, "3369": 0.0, "3394": 0.5805485932475565, "3404": 0.3932591258696538, "3405": 0.0, "3446": 0.5669646403403741, "3451": 0.07945707943276742, "3453": 0.5706417189553201, "3480": 1.0, "3490": 0.2890648263178879, "3500": 0.644824486427155, "3503": 0.6183313187362977, "3512": 0.8503449055347546, "3528": 0.6309297535714575, "3530": 0.4776237035032179, "3534": 0.0, "3566": 0.8194270280062366, "3569": 0.5, "3594": 0.123151194370365, "3612": 0.11751610475642714, "3615": 0.45749452618986164, "3625": 0.20210734650054757, "3682": 0.20438239758848611, "3683": 0.5982291239616433, "3694": 0.6131471927654584, "3724": 0.0, "3735": 0.0, "3759": 0.0, "3767": 0.2716774977597197, "3771": 0.19632790425573848, "3781": 0.9197207891481876, "3789": 0.8048099750039491, "3791": 0.431733884398313, "3801": 0.0, "3822": 0.0, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5517854393872873, "3932": 0.0, "3934": 0.16812753627111746, "3995": 0.0, "4007": 0.7039180890341347, "4011": 0.9674679834891693, "4019": 0.31608365985743925, "4031": 0.0, "4037": 0.18457569677956817, "4047": 0.8318724637288826, "4071": 0.0, "4084": 0.5012658353418871, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 0.6131471927654584, "4125": 0.0, "4142": 0.39233405721184117, "4153": 0.31546487678572877, "4179": 0.6309297535714575, "4188": 0.0, "4205": 0.4525081529734507, "4233": 0.8687949224876582, "4265": 0.7751482523375255, "4286": 0.6131471927654584, "4289": 0.23719771276929622, "4306": 0.7598336331031966, "4312": 0.7237478018511487, "4335": 0.8318724637288826, "4339": 1.0, "4394": 0.0, "4409": 0.19518900079066107, "4411": 0.46927872602275644, "4414": 0.8315546295836225, "4415": 0.0, "4433": 0.0, "4447": 1.0, "4464": 0.5, "4465": 0.2960819109658652, "4484": 0.6309297535714575, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 0.4034698846650846, "4523": 0.21398626473452756, "4539": 0.21840743681816419, "4571": 0.7653606369886217, "4600": 1.0, "4605": 0.5585830187272559, "4615": 0.4909156065077737, "4640": 0.0, "4641": 0.252016380211836, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75369761125927, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.6131471927654584, "4813": 1.0, "4823": 0.46927872602275644, "4827": 0.3562071871080222, "4837": 0.0, "4844": 1.0, "4845": 0.7653606369886217, "4846": 0.0, "4863": 0.49818925746641285, "4865": 0.0, "4920": 1.0, "4942": 0.31546487678572877, "4946": 1.0, "4955": 0.0, "4962": 0.6131471927654584, "4968": 0.6309297535714575, "4981": 0.7039180890341347, "4999": 0.5294362295028773, "5021": 0.0, "5030": 0.0, "5045": 0.3010299956639812, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.6309297535714575, "5080": 0.0, "5083": 0.23719771276929622, "5085": 1.0, "5086": 1.0, "5090": 0.2640681225725909, "5125": 1.0, "5134": 0.38685280723454163, "5150": 0.0, "5155": 1.0, "5172": 0.0, "5178": 0.252016380211836, "5185": 0.0, "5196": 0.2960819109658652, "5206": 0.0, "5228": 0.31546487678572877, "5231": 0.0, "5241": 0.44615333764087994, "5254": 0.6131471927654584, "5255": 1.0, "5264": 0.6843515475204855, "5271": 0.41618115554873086, "5331": 1.0, "5343": 0.3333333333333333, "5347": 1.0, "5356": 0.1208113026994942, "5369": 0.0, "5374": 0.3010299956639812, "5380": 1.0, "5402": 0.0, "5410": 0.2960819109658652, "5422": 0.24630238874073, "5427": 0.23719771276929622, "5460": 0.0, "5464": 0.16716045496620227, "5503": 0.6131471927654584, "5505": 1.0, "5511": 0.1610425878239398, "5534": 0.38685280723454163, "5549": 0.0, "5585": 0.31546487678572877, "5592": 1.0, "5616": 0.3562071871080222, "5620": 0.0, "5646": 0.9217868789962071, "5653": 1.0, "5683": 0.6934264036172708, "5710": 0.0, "5741": 0.0, "5763": 0.23463936301137822, "5782": 0.4371994911049732, "5790": 0.0, "5808": 0.5, "5853": 0.21078558563971425, "5862": 0.29127873064148246, "5888": 0.2640681225725909, "5903": 0.0, "5906": 0.44130740935663865, "5940": 0.0, "5951": 0.42750379661745125, "5970": 0.23463936301137822, "5981": 0.9896062251871525, "5993": 0.31488013066763093, "6002": 0.7788065869851872, "6004": 0.0, "6005": 0.5388856921828065, "6009": 0.0, "6041": 0.7668091220635322, "6080": 0.6131471927654584, "6110": 0.41918605213740834, "6121": 0.3065735963827292, "6122": 0.46927872602275644, "6131": 0.5810820381115676, "6133": 0.6131471927654584, "6142": 0.6131471927654584, "6146": 0.8315546295836225, "6199": 0.5135312443624667, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.6713860725233041, "6278": 0.23719771276929622, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.3562071871080222, "6467": 0.0, "6468": 0.6131471927654584, "6479": 1.0, "6525": 0.49818925746641285, "6554": 0.6131471927654584, "6562": 0.11751610475642714, "6611": 0.7903864795495061, "6612": 0.16716045496620227, "6625": 0.3010299956639812, "6629": 0.0, "6635": 0.6366824387328317, "6644": 0.0, "6647": 0.7653606369886217, "6668": 0.8503449055347546, "6679": 0.4632423659320675, "6683": 0.8772153153380493, "6713": 0.6131471927654584, "6715": 0.6131471927654584, "6746": 0.3065735963827292, "6787": 0.0, "6792": 0.0, "6800": 0.43067655807339306, "6803": 0.5205067333228022, "6807": 1.0, "6814": 0.0, "6832": 0.430624116386567, "6835": 0.6131471927654584, "6849": 0.43067655807339306, "6862": 1.0, "6867": 0.22471947398559278, "6875": 0.0, "6890": 0.5012658353418871, "6891": 0.617319681505689, "6896": 0.0, "6901": 0.21840743681816419, "6907": 0.0, "6909": 0.0, "6959": 0.38685280723454163, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 0.8772153153380493, "7096": 1.0, "7098": 0.6131471927654584, "7105": 0.38685280723454163, "7109": 0.23463936301137822, "7124": 0.6525874238732422, "7141": 0.0, "7145": 0.13565197343244778, "7178": 0.3562071871080222, "7188": 0.31546487678572877, "7205": 1.0, "7206": 0.0, "7218": 0.5, "7221": 0.0, "7269": 0.0, "7279": 0.44229834886928765, "7295": 0.0, "7311": 0.0, "7326": 0.0, "7329": 0.46845052016107697, "7344": 0.6131471927654584, "7345": 0.15642624200758548, "7377": 1.0, "7431": 0.0, "7441": 0.20210734650054757, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.14606834984270645, "7467": 0.6309297535714575, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.16716045496620227, "7529": 0.38685280723454163, "7533": 0.0, "7534": 0.46927872602275644, "7590": 0.5714285040141098, "7592": 0.0, "7594": 0.3333333333333333, "7622": 0.0, "7633": 0.3903800499921017, "7674": 0.0, "7700": 0.123151194370365, "7702": 0.38685280723454163, "7705": 0.49818925746641285, "7734": 0.6131471927654584, "7747": 0.21840743681816419, "7754": 0.0, "7758": 0.6309297535714575, "7801": 1.0, "7803": 0.19342640361727081, "7823": 0.6105456988825855, "7876": 0.43067655807339306, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.19092086617893467, "7928": 0.24630238874073, "7936": 0.39106560501896365, "7992": 0.38685280723454163, "8002": 0.6257049680303419, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.31546487678572877, "8040": 1.0, "8072": 0.6508205185601091, "8079": 0.7039180890341347, "8102": 0.11284514134893527, "8116": 0.45749452618986164, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 0.6131471927654584, "8275": 0.43067655807339306, "8296": 0.8315546295836225, "8332": 0.9060254355346823, "8351": 0.6131471927654584, "8378": 0.1480409554829326, "8456": 0.6049306994552042, "8475": 0.7957503936198023, "8507": 0.0, "8512": 1.0, "8513": 0.0, "8532": 0.9325210919548239, "8537": 0.3562071871080222, "8539": 0.0, "8544": 0.6131471927654584, "8592": 0.23463936301137822, "8632": 0.2640681225725909, "8635": 0.0, "8702": 0.7653606369886217, "8779": 0.0, "8789": 0.7653606369886217, "8795": 1.0, "8832": 1.0, "8834": 0.3980628465882808, "8855": 0.0, "8874": 0.31546487678572877, "8934": 0.8065735963827293, "8937": 0.3562071871080222, "8947": 0.23719771276929622, "8959": 0.43067655807339306, "8970": 0.6509209298071326, "8974": 0.41253177989255346, "8982": 0.0, "9060": 0.38685280723454163, "9088": 0.0, "9108": 0.0, "9115": 0.5256940434743352, "9126": 0.0, "9164": 0.1480409554829326, "9174": 0.0, "9188": 0.6131471927654584, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.20438239758848611, "9329": 1.0, "9332": 0.3562071871080222, "9381": 0.5706417189553201, "9385": 0.3562071871080222, "9391": 0.5143371794949736, "9403": 0.0, "9481": 0.6240505200038379, "9487": 0.0, "9548": 0.5, "9556": 1.0, "9565": 1.0, "9598": 0.3562071871080222, "9617": 0.3152014104491349, "9633": 0.6131471927654584, "9643": 0.5, "9644": 0.6309297535714575, "9646": 0.45749452618986164, "9668": 0.6508205185601091, "9701": 0.0, "9733": 0.18154179253735267, "9735": 0.3333333333333333, "9737": 0.7653606369886217, "9771": 0.0, "9808": 0.2960819109658652, "9824": 1.0, "9871": 0.24630238874073, "9882": 0.0, "9925": 0.8315546295836225, "9929": 0.5714285040141098, "9961": 0.0, "9979": 0.3903800499921017, "10034": 0.3065735963827292, "10039": 0.6309297535714575, "10109": 0.5399294342072939, "10122": 0.19519002499605084, "10136": 0.34337431936037655, "10137": 0.3333333333333333, "10152": 1.0, "10183": 0.3065735963827292, "10213": 0.46927872602275644, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.18154179253735267, "10462": 0.0, "10482": 0.0, "10497": 0.48864468918810133, "10526": 0.20438239758848611, "10547": 0.0, "10558": 0.0, "10596": 0.9427731067700877, "10601": 1.0, "10628": 0.3010299956639812, "10639": 0.3120255505658846, "10645": 0.38685280723454163, "10674": 0.7095272044910244, "10710": 0.3333333333333333, "10734": 0.8696762340896295, "10792": 0.46845052016107697, "10808": 0.4652347214527331, "10809": 0.6049306994552042, "10812": 0.9493885684853097, "10827": 0.14606834984270645, "10845": 1.0, "10912": 0.0, "10932": 1.0, "10975": 0.0, "10979": 0.6366824387328317, "10994": 0.5, "11039": 0.315526956404728, "11054": 1.0, "11088": 0.43067655807339306}, "Recall@20": {"8": 0.5, "15": 1.0, "18": 1.0, "26": 0.5, "34": 0.0, "42": 0.3333333333333333, "56": 1.0, "68": 0.0, "89": 0.5, "90": 1.0, "94": 1.0, "98": 0.5, "104": 0.5, "106": 1.0, "109": 1.0, "475": 1.0, "503": 1.0, "504": 0.3333333333333333, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 1.0, "585": 0.5, "588": 1.0, "594": 1.0, "603": 0.0, "604": 1.0, "620": 0.2, "622": 0.5, "659": 0.7, "672": 0.5, "684": 1.0, "687": 0.5, "689": 0.0, "691": 1.0, "699": 1.0, "701": 0.3333333333333333, "715": 0.0, "721": 1.0, "744": 0.3333333333333333, "750": 1.0, "753": 0.5, "766": 0.3333333333333333, "776": 0.25, "810": 1.0, "813": 1.0, "849": 1.0, "852": 0.6666666666666666, "853": 0.5, "858": 0.0, "859": 0.5, "864": 0.6666666666666666, "879": 1.0, "885": 0.5, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.5, "939": 1.0, "945": 1.0, "957": 0.6666666666666666, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.5, "1150": 0.0, "1157": 0.5, "1159": 0.0, "1198": 1.0, "1230": 1.0, "1281": 1.0, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.75, "1310": 1.0, "1321": 0.0, "1322": 0.25, "1391": 0.6666666666666666, "1393": 0.5, "1415": 1.0, "1416": 1.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 1.0, "1670": 0.6666666666666666, "1676": 1.0, "1736": 0.4, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 1.0, "1826": 1.0, "1832": 1.0, "1871": 0.75, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.4, "2010": 1.0, "2051": 1.0, "2070": 0.6666666666666666, "2075": 0.6666666666666666, "2076": 1.0, "2088": 0.0, "2108": 0.3333333333333333, "2118": 0.0, "2154": 0.0, "2181": 0.5, "2183": 0.8333333333333334, "2204": 1.0, "2264": 0.5, "2296": 0.2222222222222222, "2306": 0.5, "2316": 1.0, "2318": 1.0, "2330": 0.5, "2334": 0.6666666666666666, "2348": 0.26666666666666666, "2376": 1.0, "2383": 0.5, "2384": 1.0, "2385": 0.5, "2388": 1.0, "2395": 1.0, "2398": 0.2, "2399": 1.0, "2400": 0.6666666666666666, "2407": 0.5, "2416": 1.0, "2423": 0.8571428571428571, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.3333333333333333, "2472": 0.0, "2486": 1.0, "2498": 1.0, "2513": 1.0, "2516": 0.75, "2549": 0.5, "2551": 1.0, "2568": 0.2857142857142857, "2579": 0.5, "2580": 0.3333333333333333, "2587": 1.0, "2589": 0.6666666666666666, "2590": 0.6666666666666666, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 1.0, "2685": 0.25, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 1.0, "2790": 0.42857142857142855, "2801": 1.0, "2856": 0.75, "2857": 0.75, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 1.0, "2923": 0.8, "2964": 0.5, "2968": 1.0, "2994": 0.3333333333333333, "3006": 0.5, "3008": 0.25, "3014": 1.0, "3033": 0.5, "3039": 1.0, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 1.0, "3091": 0.8571428571428571, "3103": 1.0, "3115": 0.5, "3125": 1.0, "3148": 0.6, "3149": 0.0, "3177": 0.6666666666666666, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.6666666666666666, "3264": 0.75, "3357": 1.0, "3369": 0.16666666666666666, "3394": 0.75, "3404": 0.42857142857142855, "3405": 1.0, "3446": 0.8, "3451": 0.42857142857142855, "3453": 1.0, "3480": 1.0, "3490": 1.0, "3500": 0.5, "3503": 0.7142857142857143, "3512": 1.0, "3528": 1.0, "3530": 0.6666666666666666, "3534": 0.0, "3566": 1.0, "3569": 1.0, "3594": 0.25, "3612": 0.75, "3615": 1.0, "3625": 0.3333333333333333, "3682": 1.0, "3683": 0.6666666666666666, "3694": 0.5, "3724": 0.0, "3735": 1.0, "3759": 0.0, "3767": 0.4, "3771": 0.3333333333333333, "3781": 1.0, "3789": 0.75, "3791": 0.6666666666666666, "3801": 0.0, "3822": 0.25, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 0.6666666666666666, "4011": 1.0, "4019": 0.4, "4031": 0.0, "4037": 0.5, "4047": 0.75, "4071": 0.0, "4084": 1.0, "4102": 0.25, "4103": 0.0, "4105": 0.0, "4116": 0.5, "4125": 0.0, "4142": 0.75, "4153": 1.0, "4179": 1.0, "4188": 1.0, "4205": 0.6666666666666666, "4233": 0.8, "4265": 0.6666666666666666, "4286": 0.5, "4289": 0.5, "4306": 0.75, "4312": 0.8571428571428571, "4335": 0.75, "4339": 1.0, "4394": 0.0, "4409": 0.2, "4411": 0.6666666666666666, "4414": 1.0, "4415": 0.0, "4433": 0.5, "4447": 1.0, "4464": 1.0, "4465": 0.6666666666666666, "4484": 1.0, "4499": 0.3333333333333333, "4500": 0.5, "4504": 1.0, "4514": 0.5, "4523": 0.2, "4539": 0.5, "4571": 0.6666666666666666, "4600": 1.0, "4605": 0.7142857142857143, "4615": 0.6666666666666666, "4640": 0.0, "4641": 0.6, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 0.6666666666666666, "4846": 0.0, "4863": 1.0, "4865": 0.0, "4920": 1.0, "4942": 1.0, "4946": 1.0, "4955": 0.0, "4962": 0.5, "4968": 1.0, "4981": 0.6666666666666666, "4999": 0.75, "5021": 0.0, "5030": 0.0, "5045": 1.0, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 1.0, "5080": 0.0, "5083": 0.5, "5085": 1.0, "5086": 1.0, "5090": 0.5, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 1.0, "5178": 0.4, "5185": 0.0, "5196": 0.3333333333333333, "5206": 0.0, "5228": 1.0, "5231": 1.0, "5241": 0.4, "5254": 0.5, "5255": 1.0, "5264": 0.6, "5271": 1.0, "5331": 1.0, "5343": 1.0, "5347": 1.0, "5356": 0.4, "5369": 0.0, "5374": 1.0, "5380": 1.0, "5402": 0.5, "5410": 0.3333333333333333, "5422": 0.75, "5427": 0.5, "5460": 0.0, "5464": 0.3333333333333333, "5503": 0.5, "5505": 1.0, "5511": 0.3, "5534": 0.5, "5549": 0.0, "5585": 1.0, "5592": 1.0, "5616": 1.0, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.5, "5763": 0.6666666666666666, "5782": 0.4, "5790": 0.0, "5808": 1.0, "5853": 0.375, "5862": 0.5, "5888": 0.5, "5903": 0.0, "5906": 1.0, "5940": 0.0, "5951": 0.5555555555555556, "5970": 0.6666666666666666, "5981": 1.0, "5993": 0.2, "6002": 0.9090909090909091, "6004": 0.0, "6005": 0.38461538461538464, "6009": 1.0, "6041": 0.75, "6080": 0.5, "6110": 0.375, "6121": 0.5, "6122": 0.3333333333333333, "6131": 0.4166666666666667, "6133": 0.5, "6142": 0.5, "6146": 1.0, "6199": 0.5, "6219": 1.0, "6221": 0.125, "6252": 0.0, "6262": 0.6666666666666666, "6278": 1.0, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 1.0, "6467": 0.0, "6468": 0.5, "6479": 1.0, "6525": 0.6666666666666666, "6554": 0.5, "6562": 0.5, "6611": 1.0, "6612": 0.3333333333333333, "6625": 1.0, "6629": 0.0, "6635": 0.5, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 1.0, "6683": 1.0, "6713": 0.5, "6715": 1.0, "6746": 0.5, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.75, "6807": 1.0, "6814": 0.0, "6832": 1.0, "6835": 0.5, "6849": 1.0, "6862": 1.0, "6867": 0.42857142857142855, "6875": 0.0, "6890": 1.0, "6891": 0.6666666666666666, "6896": 0.0, "6901": 0.5, "6907": 0.0, "6909": 0.0, "6959": 1.0, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 1.0, "7109": 0.6666666666666666, "7124": 0.75, "7141": 0.5, "7145": 0.3333333333333333, "7178": 1.0, "7188": 1.0, "7205": 1.0, "7206": 0.0, "7218": 1.0, "7221": 0.0, "7269": 1.0, "7279": 0.75, "7295": 0.0, "7311": 1.0, "7326": 0.0, "7329": 1.0, "7344": 0.5, "7345": 0.6666666666666666, "7377": 1.0, "7431": 0.0, "7441": 0.3333333333333333, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.4, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 1.0, "7513": 0.6666666666666666, "7529": 0.5, "7533": 0.5, "7534": 0.3333333333333333, "7590": 1.0, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 0.5, "7674": 0.3333333333333333, "7700": 0.25, "7702": 1.0, "7705": 0.6666666666666666, "7734": 0.5, "7747": 0.5, "7754": 1.0, "7758": 1.0, "7801": 1.0, "7803": 0.5, "7823": 0.6666666666666666, "7876": 1.0, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.16666666666666666, "7928": 0.25, "7936": 0.6666666666666666, "7992": 1.0, "8002": 0.6666666666666666, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 1.0, "8040": 1.0, "8072": 0.6666666666666666, "8079": 1.0, "8102": 0.25, "8116": 1.0, "8121": 1.0, "8202": 0.0, "8230": 1.0, "8247": 0.0, "8271": 0.5, "8275": 1.0, "8296": 1.0, "8332": 1.0, "8351": 0.5, "8378": 0.3333333333333333, "8456": 0.6666666666666666, "8475": 0.8571428571428571, "8507": 0.3333333333333333, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 1.0, "8539": 0.25, "8544": 0.5, "8592": 0.6666666666666666, "8632": 1.0, "8635": 0.0, "8702": 0.6666666666666666, "8779": 1.0, "8789": 0.6666666666666666, "8795": 1.0, "8832": 1.0, "8834": 0.5, "8855": 0.0, "8874": 1.0, "8934": 1.0, "8937": 1.0, "8947": 0.5, "8959": 1.0, "8970": 1.0, "8974": 0.25, "8982": 0.0, "9060": 0.5, "9088": 0.0, "9108": 0.0, "9115": 1.0, "9126": 0.0, "9164": 0.3333333333333333, "9174": 0.2, "9188": 0.5, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.5, "9329": 1.0, "9332": 1.0, "9381": 1.0, "9385": 1.0, "9391": 0.75, "9403": 0.0, "9481": 1.0, "9487": 1.0, "9548": 1.0, "9556": 1.0, "9565": 1.0, "9598": 1.0, "9617": 0.6666666666666666, "9633": 0.5, "9643": 1.0, "9644": 1.0, "9646": 1.0, "9668": 1.0, "9701": 0.0, "9733": 0.3333333333333333, "9735": 1.0, "9737": 0.6666666666666666, "9771": 0.0, "9808": 0.3333333333333333, "9824": 1.0, "9871": 0.25, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 0.0, "9979": 0.5, "10034": 0.5, "10039": 1.0, "10109": 0.42857142857142855, "10122": 0.5, "10136": 0.6666666666666666, "10137": 1.0, "10152": 1.0, "10183": 1.0, "10213": 0.3333333333333333, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.3333333333333333, "10462": 0.0, "10482": 0.0, "10497": 0.7, "10526": 0.5, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 1.0, "10639": 0.4, "10645": 1.0, "10674": 1.0, "10710": 1.0, "10734": 1.0, "10792": 1.0, "10808": 1.0, "10809": 0.6666666666666666, "10812": 1.0, "10827": 0.2, "10845": 1.0, "10912": 1.0, "10932": 1.0, "10975": 0.0, "10979": 0.75, "10994": 1.0, "11039": 0.5, "11054": 1.0, "11088": 1.0}, "MRR@10": {"8": 1.0, "15": 0.16666666666666666, "18": 0.5, "26": 0.2, "34": 0.0, "42": 1.0, "56": 0.16666666666666666, "68": 0.0, "89": 0.5, "90": 0.5, "94": 1.0, "98": 1.0, "104": 1.0, "106": 0.3333333333333333, "109": 1.0, "475": 1.0, "503": 0.08333333333333333, "504": 0.05555555555555555, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.14285714285714285, "559": 0.0, "570": 0.06666666666666667, "585": 0.25, "588": 0.25, "594": 1.0, "603": 0.0, "604": 0.16666666666666666, "620": 0.08333333333333333, "622": 0.5, "659": 1.0, "672": 0.07692307692307693, "684": 0.05555555555555555, "687": 0.125, "689": 0.0, "691": 0.5, "699": 1.0, "701": 0.1, "715": 0.0, "721": 1.0, "744": 0.2, "750": 1.0, "753": 0.5, "766": 0.2, "776": 1.0, "810": 0.14285714285714285, "813": 0.14285714285714285, "849": 0.14285714285714285, "852": 1.0, "853": 0.09090909090909091, "858": 0.0, "859": 0.5, "864": 1.0, "879": 0.5, "885": 0.125, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.3333333333333333, "939": 0.3333333333333333, "945": 0.1111111111111111, "957": 1.0, "988": 0.0, "1074": 0.5, "1085": 0.0, "1090": 0.2, "1150": 0.0, "1157": 1.0, "1159": 0.0, "1198": 1.0, "1230": 0.07142857142857142, "1281": 0.2, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.5, "1310": 1.0, "1321": 0.0, "1322": 0.5, "1391": 0.2, "1393": 1.0, "1415": 1.0, "1416": 0.058823529411764705, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.5, "1670": 1.0, "1676": 0.5, "1736": 1.0, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 0.125, "1826": 1.0, "1832": 0.3333333333333333, "1871": 1.0, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.5, "2010": 1.0, "2051": 0.07142857142857142, "2070": 0.2, "2075": 1.0, "2076": 0.3333333333333333, "2088": 0.0, "2108": 0.16666666666666666, "2118": 0.0, "2154": 0.0, "2181": 0.05263157894736842, "2183": 1.0, "2204": 1.0, "2264": 0.3333333333333333, "2296": 1.0, "2306": 1.0, "2316": 1.0, "2318": 0.125, "2330": 1.0, "2334": 1.0, "2348": 1.0, "2376": 1.0, "2383": 1.0, "2384": 0.5, "2385": 1.0, "2388": 1.0, "2395": 1.0, "2398": 0.08333333333333333, "2399": 1.0, "2400": 0.5, "2407": 0.5, "2416": 1.0, "2423": 1.0, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0625, "2472": 0.0, "2486": 1.0, "2498": 0.25, "2513": 0.0625, "2516": 1.0, "2549": 1.0, "2551": 1.0, "2568": 0.5, "2579": 0.5, "2580": 1.0, "2587": 1.0, "2589": 1.0, "2590": 1.0, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 0.3333333333333333, "2685": 1.0, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 0.3333333333333333, "2749": 1.0, "2790": 1.0, "2801": 1.0, "2856": 1.0, "2857": 1.0, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.1111111111111111, "2923": 1.0, "2964": 0.07692307692307693, "2968": 1.0, "2994": 1.0, "3006": 0.3333333333333333, "3008": 0.5, "3014": 1.0, "3033": 0.5, "3039": 0.2, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 0.125, "3091": 1.0, "3103": 0.5, "3115": 1.0, "3125": 1.0, "3148": 1.0, "3149": 0.0, "3177": 0.14285714285714285, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 1.0, "3264": 1.0, "3357": 1.0, "3369": 0.09090909090909091, "3394": 0.5, "3404": 1.0, "3405": 0.09090909090909091, "3446": 1.0, "3451": 0.1, "3453": 0.3333333333333333, "3480": 1.0, "3490": 0.1, "3500": 1.0, "3503": 0.5, "3512": 1.0, "3528": 0.5, "3530": 0.5, "3534": 0.0, "3566": 1.0, "3569": 0.3333333333333333, "3594": 0.125, "3612": 0.1111111111111111, "3615": 0.25, "3625": 0.25, "3682": 0.14285714285714285, "3683": 1.0, "3694": 1.0, "3724": 0.0, "3735": 0.09090909090909091, "3759": 0.0, "3767": 0.3333333333333333, "3771": 0.14285714285714285, "3781": 1.0, "3789": 1.0, "3791": 0.5, "3801": 0.0, "3822": 0.05263157894736842, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 1.0, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 1.0, "4011": 1.0, "4019": 0.5, "4031": 0.0, "4037": 0.1111111111111111, "4047": 1.0, "4071": 0.0, "4084": 0.25, "4102": 0.06666666666666667, "4103": 0.0, "4105": 0.0, "4116": 1.0, "4125": 0.0, "4142": 0.16666666666666666, "4153": 0.125, "4179": 0.5, "4188": 0.0625, "4205": 0.5, "4233": 1.0, "4265": 1.0, "4286": 1.0, "4289": 0.2, "4306": 1.0, "4312": 1.0, "4335": 1.0, "4339": 1.0, "4394": 0.0, "4409": 0.3333333333333333, "4411": 1.0, "4414": 1.0, "4415": 0.0, "4433": 0.09090909090909091, "4447": 1.0, "4464": 0.3333333333333333, "4465": 0.5, "4484": 0.5, "4499": 0.05555555555555555, "4500": 0.07142857142857142, "4504": 1.0, "4514": 1.0, "4523": 0.5, "4539": 0.16666666666666666, "4571": 1.0, "4600": 1.0, "4605": 1.0, "4615": 0.5, "4640": 0.0, "4641": 0.2, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 1.0, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 0.16666666666666666, "4837": 0.0, "4844": 1.0, "4845": 1.0, "4846": 0.0, "4863": 0.5, "4865": 0.0, "4920": 1.0, "4942": 0.125, "4946": 1.0, "4955": 0.0, "4962": 1.0, "4968": 0.5, "4981": 1.0, "4999": 1.0, "5021": 0.0, "5030": 0.0, "5045": 0.1111111111111111, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.5, "5080": 0.0, "5083": 0.2, "5085": 1.0, "5086": 1.0, "5090": 0.25, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 0.09090909090909091, "5178": 0.2, "5185": 0.0, "5196": 0.5, "5206": 0.0, "5228": 0.125, "5231": 0.058823529411764705, "5241": 1.0, "5254": 1.0, "5255": 1.0, "5264": 1.0, "5271": 0.3333333333333333, "5331": 1.0, "5343": 0.14285714285714285, "5347": 1.0, "5356": 0.16666666666666666, "5369": 0.0, "5374": 0.1111111111111111, "5380": 1.0, "5402": 0.0625, "5410": 0.5, "5422": 0.5, "5427": 0.2, "5460": 0.0, "5464": 0.16666666666666666, "5503": 1.0, "5505": 1.0, "5511": 0.25, "5534": 0.5, "5549": 0.0, "5585": 0.125, "5592": 1.0, "5616": 0.16666666666666666, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 0.5, "5710": 0.0, "5741": 0.0625, "5763": 0.3333333333333333, "5782": 1.0, "5790": 0.0, "5808": 0.3333333333333333, "5853": 0.3333333333333333, "5862": 0.25, "5888": 0.25, "5903": 0.0, "5906": 0.25, "5940": 0.0, "5951": 0.5, "5970": 0.3333333333333333, "5981": 1.0, "5993": 1.0, "6002": 1.0, "6004": 0.0, "6005": 1.0, "6009": 0.09090909090909091, "6041": 1.0, "6080": 1.0, "6110": 1.0, "6121": 0.3333333333333333, "6122": 1.0, "6131": 1.0, "6133": 1.0, "6142": 1.0, "6146": 1.0, "6199": 1.0, "6219": 1.0, "6221": 0.05, "6252": 0.0, "6262": 1.0, "6278": 0.2, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.16666666666666666, "6467": 0.0, "6468": 1.0, "6479": 1.0, "6525": 0.5, "6554": 1.0, "6562": 0.1111111111111111, "6611": 1.0, "6612": 0.16666666666666666, "6625": 0.1111111111111111, "6629": 0.0, "6635": 1.0, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 0.5, "6683": 1.0, "6713": 1.0, "6715": 1.0, "6746": 0.3333333333333333, "6787": 0.0, "6792": 0.0, "6800": 0.25, "6803": 1.0, "6807": 1.0, "6814": 0.0, "6832": 0.2, "6835": 1.0, "6849": 0.25, "6862": 1.0, "6867": 0.25, "6875": 0.0, "6890": 0.25, "6891": 1.0, "6896": 0.0, "6901": 0.16666666666666666, "6907": 0.0, "6909": 0.0, "6959": 0.5, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 0.2, "7109": 0.3333333333333333, "7124": 1.0, "7141": 0.0625, "7145": 0.1, "7178": 0.16666666666666666, "7188": 0.125, "7205": 1.0, "7206": 0.0, "7218": 0.3333333333333333, "7221": 0.0, "7269": 0.06666666666666667, "7279": 0.25, "7295": 0.0, "7311": 0.09090909090909091, "7326": 0.0, "7329": 0.25, "7344": 1.0, "7345": 0.14285714285714285, "7377": 1.0, "7431": 0.0, "7441": 0.25, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.25, "7467": 0.5, "7484": 1.0, "7509": 0.0, "7512": 0.0625, "7513": 0.16666666666666666, "7529": 0.5, "7533": 0.08333333333333333, "7534": 1.0, "7590": 0.5, "7592": 0.0, "7594": 0.14285714285714285, "7622": 0.0, "7633": 1.0, "7674": 0.0625, "7700": 0.125, "7702": 0.5, "7705": 0.5, "7734": 1.0, "7747": 0.16666666666666666, "7754": 0.08333333333333333, "7758": 0.5, "7801": 1.0, "7803": 0.125, "7823": 1.0, "7876": 0.25, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.5, "7928": 0.5, "7936": 0.3333333333333333, "7992": 0.5, "8002": 1.0, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.125, "8040": 1.0, "8072": 1.0, "8079": 1.0, "8102": 0.1, "8116": 0.25, "8121": 1.0, "8202": 0.0, "8230": 0.0625, "8247": 0.0, "8271": 1.0, "8275": 0.25, "8296": 1.0, "8332": 1.0, "8351": 1.0, "8378": 0.125, "8456": 1.0, "8475": 1.0, "8507": 0.09090909090909091, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 0.16666666666666666, "8539": 0.09090909090909091, "8544": 1.0, "8592": 0.3333333333333333, "8632": 0.25, "8635": 0.0, "8702": 1.0, "8779": 0.09090909090909091, "8789": 1.0, "8795": 1.0, "8832": 1.0, "8834": 1.0, "8855": 0.0, "8874": 0.125, "8934": 1.0, "8937": 0.16666666666666666, "8947": 0.2, "8959": 0.25, "8970": 0.5, "8974": 1.0, "8982": 0.0, "9060": 0.5, "9088": 0.0, "9108": 0.0, "9115": 0.25, "9126": 0.0, "9164": 0.125, "9174": 0.07142857142857142, "9188": 1.0, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.14285714285714285, "9329": 1.0, "9332": 0.16666666666666666, "9381": 0.3333333333333333, "9385": 0.16666666666666666, "9391": 0.3333333333333333, "9403": 0.0, "9481": 0.5, "9487": 0.08333333333333333, "9548": 0.3333333333333333, "9556": 1.0, "9565": 1.0, "9598": 0.16666666666666666, "9617": 0.16666666666666666, "9633": 1.0, "9643": 0.3333333333333333, "9644": 0.5, "9646": 0.25, "9668": 1.0, "9701": 0.0, "9733": 0.2, "9735": 0.14285714285714285, "9737": 1.0, "9771": 0.0, "9808": 0.5, "9824": 1.0, "9871": 0.5, "9882": 0.0, "9925": 1.0, "9929": 0.5, "9961": 0.0, "9979": 1.0, "10034": 0.3333333333333333, "10039": 0.5, "10109": 1.0, "10122": 0.3333333333333333, "10136": 0.25, "10137": 0.14285714285714285, "10152": 1.0, "10183": 0.3333333333333333, "10213": 1.0, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.2, "10462": 0.0, "10482": 0.0, "10497": 1.0, "10526": 0.14285714285714285, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 0.1111111111111111, "10639": 0.5, "10645": 0.5, "10674": 1.0, "10710": 0.14285714285714285, "10734": 1.0, "10792": 0.25, "10808": 0.2, "10809": 1.0, "10812": 1.0, "10827": 0.25, "10845": 1.0, "10912": 0.08333333333333333, "10932": 1.0, "10975": 0.0, "10979": 1.0, "10994": 0.3333333333333333, "11039": 0.5, "11054": 1.0, "11088": 0.25}}} diff --git a/evals/benchmark/results/fiqa/ir-w0.75.jsonl b/evals/benchmark/results/fiqa/ir-w0.75.jsonl new file mode 100644 index 000000000..8d217a03c --- /dev/null +++ b/evals/benchmark/results/fiqa/ir-w0.75.jsonl @@ -0,0 +1 @@ +{"dataset": "fiqa", "run_tag": "w0.75", "aggregated": {"nDCG@10": 0.4289, "Recall@20": 0.6186, "MRR@10": 0.4986}, "per_query": {"nDCG@10": {"8": 0.6131471927654584, "15": 0.3562071871080222, "18": 0.6309297535714575, "26": 0.23719771276929622, "34": 0.0, "42": 0.46927872602275644, "56": 0.3562071871080222, "68": 0.0, "89": 0.286381299268402, "90": 0.6309297535714575, "94": 1.0, "98": 0.6131471927654584, "104": 0.6131471927654584, "106": 0.5, "109": 1.0, "475": 1.0, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.3333333333333333, "559": 0.0, "570": 0.0, "585": 0.2640681225725909, "588": 0.5714285040141098, "594": 0.6131471927654584, "603": 0.0, "604": 0.3235866969737877, "620": 0.0, "622": 0.38685280723454163, "659": 0.43828689407543014, "672": 0.0, "684": 0.0, "687": 0.19342640361727081, "689": 0.0, "691": 0.6309297535714575, "699": 1.0, "701": 0.13565197343244778, "715": 0.0, "721": 1.0, "744": 0.18154179253735267, "750": 0.8503449055347546, "753": 0.38685280723454163, "766": 0.18154179253735267, "776": 0.3557772116892465, "810": 0.3333333333333333, "813": 0.20438239758848611, "849": 0.3333333333333333, "852": 0.7653606369886217, "853": 0.0, "858": 0.0, "859": 0.38685280723454163, "864": 0.7247145167543899, "879": 0.6309297535714575, "885": 0.19342640361727081, "895": 1.0, "904": 1.0, "928": 0.6131471927654584, "929": 1.0, "932": 0.3065735963827292, "939": 0.5, "945": 0.3010299956639812, "957": 0.7039180890341347, "988": 0.0, "1074": 0.6309297535714575, "1085": 0.0, "1090": 0.23719771276929622, "1150": 0.0, "1157": 0.6131471927654584, "1159": 0.0, "1198": 1.0, "1230": 0.0, "1281": 0.38685280723454163, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.5319460297682747, "1310": 1.0, "1321": 0.0, "1322": 0.24630238874073, "1391": 0.18154179253735267, "1393": 0.19092086617893467, "1415": 0.9828920819566879, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.431733884398313, "1670": 0.7653606369886217, "1676": 0.6542448095688422, "1736": 0.5531464700081437, "1748": 0.6049306994552042, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.286381299268402, "1824": 0.31546487678572877, "1826": 1.0, "1832": 0.5, "1871": 0.6422813708518126, "1877": 1.0, "1889": 0.0, "1915": 0.8772153153380493, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.21398626473452756, "2010": 0.8194270280062366, "2051": 0.0, "2070": 0.23463936301137822, "2075": 0.4880470105041527, "2076": 0.5706417189553201, "2088": 0.0, "2108": 0.16716045496620227, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 0.7526136409516659, "2204": 0.9830168297915423, "2264": 0.38685280723454163, "2296": 0.33627415762678553, "2306": 0.6131471927654584, "2316": 1.0, "2318": 0.31546487678572877, "2330": 0.6131471927654584, "2334": 0.46927872602275644, "2348": 0.43231813227099475, "2376": 0.8910635073822442, "2383": 0.6131471927654584, "2384": 0.6309297535714575, "2385": 0.6131471927654584, "2388": 1.0, "2395": 1.0, "2398": 0.0, "2399": 1.0, "2400": 0.5307212739772434, "2407": 0.24630238874073, "2416": 0.9058653015743079, "2423": 0.3576223542196109, "2443": 0.0, "2445": 1.0, "2460": 0.9469024295259745, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 0.43067655807339306, "2513": 0.0, "2516": 0.5585075862632192, "2549": 0.6131471927654584, "2551": 1.0, "2568": 0.21689524588991285, "2579": 0.2423936099728648, "2580": 0.46927872602275644, "2587": 1.0, "2589": 0.7653606369886217, "2590": 0.46927872602275644, "2593": 0.8854598815714874, "2598": 0.8503449055347546, "2648": 1.0, "2676": 0.5, "2685": 0.35079429740281753, "2695": 1.0, "2713": 0.6131471927654584, "2724": 0.8175295903539445, "2737": 0.6309297535714575, "2747": 0.5, "2749": 0.8772153153380493, "2790": 0.47997295436377346, "2801": 0.6131471927654584, "2856": 0.7095272044910244, "2857": 0.6595629128335778, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.3010299956639812, "2923": 0.8207658584763088, "2964": 0.0, "2968": 0.6509209298071326, "2994": 0.46927872602275644, "3006": 0.25909036127391766, "3008": 0.24630238874073, "3014": 1.0, "3033": 0.3065735963827292, "3039": 0.43067655807339306, "3049": 0.24630238874073, "3051": 0.9197207891481876, "3067": 0.0, "3085": 0.19342640361727081, "3091": 0.9020870746500321, "3103": 0.5, "3115": 0.4935232796777481, "3125": 0.6309297535714575, "3148": 0.5531464700081437, "3149": 0.0, "3177": 0.15642624200758548, "3179": 0.8315546295836225, "3186": 1.0, "3189": 0.0, "3254": 0.6713860725233041, "3264": 0.8048099750039491, "3357": 1.0, "3369": 0.0, "3394": 0.5805485932475565, "3404": 0.3932591258696538, "3405": 0.0, "3446": 0.4340327988596634, "3451": 0.07945707943276742, "3453": 0.5706417189553201, "3480": 1.0, "3490": 0.2890648263178879, "3500": 0.644824486427155, "3503": 0.6183313187362977, "3512": 0.8503449055347546, "3528": 0.6309297535714575, "3530": 0.4776237035032179, "3534": 0.0, "3566": 0.8194270280062366, "3569": 0.5, "3594": 0.123151194370365, "3612": 0.11751610475642714, "3615": 0.430624116386567, "3625": 0.20210734650054757, "3682": 0.20438239758848611, "3683": 0.5982291239616433, "3694": 0.6131471927654584, "3724": 0.0, "3735": 0.0, "3759": 0.0, "3767": 0.2716774977597197, "3771": 0.19632790425573848, "3781": 0.9197207891481876, "3789": 0.75369761125927, "3791": 0.431733884398313, "3801": 0.0, "3822": 0.0, "3829": 0.0, "3830": 0.6309297535714575, "3837": 0.6309297535714575, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5517854393872873, "3932": 0.0, "3934": 0.16812753627111746, "3995": 0.0, "4007": 0.7653606369886217, "4011": 0.9674679834891693, "4019": 0.31608365985743925, "4031": 0.0, "4037": 0.18457569677956817, "4047": 0.8318724637288826, "4071": 0.0, "4084": 0.5012658353418871, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 0.6131471927654584, "4125": 0.0, "4142": 0.39233405721184117, "4153": 0.31546487678572877, "4179": 0.6309297535714575, "4188": 0.0, "4205": 0.4525081529734507, "4233": 0.8687949224876582, "4265": 0.7751482523375255, "4286": 0.6131471927654584, "4289": 0.23719771276929622, "4306": 0.7598336331031966, "4312": 0.7837042816946693, "4335": 0.8318724637288826, "4339": 1.0, "4394": 0.0, "4409": 0.19518900079066107, "4411": 0.46927872602275644, "4414": 0.8315546295836225, "4415": 0.0, "4433": 0.0, "4447": 1.0, "4464": 0.5, "4465": 0.2960819109658652, "4484": 0.6309297535714575, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 0.4034698846650846, "4523": 0.21398626473452756, "4539": 0.21840743681816419, "4571": 0.7653606369886217, "4600": 1.0, "4605": 0.5585830187272559, "4615": 0.4816421678943211, "4640": 0.0, "4641": 0.252016380211836, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75369761125927, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.6131471927654584, "4813": 1.0, "4823": 0.46927872602275644, "4827": 0.3562071871080222, "4837": 0.0, "4844": 1.0, "4845": 0.7653606369886217, "4846": 0.0, "4863": 0.49818925746641285, "4865": 0.0, "4920": 1.0, "4942": 0.31546487678572877, "4946": 1.0, "4955": 0.0, "4962": 0.6131471927654584, "4968": 0.6309297535714575, "4981": 0.7653606369886217, "4999": 0.5294362295028773, "5021": 0.0, "5030": 0.0, "5045": 0.3010299956639812, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.5, "5080": 0.0, "5083": 0.23719771276929622, "5085": 1.0, "5086": 0.6309297535714575, "5090": 0.2640681225725909, "5125": 1.0, "5134": 0.38685280723454163, "5150": 0.0, "5155": 1.0, "5172": 0.0, "5178": 0.252016380211836, "5185": 0.0, "5196": 0.2960819109658652, "5206": 0.0, "5228": 0.31546487678572877, "5231": 0.0, "5241": 0.44615333764087994, "5254": 0.6131471927654584, "5255": 1.0, "5264": 0.6843515475204855, "5271": 0.41618115554873086, "5331": 1.0, "5343": 0.3333333333333333, "5347": 1.0, "5356": 0.1208113026994942, "5369": 0.0, "5374": 0.3010299956639812, "5380": 1.0, "5402": 0.0, "5410": 0.2960819109658652, "5422": 0.19519002499605084, "5427": 0.23719771276929622, "5460": 0.0, "5464": 0.16716045496620227, "5503": 0.38685280723454163, "5505": 1.0, "5511": 0.1610425878239398, "5534": 0.38685280723454163, "5549": 0.0, "5585": 0.31546487678572877, "5592": 1.0, "5616": 0.3562071871080222, "5620": 0.0, "5646": 0.9217868789962071, "5653": 1.0, "5683": 0.9197207891481876, "5710": 0.0, "5741": 0.0, "5763": 0.23463936301137822, "5782": 0.4371994911049732, "5790": 0.0, "5808": 0.5, "5853": 0.21078558563971425, "5862": 0.29127873064148246, "5888": 0.2640681225725909, "5903": 0.0, "5906": 0.44130740935663865, "5940": 0.0, "5951": 0.42030067999766585, "5970": 0.23463936301137822, "5981": 0.9896062251871525, "5993": 0.31488013066763093, "6002": 0.762416491594312, "6004": 0.0, "6005": 0.5388856921828065, "6009": 0.0, "6041": 0.7668091220635322, "6080": 0.6131471927654584, "6110": 0.41918605213740834, "6121": 0.3065735963827292, "6122": 0.46927872602275644, "6131": 0.5810820381115676, "6133": 0.6131471927654584, "6142": 0.6131471927654584, "6146": 0.8315546295836225, "6199": 0.5135312443624667, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.49818925746641285, "6278": 0.2640681225725909, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.38685280723454163, "6467": 0.0, "6468": 0.6131471927654584, "6479": 1.0, "6525": 0.49818925746641285, "6554": 0.6131471927654584, "6562": 0.11751610475642714, "6611": 0.7903864795495061, "6612": 0.16716045496620227, "6625": 0.0, "6629": 0.0, "6635": 0.6366824387328317, "6644": 0.0, "6647": 0.7653606369886217, "6668": 0.8503449055347546, "6679": 0.4632423659320675, "6683": 0.8772153153380493, "6713": 0.6131471927654584, "6715": 0.6131471927654584, "6746": 0.2640681225725909, "6787": 0.0, "6792": 0.0, "6800": 0.43067655807339306, "6803": 0.5135312443624667, "6807": 1.0, "6814": 0.0, "6832": 0.430624116386567, "6835": 0.6131471927654584, "6849": 0.5, "6862": 1.0, "6867": 0.22471947398559278, "6875": 0.0, "6890": 0.5012658353418871, "6891": 0.617319681505689, "6896": 0.0, "6901": 0.21840743681816419, "6907": 0.0, "6909": 0.0, "6959": 0.38685280723454163, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 0.8503449055347546, "7096": 0.6309297535714575, "7098": 0.6131471927654584, "7105": 0.38685280723454163, "7109": 0.2960819109658652, "7124": 0.5085097626218706, "7141": 0.0, "7145": 0.13565197343244778, "7178": 0.3562071871080222, "7188": 0.31546487678572877, "7205": 1.0, "7206": 0.0, "7218": 0.43067655807339306, "7221": 0.0, "7269": 0.0, "7279": 0.44229834886928765, "7295": 0.0, "7311": 0.0, "7326": 0.0, "7329": 0.46845052016107697, "7344": 0.6131471927654584, "7345": 0.15642624200758548, "7377": 1.0, "7431": 0.0, "7441": 0.20210734650054757, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.14606834984270645, "7467": 0.6309297535714575, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.16716045496620227, "7529": 0.3065735963827292, "7533": 0.0, "7534": 0.46927872602275644, "7590": 0.7977228895450266, "7592": 0.0, "7594": 0.3333333333333333, "7622": 0.0, "7633": 0.3903800499921017, "7674": 0.0, "7700": 0.123151194370365, "7702": 0.38685280723454163, "7705": 0.49818925746641285, "7734": 0.6131471927654584, "7747": 0.20438239758848611, "7754": 0.0, "7758": 0.6309297535714575, "7801": 1.0, "7803": 0.19342640361727081, "7823": 0.6105456988825855, "7876": 0.38685280723454163, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.19092086617893467, "7928": 0.24630238874073, "7936": 0.40179981797758046, "7992": 0.38685280723454163, "8002": 0.6257049680303419, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.31546487678572877, "8040": 1.0, "8072": 0.4776237035032179, "8079": 0.7039180890341347, "8102": 0.11284514134893527, "8116": 0.45749452618986164, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 0.6131471927654584, "8275": 0.43067655807339306, "8296": 0.8315546295836225, "8332": 0.9060254355346823, "8351": 0.6131471927654584, "8378": 0.13565197343244778, "8456": 0.6049306994552042, "8475": 0.7957503936198023, "8507": 0.0, "8512": 1.0, "8513": 0.0, "8532": 0.9325210919548239, "8537": 0.3562071871080222, "8539": 0.0, "8544": 0.6131471927654584, "8592": 0.23463936301137822, "8632": 0.21840743681816419, "8635": 0.0, "8702": 0.7653606369886217, "8779": 0.17723928678404774, "8789": 0.7653606369886217, "8795": 1.0, "8832": 1.0, "8834": 0.3980628465882808, "8855": 0.0, "8874": 0.3333333333333333, "8934": 0.8065735963827293, "8937": 0.3562071871080222, "8947": 0.23719771276929622, "8959": 0.43067655807339306, "8970": 0.8772153153380493, "8974": 0.41253177989255346, "8982": 0.0, "9060": 0.6131471927654584, "9088": 0.0, "9108": 0.0, "9115": 0.5256940434743352, "9126": 0.0, "9164": 0.1480409554829326, "9174": 0.0, "9188": 0.6131471927654584, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.20438239758848611, "9329": 1.0, "9332": 0.3562071871080222, "9381": 0.5706417189553201, "9385": 0.3562071871080222, "9391": 0.5143371794949736, "9403": 0.0, "9481": 0.6240505200038379, "9487": 0.0, "9548": 0.43067655807339306, "9556": 1.0, "9565": 1.0, "9598": 0.3562071871080222, "9617": 0.3152014104491349, "9633": 0.6131471927654584, "9643": 0.5, "9644": 0.6309297535714575, "9646": 0.45749452618986164, "9668": 0.6508205185601091, "9701": 0.0, "9733": 0.18154179253735267, "9735": 0.3333333333333333, "9737": 0.7653606369886217, "9771": 0.0, "9808": 0.2960819109658652, "9824": 1.0, "9871": 0.24630238874073, "9882": 0.0, "9925": 0.8315546295836225, "9929": 0.5714285040141098, "9961": 0.0, "9979": 0.3903800499921017, "10034": 0.3065735963827292, "10039": 0.6309297535714575, "10109": 0.5399294342072939, "10122": 0.19519002499605084, "10136": 0.34337431936037655, "10137": 0.3333333333333333, "10152": 1.0, "10183": 0.3065735963827292, "10213": 0.46927872602275644, "10246": 0.0, "10267": 0.0, "10414": 0.9197207891481876, "10447": 0.18154179253735267, "10462": 0.0, "10482": 0.0, "10497": 0.49828993591603105, "10526": 0.20438239758848611, "10547": 0.0, "10558": 0.0, "10596": 0.9427731067700877, "10601": 1.0, "10628": 0.3010299956639812, "10639": 0.3120255505658846, "10645": 0.38685280723454163, "10674": 0.7095272044910244, "10710": 0.3333333333333333, "10734": 0.8486987932505933, "10792": 0.4227898344066503, "10808": 0.4652347214527331, "10809": 0.6049306994552042, "10812": 0.9619991470595832, "10827": 0.14606834984270645, "10845": 1.0, "10912": 0.0, "10932": 1.0, "10975": 0.0, "10979": 0.6366824387328317, "10994": 0.4911492931622974, "11039": 0.315526956404728, "11054": 1.0, "11088": 0.43067655807339306}, "Recall@20": {"8": 0.5, "15": 1.0, "18": 1.0, "26": 0.5, "34": 0.0, "42": 0.3333333333333333, "56": 1.0, "68": 0.0, "89": 0.5, "90": 1.0, "94": 1.0, "98": 0.5, "104": 0.5, "106": 1.0, "109": 1.0, "475": 1.0, "503": 1.0, "504": 0.3333333333333333, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 1.0, "585": 0.5, "588": 1.0, "594": 1.0, "603": 0.0, "604": 1.0, "620": 0.2, "622": 0.5, "659": 0.7, "672": 0.5, "684": 1.0, "687": 0.5, "689": 0.0, "691": 1.0, "699": 1.0, "701": 0.3333333333333333, "715": 0.0, "721": 1.0, "744": 0.3333333333333333, "750": 1.0, "753": 0.5, "766": 0.3333333333333333, "776": 0.25, "810": 1.0, "813": 1.0, "849": 1.0, "852": 0.6666666666666666, "853": 0.5, "858": 0.0, "859": 0.5, "864": 0.6666666666666666, "879": 1.0, "885": 0.5, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.5, "939": 1.0, "945": 1.0, "957": 0.6666666666666666, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.5, "1150": 0.0, "1157": 0.5, "1159": 0.0, "1198": 1.0, "1230": 1.0, "1281": 1.0, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.75, "1310": 1.0, "1321": 0.0, "1322": 0.25, "1391": 0.6666666666666666, "1393": 0.5, "1415": 1.0, "1416": 1.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 1.0, "1670": 0.6666666666666666, "1676": 1.0, "1736": 0.4, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 1.0, "1826": 1.0, "1832": 1.0, "1871": 0.75, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.4, "2010": 1.0, "2051": 1.0, "2070": 0.6666666666666666, "2075": 0.6666666666666666, "2076": 1.0, "2088": 0.0, "2108": 0.3333333333333333, "2118": 0.0, "2154": 0.0, "2181": 0.5, "2183": 0.8333333333333334, "2204": 1.0, "2264": 0.5, "2296": 0.2222222222222222, "2306": 0.5, "2316": 1.0, "2318": 1.0, "2330": 0.5, "2334": 0.6666666666666666, "2348": 0.26666666666666666, "2376": 1.0, "2383": 0.5, "2384": 1.0, "2385": 0.5, "2388": 1.0, "2395": 1.0, "2398": 0.2, "2399": 1.0, "2400": 0.6666666666666666, "2407": 0.5, "2416": 1.0, "2423": 0.8571428571428571, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.3333333333333333, "2472": 0.0, "2486": 1.0, "2498": 1.0, "2513": 1.0, "2516": 0.75, "2549": 0.5, "2551": 1.0, "2568": 0.2857142857142857, "2579": 0.5, "2580": 0.3333333333333333, "2587": 1.0, "2589": 0.6666666666666666, "2590": 0.6666666666666666, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 1.0, "2685": 0.25, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 1.0, "2790": 0.42857142857142855, "2801": 1.0, "2856": 0.75, "2857": 0.75, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 1.0, "2923": 0.8, "2964": 0.5, "2968": 1.0, "2994": 0.3333333333333333, "3006": 0.5, "3008": 0.25, "3014": 1.0, "3033": 0.5, "3039": 1.0, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 1.0, "3091": 0.8571428571428571, "3103": 1.0, "3115": 0.5, "3125": 1.0, "3148": 0.6, "3149": 0.0, "3177": 0.6666666666666666, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.6666666666666666, "3264": 0.75, "3357": 1.0, "3369": 0.16666666666666666, "3394": 0.75, "3404": 0.42857142857142855, "3405": 1.0, "3446": 0.8, "3451": 0.42857142857142855, "3453": 1.0, "3480": 1.0, "3490": 1.0, "3500": 0.5, "3503": 0.7142857142857143, "3512": 1.0, "3528": 1.0, "3530": 0.6666666666666666, "3534": 0.0, "3566": 1.0, "3569": 1.0, "3594": 0.25, "3612": 0.75, "3615": 1.0, "3625": 0.3333333333333333, "3682": 1.0, "3683": 0.6666666666666666, "3694": 0.5, "3724": 0.0, "3735": 1.0, "3759": 0.0, "3767": 0.4, "3771": 0.3333333333333333, "3781": 1.0, "3789": 0.75, "3791": 0.6666666666666666, "3801": 0.0, "3822": 0.25, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 0.6666666666666666, "4011": 1.0, "4019": 0.4, "4031": 0.0, "4037": 0.5, "4047": 0.75, "4071": 0.0, "4084": 1.0, "4102": 0.25, "4103": 0.0, "4105": 0.0, "4116": 0.5, "4125": 0.0, "4142": 0.75, "4153": 1.0, "4179": 1.0, "4188": 1.0, "4205": 0.6666666666666666, "4233": 0.8, "4265": 0.6666666666666666, "4286": 0.5, "4289": 0.5, "4306": 0.75, "4312": 0.8571428571428571, "4335": 0.75, "4339": 1.0, "4394": 0.0, "4409": 0.2, "4411": 0.6666666666666666, "4414": 1.0, "4415": 0.0, "4433": 0.5, "4447": 1.0, "4464": 1.0, "4465": 0.6666666666666666, "4484": 1.0, "4499": 0.3333333333333333, "4500": 0.5, "4504": 1.0, "4514": 0.5, "4523": 0.2, "4539": 0.5, "4571": 0.6666666666666666, "4600": 1.0, "4605": 0.7142857142857143, "4615": 0.6666666666666666, "4640": 0.0, "4641": 0.6, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 0.6666666666666666, "4846": 0.0, "4863": 1.0, "4865": 0.0, "4920": 1.0, "4942": 1.0, "4946": 1.0, "4955": 0.0, "4962": 0.5, "4968": 1.0, "4981": 0.6666666666666666, "4999": 0.75, "5021": 0.0, "5030": 0.0, "5045": 1.0, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 1.0, "5080": 0.0, "5083": 0.5, "5085": 1.0, "5086": 1.0, "5090": 0.5, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 1.0, "5178": 0.4, "5185": 0.0, "5196": 0.3333333333333333, "5206": 0.0, "5228": 1.0, "5231": 1.0, "5241": 0.4, "5254": 0.5, "5255": 1.0, "5264": 0.6, "5271": 1.0, "5331": 1.0, "5343": 1.0, "5347": 1.0, "5356": 0.4, "5369": 0.0, "5374": 1.0, "5380": 1.0, "5402": 0.5, "5410": 0.3333333333333333, "5422": 0.75, "5427": 0.5, "5460": 0.0, "5464": 0.3333333333333333, "5503": 0.5, "5505": 1.0, "5511": 0.3, "5534": 0.5, "5549": 0.0, "5585": 1.0, "5592": 1.0, "5616": 1.0, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.5, "5763": 0.6666666666666666, "5782": 0.4, "5790": 0.0, "5808": 1.0, "5853": 0.375, "5862": 0.5, "5888": 0.5, "5903": 0.0, "5906": 1.0, "5940": 0.0, "5951": 0.5555555555555556, "5970": 0.6666666666666666, "5981": 1.0, "5993": 0.2, "6002": 0.9090909090909091, "6004": 0.0, "6005": 0.38461538461538464, "6009": 1.0, "6041": 0.75, "6080": 0.5, "6110": 0.375, "6121": 0.5, "6122": 0.3333333333333333, "6131": 0.4166666666666667, "6133": 0.5, "6142": 0.5, "6146": 1.0, "6199": 0.5, "6219": 1.0, "6221": 0.125, "6252": 0.0, "6262": 0.6666666666666666, "6278": 1.0, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 1.0, "6467": 0.0, "6468": 0.5, "6479": 1.0, "6525": 0.6666666666666666, "6554": 0.5, "6562": 0.5, "6611": 1.0, "6612": 0.3333333333333333, "6625": 1.0, "6629": 0.0, "6635": 0.5, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 1.0, "6683": 1.0, "6713": 0.5, "6715": 1.0, "6746": 0.5, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.75, "6807": 1.0, "6814": 0.0, "6832": 1.0, "6835": 0.5, "6849": 1.0, "6862": 1.0, "6867": 0.42857142857142855, "6875": 0.0, "6890": 1.0, "6891": 0.6666666666666666, "6896": 0.0, "6901": 0.5, "6907": 0.0, "6909": 0.0, "6959": 1.0, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 1.0, "7109": 0.6666666666666666, "7124": 0.75, "7141": 0.5, "7145": 0.3333333333333333, "7178": 1.0, "7188": 1.0, "7205": 1.0, "7206": 0.0, "7218": 1.0, "7221": 0.0, "7269": 1.0, "7279": 0.75, "7295": 0.0, "7311": 1.0, "7326": 0.0, "7329": 1.0, "7344": 0.5, "7345": 0.6666666666666666, "7377": 1.0, "7431": 0.0, "7441": 0.3333333333333333, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.4, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 1.0, "7513": 0.6666666666666666, "7529": 0.5, "7533": 0.5, "7534": 0.3333333333333333, "7590": 1.0, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 0.5, "7674": 0.3333333333333333, "7700": 0.25, "7702": 1.0, "7705": 0.6666666666666666, "7734": 0.5, "7747": 0.5, "7754": 1.0, "7758": 1.0, "7801": 1.0, "7803": 0.5, "7823": 0.6666666666666666, "7876": 1.0, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.16666666666666666, "7928": 0.25, "7936": 0.6666666666666666, "7992": 1.0, "8002": 0.6666666666666666, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 1.0, "8040": 1.0, "8072": 0.6666666666666666, "8079": 1.0, "8102": 0.25, "8116": 1.0, "8121": 1.0, "8202": 0.0, "8230": 1.0, "8247": 0.0, "8271": 0.5, "8275": 1.0, "8296": 1.0, "8332": 1.0, "8351": 0.5, "8378": 0.3333333333333333, "8456": 0.6666666666666666, "8475": 0.8571428571428571, "8507": 0.3333333333333333, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 1.0, "8539": 0.25, "8544": 0.5, "8592": 0.6666666666666666, "8632": 1.0, "8635": 0.0, "8702": 0.6666666666666666, "8779": 1.0, "8789": 0.6666666666666666, "8795": 1.0, "8832": 1.0, "8834": 0.5, "8855": 0.0, "8874": 1.0, "8934": 1.0, "8937": 1.0, "8947": 0.5, "8959": 1.0, "8970": 1.0, "8974": 0.25, "8982": 0.0, "9060": 0.5, "9088": 0.0, "9108": 0.0, "9115": 1.0, "9126": 0.0, "9164": 0.3333333333333333, "9174": 0.2, "9188": 0.5, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.5, "9329": 1.0, "9332": 1.0, "9381": 1.0, "9385": 1.0, "9391": 0.75, "9403": 0.0, "9481": 1.0, "9487": 1.0, "9548": 1.0, "9556": 1.0, "9565": 1.0, "9598": 1.0, "9617": 0.6666666666666666, "9633": 0.5, "9643": 1.0, "9644": 1.0, "9646": 1.0, "9668": 1.0, "9701": 0.0, "9733": 0.3333333333333333, "9735": 1.0, "9737": 0.6666666666666666, "9771": 0.0, "9808": 0.3333333333333333, "9824": 1.0, "9871": 0.25, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 0.0, "9979": 0.5, "10034": 0.5, "10039": 1.0, "10109": 0.42857142857142855, "10122": 0.5, "10136": 0.6666666666666666, "10137": 1.0, "10152": 1.0, "10183": 1.0, "10213": 0.3333333333333333, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.3333333333333333, "10462": 0.0, "10482": 0.0, "10497": 0.7, "10526": 0.5, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 1.0, "10639": 0.4, "10645": 1.0, "10674": 1.0, "10710": 1.0, "10734": 1.0, "10792": 1.0, "10808": 1.0, "10809": 0.6666666666666666, "10812": 1.0, "10827": 0.2, "10845": 1.0, "10912": 1.0, "10932": 1.0, "10975": 0.0, "10979": 0.75, "10994": 1.0, "11039": 0.5, "11054": 1.0, "11088": 1.0}, "MRR@10": {"8": 1.0, "15": 0.16666666666666666, "18": 0.5, "26": 0.2, "34": 0.0, "42": 1.0, "56": 0.16666666666666666, "68": 0.0, "89": 0.5, "90": 0.5, "94": 1.0, "98": 1.0, "104": 1.0, "106": 0.3333333333333333, "109": 1.0, "475": 1.0, "503": 0.08333333333333333, "504": 0.05555555555555555, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.14285714285714285, "559": 0.0, "570": 0.06666666666666667, "585": 0.25, "588": 0.5, "594": 1.0, "603": 0.0, "604": 0.16666666666666666, "620": 0.08333333333333333, "622": 0.5, "659": 1.0, "672": 0.07692307692307693, "684": 0.05555555555555555, "687": 0.125, "689": 0.0, "691": 0.5, "699": 1.0, "701": 0.1, "715": 0.0, "721": 1.0, "744": 0.2, "750": 1.0, "753": 0.5, "766": 0.2, "776": 1.0, "810": 0.14285714285714285, "813": 0.14285714285714285, "849": 0.14285714285714285, "852": 1.0, "853": 0.09090909090909091, "858": 0.0, "859": 0.5, "864": 1.0, "879": 0.5, "885": 0.125, "895": 1.0, "904": 1.0, "928": 1.0, "929": 1.0, "932": 0.3333333333333333, "939": 0.3333333333333333, "945": 0.1111111111111111, "957": 1.0, "988": 0.0, "1074": 0.5, "1085": 0.0, "1090": 0.2, "1150": 0.0, "1157": 1.0, "1159": 0.0, "1198": 1.0, "1230": 0.07142857142857142, "1281": 0.2, "1284": 1.0, "1297": 1.0, "1306": 0.0, "1309": 0.5, "1310": 1.0, "1321": 0.0, "1322": 0.5, "1391": 0.2, "1393": 0.5, "1415": 1.0, "1416": 0.058823529411764705, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.5, "1670": 1.0, "1676": 1.0, "1736": 1.0, "1748": 1.0, "1783": 0.0, "1812": 0.0, "1815": 1.0, "1819": 0.5, "1824": 0.125, "1826": 1.0, "1832": 0.3333333333333333, "1871": 1.0, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.5, "2010": 1.0, "2051": 0.07142857142857142, "2070": 0.3333333333333333, "2075": 1.0, "2076": 0.3333333333333333, "2088": 0.0, "2108": 0.16666666666666666, "2118": 0.0, "2154": 0.0, "2181": 0.05263157894736842, "2183": 1.0, "2204": 1.0, "2264": 0.5, "2296": 1.0, "2306": 1.0, "2316": 1.0, "2318": 0.125, "2330": 1.0, "2334": 1.0, "2348": 1.0, "2376": 1.0, "2383": 1.0, "2384": 0.5, "2385": 1.0, "2388": 1.0, "2395": 1.0, "2398": 0.08333333333333333, "2399": 1.0, "2400": 0.5, "2407": 0.5, "2416": 1.0, "2423": 1.0, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0625, "2472": 0.0, "2486": 1.0, "2498": 0.25, "2513": 0.0625, "2516": 1.0, "2549": 1.0, "2551": 1.0, "2568": 0.3333333333333333, "2579": 0.3333333333333333, "2580": 1.0, "2587": 1.0, "2589": 1.0, "2590": 1.0, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 0.3333333333333333, "2685": 1.0, "2695": 1.0, "2713": 1.0, "2724": 1.0, "2737": 0.5, "2747": 0.3333333333333333, "2749": 1.0, "2790": 1.0, "2801": 1.0, "2856": 1.0, "2857": 1.0, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.1111111111111111, "2923": 1.0, "2964": 0.07692307692307693, "2968": 0.5, "2994": 1.0, "3006": 0.3333333333333333, "3008": 0.5, "3014": 1.0, "3033": 0.3333333333333333, "3039": 0.25, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 0.125, "3091": 1.0, "3103": 0.3333333333333333, "3115": 1.0, "3125": 0.5, "3148": 1.0, "3149": 0.0, "3177": 0.14285714285714285, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 1.0, "3264": 1.0, "3357": 1.0, "3369": 0.09090909090909091, "3394": 0.5, "3404": 1.0, "3405": 0.09090909090909091, "3446": 0.5, "3451": 0.1, "3453": 0.3333333333333333, "3480": 1.0, "3490": 0.1, "3500": 1.0, "3503": 0.5, "3512": 1.0, "3528": 0.5, "3530": 0.5, "3534": 0.0, "3566": 1.0, "3569": 0.3333333333333333, "3594": 0.125, "3612": 0.1111111111111111, "3615": 0.2, "3625": 0.25, "3682": 0.14285714285714285, "3683": 1.0, "3694": 1.0, "3724": 0.0, "3735": 0.09090909090909091, "3759": 0.0, "3767": 0.3333333333333333, "3771": 0.14285714285714285, "3781": 1.0, "3789": 1.0, "3791": 0.5, "3801": 0.0, "3822": 0.05263157894736842, "3829": 0.0, "3830": 0.5, "3837": 0.5, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 1.0, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 1.0, "4011": 1.0, "4019": 0.5, "4031": 0.0, "4037": 0.1111111111111111, "4047": 1.0, "4071": 0.0, "4084": 0.25, "4102": 0.06666666666666667, "4103": 0.0, "4105": 0.0, "4116": 1.0, "4125": 0.0, "4142": 0.16666666666666666, "4153": 0.125, "4179": 0.5, "4188": 0.0625, "4205": 0.5, "4233": 1.0, "4265": 1.0, "4286": 1.0, "4289": 0.2, "4306": 1.0, "4312": 1.0, "4335": 1.0, "4339": 1.0, "4394": 0.0, "4409": 0.3333333333333333, "4411": 1.0, "4414": 1.0, "4415": 0.0, "4433": 0.09090909090909091, "4447": 1.0, "4464": 0.3333333333333333, "4465": 0.5, "4484": 0.5, "4499": 0.05555555555555555, "4500": 0.07142857142857142, "4504": 1.0, "4514": 1.0, "4523": 0.5, "4539": 0.16666666666666666, "4571": 1.0, "4600": 1.0, "4605": 1.0, "4615": 0.5, "4640": 0.0, "4641": 0.2, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 1.0, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 0.16666666666666666, "4837": 0.0, "4844": 1.0, "4845": 1.0, "4846": 0.0, "4863": 0.5, "4865": 0.0, "4920": 1.0, "4942": 0.125, "4946": 1.0, "4955": 0.0, "4962": 1.0, "4968": 0.5, "4981": 1.0, "4999": 1.0, "5021": 0.0, "5030": 0.0, "5045": 0.1111111111111111, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.3333333333333333, "5080": 0.0, "5083": 0.2, "5085": 1.0, "5086": 0.5, "5090": 0.25, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 0.09090909090909091, "5178": 0.2, "5185": 0.0, "5196": 0.5, "5206": 0.0, "5228": 0.125, "5231": 0.058823529411764705, "5241": 1.0, "5254": 1.0, "5255": 1.0, "5264": 1.0, "5271": 0.3333333333333333, "5331": 1.0, "5343": 0.14285714285714285, "5347": 1.0, "5356": 0.16666666666666666, "5369": 0.0, "5374": 0.1111111111111111, "5380": 1.0, "5402": 0.0625, "5410": 0.5, "5422": 0.3333333333333333, "5427": 0.2, "5460": 0.0, "5464": 0.16666666666666666, "5503": 0.5, "5505": 1.0, "5511": 0.25, "5534": 0.5, "5549": 0.0, "5585": 0.125, "5592": 1.0, "5616": 0.16666666666666666, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.0625, "5763": 0.3333333333333333, "5782": 1.0, "5790": 0.0, "5808": 0.3333333333333333, "5853": 0.3333333333333333, "5862": 0.25, "5888": 0.25, "5903": 0.0, "5906": 0.25, "5940": 0.0, "5951": 0.5, "5970": 0.3333333333333333, "5981": 1.0, "5993": 1.0, "6002": 1.0, "6004": 0.0, "6005": 1.0, "6009": 0.09090909090909091, "6041": 1.0, "6080": 1.0, "6110": 1.0, "6121": 0.3333333333333333, "6122": 1.0, "6131": 1.0, "6133": 1.0, "6142": 1.0, "6146": 1.0, "6199": 1.0, "6219": 1.0, "6221": 0.05, "6252": 0.0, "6262": 0.5, "6278": 0.25, "6395": 0.0, "6410": 0.0, "6420": 1.0, "6441": 0.2, "6467": 0.0, "6468": 1.0, "6479": 1.0, "6525": 0.5, "6554": 1.0, "6562": 0.1111111111111111, "6611": 1.0, "6612": 0.16666666666666666, "6625": 0.08333333333333333, "6629": 0.0, "6635": 1.0, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 0.5, "6683": 1.0, "6713": 1.0, "6715": 1.0, "6746": 0.25, "6787": 0.0, "6792": 0.0, "6800": 0.25, "6803": 1.0, "6807": 1.0, "6814": 0.0, "6832": 0.2, "6835": 1.0, "6849": 0.3333333333333333, "6862": 1.0, "6867": 0.25, "6875": 0.0, "6890": 0.25, "6891": 1.0, "6896": 0.0, "6901": 0.16666666666666666, "6907": 0.0, "6909": 0.0, "6959": 0.5, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.0, "7080": 1.0, "7096": 0.5, "7098": 1.0, "7105": 0.2, "7109": 0.5, "7124": 0.5, "7141": 0.0625, "7145": 0.1, "7178": 0.16666666666666666, "7188": 0.125, "7205": 1.0, "7206": 0.0, "7218": 0.25, "7221": 0.0, "7269": 0.06666666666666667, "7279": 0.25, "7295": 0.0, "7311": 0.09090909090909091, "7326": 0.0, "7329": 0.25, "7344": 1.0, "7345": 0.14285714285714285, "7377": 1.0, "7431": 0.0, "7441": 0.25, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.25, "7467": 0.5, "7484": 1.0, "7509": 0.0, "7512": 0.0625, "7513": 0.16666666666666666, "7529": 0.3333333333333333, "7533": 0.08333333333333333, "7534": 1.0, "7590": 1.0, "7592": 0.0, "7594": 0.14285714285714285, "7622": 0.0, "7633": 1.0, "7674": 0.0625, "7700": 0.125, "7702": 0.5, "7705": 0.5, "7734": 1.0, "7747": 0.14285714285714285, "7754": 0.08333333333333333, "7758": 0.5, "7801": 1.0, "7803": 0.125, "7823": 1.0, "7876": 0.2, "7879": 0.0, "7880": 0.0, "7911": 1.0, "7925": 0.5, "7928": 0.5, "7936": 0.3333333333333333, "7992": 0.5, "8002": 1.0, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.125, "8040": 1.0, "8072": 0.5, "8079": 1.0, "8102": 0.1, "8116": 0.25, "8121": 1.0, "8202": 0.0, "8230": 0.0625, "8247": 0.0, "8271": 1.0, "8275": 0.25, "8296": 1.0, "8332": 1.0, "8351": 1.0, "8378": 0.1, "8456": 1.0, "8475": 1.0, "8507": 0.09090909090909091, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 0.16666666666666666, "8539": 0.09090909090909091, "8544": 1.0, "8592": 0.3333333333333333, "8632": 0.16666666666666666, "8635": 0.0, "8702": 1.0, "8779": 0.1, "8789": 1.0, "8795": 1.0, "8832": 1.0, "8834": 1.0, "8855": 0.0, "8874": 0.14285714285714285, "8934": 1.0, "8937": 0.16666666666666666, "8947": 0.2, "8959": 0.25, "8970": 1.0, "8974": 1.0, "8982": 0.0, "9060": 1.0, "9088": 0.0, "9108": 0.0, "9115": 0.25, "9126": 0.0, "9164": 0.125, "9174": 0.07142857142857142, "9188": 1.0, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.14285714285714285, "9329": 1.0, "9332": 0.16666666666666666, "9381": 0.3333333333333333, "9385": 0.16666666666666666, "9391": 0.3333333333333333, "9403": 0.0, "9481": 0.5, "9487": 0.08333333333333333, "9548": 0.25, "9556": 1.0, "9565": 1.0, "9598": 0.16666666666666666, "9617": 0.16666666666666666, "9633": 1.0, "9643": 0.3333333333333333, "9644": 0.5, "9646": 0.25, "9668": 1.0, "9701": 0.0, "9733": 0.2, "9735": 0.14285714285714285, "9737": 1.0, "9771": 0.0, "9808": 0.5, "9824": 1.0, "9871": 0.5, "9882": 0.0, "9925": 1.0, "9929": 0.5, "9961": 0.0, "9979": 1.0, "10034": 0.3333333333333333, "10039": 0.5, "10109": 1.0, "10122": 0.3333333333333333, "10136": 0.25, "10137": 0.14285714285714285, "10152": 1.0, "10183": 0.3333333333333333, "10213": 1.0, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.2, "10462": 0.0, "10482": 0.0, "10497": 1.0, "10526": 0.14285714285714285, "10547": 0.0, "10558": 0.0, "10596": 1.0, "10601": 1.0, "10628": 0.1111111111111111, "10639": 0.5, "10645": 0.5, "10674": 1.0, "10710": 0.14285714285714285, "10734": 1.0, "10792": 0.16666666666666666, "10808": 0.2, "10809": 1.0, "10812": 1.0, "10827": 0.25, "10845": 1.0, "10912": 0.08333333333333333, "10932": 1.0, "10975": 0.0, "10979": 1.0, "10994": 0.3333333333333333, "11039": 0.5, "11054": 1.0, "11088": 0.25}}} diff --git a/evals/benchmark/results/fiqa/ir-w1.0.jsonl b/evals/benchmark/results/fiqa/ir-w1.0.jsonl new file mode 100644 index 000000000..750245bf2 --- /dev/null +++ b/evals/benchmark/results/fiqa/ir-w1.0.jsonl @@ -0,0 +1 @@ +{"dataset": "fiqa", "run_tag": "w1.0", "aggregated": {"nDCG@10": 0.4033, "Recall@20": 0.5802, "MRR@10": 0.4777}, "per_query": {"nDCG@10": {"8": 0.6131471927654584, "15": 0.3562071871080222, "18": 0.6309297535714575, "26": 0.20438239758848611, "34": 0.0, "42": 0.46927872602275644, "56": 0.3333333333333333, "68": 0.0, "89": 0.27839258028034286, "90": 0.5, "94": 1.0, "98": 0.6131471927654584, "104": 0.38685280723454163, "106": 0.43067655807339306, "109": 1.0, "475": 0.6309297535714575, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.3333333333333333, "559": 0.0, "570": 0.0, "585": 0.23719771276929622, "588": 0.5640920940185894, "594": 0.6131471927654584, "603": 0.0, "604": 0.2769189462922768, "620": 0.0, "622": 0.38685280723454163, "659": 0.3785988060390612, "672": 0.0, "684": 0.0, "687": 0.0, "689": 0.0, "691": 0.6309297535714575, "699": 1.0, "701": 0.0, "715": 0.0, "721": 1.0, "744": 0.16716045496620227, "750": 0.8503449055347546, "753": 0.3065735963827292, "766": 0.1480409554829326, "776": 0.3499667779514209, "810": 0.2890648263178879, "813": 0.17723928678404774, "849": 0.31546487678572877, "852": 0.7653606369886217, "853": 0.0, "858": 0.0, "859": 0.3065735963827292, "864": 0.706046305905179, "879": 0.6309297535714575, "885": 0.0, "895": 1.0, "904": 0.6309297535714575, "928": 0.6131471927654584, "929": 1.0, "932": 0.3065735963827292, "939": 0.5, "945": 0.3010299956639812, "957": 0.7039180890341347, "988": 0.0, "1074": 0.5, "1085": 0.0, "1090": 0.21840743681816419, "1150": 0.0, "1157": 0.6131471927654584, "1159": 0.0, "1198": 1.0, "1230": 0.0, "1281": 0.38685280723454163, "1284": 1.0, "1297": 1.0, "1306": 0.3638184934971571, "1309": 0.4144299250118475, "1310": 1.0, "1321": 0.0, "1322": 0.19519002499605084, "1391": 0.1480409554829326, "1393": 0.19092086617893467, "1415": 0.9828920819566879, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.23463936301137822, "1670": 0.7653606369886217, "1676": 0.6713527276121545, "1736": 0.5531464700081437, "1748": 0.46927872602275644, "1783": 0.0, "1812": 0.21840743681816419, "1815": 1.0, "1819": 0.19092086617893467, "1824": 0.0, "1826": 1.0, "1832": 0.5, "1871": 0.5205067333228022, "1877": 1.0, "1889": 0.0, "1915": 0.8772153153380493, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.38356636737133565, "2010": 0.812653045383133, "2051": 0.0, "2070": 0.23463936301137822, "2075": 0.48384710931773217, "2076": 0.5437713091520254, "2088": 0.0, "2108": 0.14126697285982898, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 0.5499887725471476, "2204": 0.8987025955356565, "2264": 0.38685280723454163, "2296": 0.419999071625979, "2306": 0.6131471927654584, "2316": 1.0, "2318": 0.31546487678572877, "2330": 0.6131471927654584, "2334": 0.2960819109658652, "2348": 0.42257499837058643, "2376": 0.8910635073822442, "2383": 0.6131471927654584, "2384": 0.6309297535714575, "2385": 0.6131471927654584, "2388": 1.0, "2395": 1.0, "2398": 0.0, "2399": 1.0, "2400": 0.5307212739772434, "2407": 0.24630238874073, "2416": 0.8796542634124673, "2423": 0.27487633291429087, "2443": 0.0, "2445": 1.0, "2460": 0.8854598815714874, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 0.3333333333333333, "2513": 0.0, "2516": 0.5585075862632192, "2549": 0.6131471927654584, "2551": 0.8385465274895063, "2568": 0.13743816645714543, "2579": 0.13032376591037062, "2580": 0.7039180890341347, "2587": 1.0, "2589": 0.7653606369886217, "2590": 0.46927872602275644, "2593": 0.8854598815714874, "2598": 0.8065735963827293, "2648": 1.0, "2676": 0.5, "2685": 0.32908604348504067, "2695": 1.0, "2713": 0.6131471927654584, "2724": 0.8065735963827293, "2737": 0.6309297535714575, "2747": 0.5, "2749": 0.8503449055347546, "2790": 0.47600514717497383, "2801": 0.6131471927654584, "2856": 0.6804558477306826, "2857": 0.5032251913410369, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.0, "2923": 0.6843515475204855, "2964": 0.0, "2968": 0.6509209298071326, "2994": 0.46927872602275644, "3006": 0.2521686779156779, "3008": 0.19519002499605084, "3014": 0.8772153153380493, "3033": 0.3065735963827292, "3039": 0.43067655807339306, "3049": 0.19519002499605084, "3051": 0.9197207891481876, "3067": 0.0, "3085": 0.0, "3091": 0.6969904532005495, "3103": 0.5, "3115": 0.4935232796777481, "3125": 0.6309297535714575, "3148": 0.5531464700081437, "3149": 0.0, "3177": 0.15642624200758548, "3179": 0.8175295903539445, "3186": 1.0, "3189": 0.0, "3254": 0.6508205185601091, "3264": 0.5646436081071458, "3357": 1.0, "3369": 0.0, "3394": 0.5272750663607827, "3404": 0.3615901614084106, "3405": 0.0, "3446": 0.42913706161531123, "3451": 0.0, "3453": 0.6509209298071326, "3480": 0.9828920819566879, "3490": 0.0, "3500": 0.644824486427155, "3503": 0.5355852974309776, "3512": 0.8503449055347546, "3528": 0.6309297535714575, "3530": 0.37590633587120725, "3534": 0.0, "3566": 0.6257049680303419, "3569": 0.5, "3594": 0.0, "3612": 0.0, "3615": 0.42177340954886444, "3625": 0.20210734650054757, "3682": 0.20438239758848611, "3683": 0.41966500663677697, "3694": 0.6131471927654584, "3724": 0.0, "3735": 0.0, "3759": 0.0, "3767": 0.16958010263680806, "3771": 0.1919598743897292, "3781": 0.8772153153380493, "3789": 0.8048099750039491, "3791": 0.431733884398313, "3801": 0.2640681225725909, "3822": 0.0, "3829": 0.0, "3830": 0.6309297535714575, "3837": 0.6309297535714575, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.45186484096594276, "3932": 0.0, "3934": 0.16812753627111746, "3995": 0.0, "4007": 0.7653606369886217, "4011": 0.9674679834891693, "4019": 0.21398626473452756, "4031": 0.0, "4037": 0.0, "4047": 0.8318724637288826, "4071": 0.0, "4084": 0.46845052016107697, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 0.6131471927654584, "4125": 0.0, "4142": 0.2476427880871277, "4153": 0.2890648263178879, "4179": 0.6309297535714575, "4188": 0.0, "4205": 0.23463936301137822, "4233": 0.8687949224876582, "4265": 0.7247145167543899, "4286": 0.6131471927654584, "4289": 0.23719771276929622, "4306": 0.5855700749881525, "4312": 0.7837042816946693, "4335": 0.8048099750039491, "4339": 1.0, "4394": 0.0, "4409": 0.1610425878239398, "4411": 0.46927872602275644, "4414": 0.7903864795495061, "4415": 0.0, "4433": 0.0, "4447": 1.0, "4464": 0.43067655807339306, "4465": 0.20210734650054757, "4484": 0.6309297535714575, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 0.4034698846650846, "4523": 0.16958010263680806, "4539": 0.21840743681816419, "4571": 0.7039180890341347, "4600": 0.6309297535714575, "4605": 0.4728384582492843, "4615": 0.47472048453608134, "4640": 0.0, "4641": 0.23819820987960558, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75369761125927, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.6131471927654584, "4813": 1.0, "4823": 0.46927872602275644, "4827": 0.3562071871080222, "4837": 0.0, "4844": 1.0, "4845": 0.6713860725233041, "4846": 0.0, "4863": 0.4632423659320675, "4865": 0.0, "4920": 1.0, "4942": 0.0, "4946": 1.0, "4955": 0.0, "4962": 0.6131471927654584, "4968": 0.6309297535714575, "4981": 0.7653606369886217, "4999": 0.5205067333228022, "5021": 0.0, "5030": 0.0, "5045": 0.0, "5054": 0.0, "5061": 0.0, "5064": 0.9197207891481876, "5067": 0.5, "5080": 0.0, "5083": 0.23719771276929622, "5085": 0.6309297535714575, "5086": 0.6309297535714575, "5090": 0.23719771276929622, "5125": 1.0, "5134": 0.3065735963827292, "5150": 0.0, "5155": 1.0, "5172": 0.0, "5178": 0.23330247263525344, "5185": 0.0, "5196": 0.2960819109658652, "5206": 0.0, "5228": 0.3010299956639812, "5231": 0.0, "5241": 0.3391602052736161, "5254": 0.6131471927654584, "5255": 1.0, "5264": 0.6739577727076379, "5271": 0.36926780146674987, "5331": 1.0, "5343": 0.0, "5347": 1.0, "5356": 0.09803928583135704, "5369": 0.0, "5374": 0.0, "5380": 1.0, "5402": 0.0, "5410": 0.2960819109658652, "5422": 0.16812753627111746, "5427": 0.21840743681816419, "5460": 0.0, "5464": 0.15642624200758548, "5503": 0.38685280723454163, "5505": 1.0, "5511": 0.07839826897867533, "5534": 0.3065735963827292, "5549": 0.0, "5585": 0.2890648263178879, "5592": 1.0, "5616": 0.2890648263178879, "5620": 0.0, "5646": 0.9010126104210695, "5653": 1.0, "5683": 0.9197207891481876, "5710": 0.0, "5741": 0.0, "5763": 0.23463936301137822, "5782": 0.3391602052736161, "5790": 0.0, "5808": 0.5, "5853": 0.12647135138382856, "5862": 0.32531670832675136, "5888": 0.3065735963827292, "5903": 0.0, "5906": 0.2640681225725909, "5940": 0.0, "5951": 0.417488320194876, "5970": 0.20210734650054757, "5981": 0.9896062251871525, "5993": 0.31488013066763093, "6002": 0.762416491594312, "6004": 0.0, "6005": 0.5388856921828065, "6009": 0.0, "6041": 0.7668091220635322, "6080": 0.6131471927654584, "6110": 0.4273569456759105, "6121": 0.2640681225725909, "6122": 0.46927872602275644, "6131": 0.43735247915031, "6133": 0.6131471927654584, "6142": 0.6131471927654584, "6146": 0.8315546295836225, "6199": 0.3903800499921017, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.4367467095119258, "6278": 0.3065735963827292, "6395": 0.0, "6410": 0.3562071871080222, "6420": 1.0, "6441": 0.38685280723454163, "6467": 0.0, "6468": 0.6131471927654584, "6479": 1.0, "6525": 0.49818925746641285, "6554": 0.6131471927654584, "6562": 0.0, "6611": 0.7903864795495061, "6612": 0.15642624200758548, "6625": 0.0, "6629": 0.0, "6635": 0.6366824387328317, "6644": 0.0, "6647": 0.7039180890341347, "6668": 0.8175295903539445, "6679": 0.4525081529734507, "6683": 0.8315546295836225, "6713": 0.6131471927654584, "6715": 0.6131471927654584, "6746": 0.2640681225725909, "6787": 0.0, "6792": 0.0, "6800": 0.38685280723454163, "6803": 0.5135312443624667, "6807": 1.0, "6814": 0.0, "6832": 0.21840743681816419, "6835": 0.6131471927654584, "6849": 0.5, "6862": 1.0, "6867": 0.19796212533499344, "6875": 0.0, "6890": 0.48247555939075504, "6891": 0.46927872602275644, "6896": 0.0, "6901": 0.18457569677956817, "6907": 0.0, "6909": 0.0, "6959": 0.2640681225725909, "6985": 1.0, "7017": 0.0, "7068": 0.9197207891481876, "7071": 0.0, "7080": 0.8503449055347546, "7096": 0.6309297535714575, "7098": 0.6131471927654584, "7105": 0.3562071871080222, "7109": 0.2960819109658652, "7124": 0.3764290720714305, "7141": 0.0, "7145": 0.13565197343244778, "7178": 0.3333333333333333, "7188": 0.0, "7205": 1.0, "7206": 0.0, "7218": 0.43067655807339306, "7221": 0.0, "7269": 0.0, "7279": 0.3071837157818931, "7295": 0.0, "7311": 0.0, "7326": 0.0, "7329": 0.45749452618986164, "7344": 0.6131471927654584, "7345": 0.0, "7377": 1.0, "7431": 0.0, "7441": 0.16716045496620227, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.11305340175787204, "7467": 0.43067655807339306, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.14126697285982898, "7529": 0.2640681225725909, "7533": 0.0, "7534": 0.46927872602275644, "7590": 0.7977228895450266, "7592": 0.0, "7594": 0.3333333333333333, "7622": 0.0, "7633": 0.3903800499921017, "7674": 0.0, "7700": 0.0, "7702": 0.38685280723454163, "7705": 0.4367467095119258, "7734": 0.6131471927654584, "7747": 0.20438239758848611, "7754": 0.0, "7758": 0.5, "7801": 1.0, "7803": 0.18457569677956817, "7823": 0.6049306994552042, "7876": 0.38685280723454163, "7879": 0.0, "7880": 0.2640681225725909, "7911": 1.0, "7925": 0.19092086617893467, "7928": 0.24630238874073, "7936": 0.39106560501896365, "7992": 0.38685280723454163, "8002": 0.431733884398313, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.0, "8040": 1.0, "8072": 0.4776237035032179, "8079": 0.6713860725233041, "8102": 0.0, "8116": 0.448643819352159, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 0.6131471927654584, "8275": 0.43067655807339306, "8296": 0.8315546295836225, "8332": 0.9060254355346823, "8351": 0.6131471927654584, "8378": 0.13565197343244778, "8456": 0.46927872602275644, "8475": 0.7957503936198023, "8507": 0.0, "8512": 0.5, "8513": 0.0, "8532": 0.9066276098484507, "8537": 0.31546487678572877, "8539": 0.0, "8544": 0.6131471927654584, "8592": 0.23463936301137822, "8632": 0.20438239758848611, "8635": 0.0, "8702": 0.7653606369886217, "8779": 0.0, "8789": 0.7653606369886217, "8795": 1.0, "8832": 1.0, "8834": 0.19092086617893467, "8855": 0.0, "8874": 0.38685280723454163, "8934": 0.7903864795495061, "8937": 0.3562071871080222, "8947": 0.23719771276929622, "8959": 0.43067655807339306, "8970": 0.8503449055347546, "8974": 0.37941405415148566, "8982": 0.0, "9060": 0.6131471927654584, "9088": 0.0, "9108": 0.0, "9115": 0.49141527484330916, "9126": 0.0, "9164": 0.1480409554829326, "9174": 0.0, "9188": 0.6131471927654584, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.19342640361727081, "9329": 1.0, "9332": 0.3562071871080222, "9381": 0.4415801103577823, "9385": 0.3562071871080222, "9391": 0.4573973988771914, "9403": 0.0, "9481": 0.6240505200038379, "9487": 0.0, "9548": 0.38685280723454163, "9556": 1.0, "9565": 1.0, "9598": 0.31546487678572877, "9617": 0.16716045496620227, "9633": 0.6131471927654584, "9643": 0.5, "9644": 0.6309297535714575, "9646": 0.42177340954886444, "9668": 0.6508205185601091, "9701": 0.0, "9733": 0.16716045496620227, "9735": 0.3333333333333333, "9737": 0.9066276098484507, "9771": 0.0, "9808": 0.2960819109658652, "9824": 1.0, "9871": 0.24630238874073, "9882": 0.0, "9925": 0.8315546295836225, "9929": 0.3065735963827292, "9961": 0.2890648263178879, "9979": 0.3903800499921017, "10034": 0.2640681225725909, "10039": 0.6309297535714575, "10109": 0.499028327865556, "10122": 0.19519002499605084, "10136": 0.15642624200758548, "10137": 0.2890648263178879, "10152": 1.0, "10183": 0.21840743681816419, "10213": 0.46927872602275644, "10246": 0.0, "10267": 0.0, "10414": 0.8772153153380493, "10447": 0.20210734650054757, "10462": 0.0, "10482": 0.0, "10497": 0.49828993591603105, "10526": 0.19342640361727081, "10547": 0.0, "10558": 0.0, "10596": 0.9565912771023182, "10601": 1.0, "10628": 0.2890648263178879, "10639": 0.21398626473452756, "10645": 0.38685280723454163, "10674": 0.6886342695939197, "10710": 0.3333333333333333, "10734": 0.8486987932505933, "10792": 0.39780880120575696, "10808": 0.15642624200758548, "10809": 0.2960819109658652, "10812": 0.9709286432396583, "10827": 0.13120507751234178, "10845": 1.0, "10912": 0.0, "10932": 1.0, "10975": 0.0, "10979": 0.6366824387328317, "10994": 0.4911492931622974, "11039": 0.20261469210121208, "11054": 1.0, "11088": 0.3333333333333333}, "Recall@20": {"8": 0.5, "15": 1.0, "18": 1.0, "26": 0.5, "34": 0.0, "42": 0.3333333333333333, "56": 1.0, "68": 0.0, "89": 0.5, "90": 1.0, "94": 1.0, "98": 0.5, "104": 0.5, "106": 1.0, "109": 1.0, "475": 1.0, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 1.0, "559": 0.0, "570": 0.0, "585": 0.5, "588": 1.0, "594": 0.5, "603": 0.0, "604": 0.6666666666666666, "620": 0.2, "622": 0.5, "659": 0.7, "672": 0.5, "684": 0.0, "687": 0.5, "689": 0.0, "691": 1.0, "699": 1.0, "701": 0.3333333333333333, "715": 0.0, "721": 1.0, "744": 0.3333333333333333, "750": 1.0, "753": 0.5, "766": 0.3333333333333333, "776": 0.25, "810": 1.0, "813": 0.5, "849": 1.0, "852": 1.0, "853": 0.5, "858": 0.0, "859": 0.5, "864": 0.6666666666666666, "879": 1.0, "885": 0.5, "895": 1.0, "904": 1.0, "928": 0.5, "929": 1.0, "932": 0.5, "939": 1.0, "945": 1.0, "957": 0.6666666666666666, "988": 0.0, "1074": 1.0, "1085": 0.0, "1090": 0.5, "1150": 0.0, "1157": 0.5, "1159": 0.0, "1198": 1.0, "1230": 0.0, "1281": 1.0, "1284": 1.0, "1297": 1.0, "1306": 0.5, "1309": 0.75, "1310": 1.0, "1321": 0.0, "1322": 0.25, "1391": 0.3333333333333333, "1393": 0.5, "1415": 1.0, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 1.0, "1670": 0.6666666666666666, "1676": 1.0, "1736": 0.4, "1748": 0.6666666666666666, "1783": 0.0, "1812": 0.5, "1815": 1.0, "1819": 0.3333333333333333, "1824": 1.0, "1826": 1.0, "1832": 1.0, "1871": 0.75, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.4, "2010": 1.0, "2051": 0.0, "2070": 0.6666666666666666, "2075": 0.5555555555555556, "2076": 1.0, "2088": 0.0, "2108": 0.3333333333333333, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 0.6666666666666666, "2204": 1.0, "2264": 0.5, "2296": 0.3333333333333333, "2306": 0.5, "2316": 1.0, "2318": 1.0, "2330": 0.5, "2334": 1.0, "2348": 0.3333333333333333, "2376": 1.0, "2383": 0.5, "2384": 1.0, "2385": 0.5, "2388": 1.0, "2395": 1.0, "2398": 0.2, "2399": 1.0, "2400": 0.6666666666666666, "2407": 0.25, "2416": 1.0, "2423": 0.5714285714285714, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 1.0, "2513": 0.0, "2516": 0.5, "2549": 0.5, "2551": 1.0, "2568": 0.2857142857142857, "2579": 0.3333333333333333, "2580": 0.6666666666666666, "2587": 1.0, "2589": 0.6666666666666666, "2590": 0.3333333333333333, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 1.0, "2685": 0.25, "2695": 1.0, "2713": 0.5, "2724": 1.0, "2737": 1.0, "2747": 1.0, "2749": 1.0, "2790": 0.42857142857142855, "2801": 1.0, "2856": 0.75, "2857": 0.75, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 1.0, "2923": 0.8, "2964": 0.5, "2968": 1.0, "2994": 0.3333333333333333, "3006": 0.3333333333333333, "3008": 0.25, "3014": 1.0, "3033": 0.5, "3039": 1.0, "3049": 0.5, "3051": 1.0, "3067": 0.0, "3085": 0.5, "3091": 0.8571428571428571, "3103": 1.0, "3115": 0.3333333333333333, "3125": 1.0, "3148": 0.6, "3149": 0.0, "3177": 0.3333333333333333, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 0.6666666666666666, "3264": 0.75, "3357": 1.0, "3369": 0.16666666666666666, "3394": 0.75, "3404": 0.2857142857142857, "3405": 1.0, "3446": 0.6, "3451": 0.14285714285714285, "3453": 1.0, "3480": 1.0, "3490": 1.0, "3500": 0.5, "3503": 0.7142857142857143, "3512": 1.0, "3528": 1.0, "3530": 0.6666666666666666, "3534": 0.0, "3566": 1.0, "3569": 1.0, "3594": 0.25, "3612": 0.25, "3615": 1.0, "3625": 0.3333333333333333, "3682": 0.5, "3683": 0.6666666666666666, "3694": 0.5, "3724": 0.0, "3735": 0.6666666666666666, "3759": 0.0, "3767": 0.4, "3771": 0.3333333333333333, "3781": 1.0, "3789": 0.75, "3791": 0.6666666666666666, "3801": 0.5, "3822": 0.0, "3829": 0.0, "3830": 1.0, "3837": 1.0, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 0.5, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 0.6666666666666666, "4011": 1.0, "4019": 0.4, "4031": 0.0, "4037": 0.5, "4047": 0.75, "4071": 0.0, "4084": 1.0, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 0.5, "4125": 0.0, "4142": 0.75, "4153": 1.0, "4179": 1.0, "4188": 0.0, "4205": 0.6666666666666666, "4233": 0.8, "4265": 0.6666666666666666, "4286": 0.5, "4289": 0.5, "4306": 0.75, "4312": 0.8571428571428571, "4335": 0.75, "4339": 1.0, "4394": 0.0, "4409": 0.2, "4411": 0.6666666666666666, "4414": 1.0, "4415": 0.0, "4433": 0.5, "4447": 1.0, "4464": 1.0, "4465": 0.3333333333333333, "4484": 1.0, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 0.5, "4523": 0.2, "4539": 0.5, "4571": 0.6666666666666666, "4600": 1.0, "4605": 0.5714285714285714, "4615": 0.6666666666666666, "4640": 0.0, "4641": 0.4, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 0.75, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 0.5, "4813": 1.0, "4823": 0.3333333333333333, "4827": 1.0, "4837": 0.0, "4844": 1.0, "4845": 0.6666666666666666, "4846": 0.0, "4863": 0.6666666666666666, "4865": 0.0, "4920": 1.0, "4942": 1.0, "4946": 1.0, "4955": 0.0, "4962": 0.5, "4968": 1.0, "4981": 0.6666666666666666, "4999": 0.5, "5021": 0.0, "5030": 0.0, "5045": 1.0, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 1.0, "5080": 0.0, "5083": 0.5, "5085": 1.0, "5086": 1.0, "5090": 0.5, "5125": 1.0, "5134": 0.5, "5150": 0.0, "5155": 1.0, "5172": 0.5, "5178": 0.4, "5185": 0.0, "5196": 0.3333333333333333, "5206": 0.0, "5228": 1.0, "5231": 0.0, "5241": 0.4, "5254": 0.5, "5255": 1.0, "5264": 0.6, "5271": 0.6666666666666666, "5331": 1.0, "5343": 1.0, "5347": 1.0, "5356": 0.2, "5369": 0.0, "5374": 1.0, "5380": 1.0, "5402": 0.0, "5410": 0.3333333333333333, "5422": 0.75, "5427": 0.5, "5460": 0.0, "5464": 0.3333333333333333, "5503": 0.5, "5505": 1.0, "5511": 0.2, "5534": 0.5, "5549": 0.0, "5585": 1.0, "5592": 1.0, "5616": 1.0, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.0, "5763": 0.3333333333333333, "5782": 0.4, "5790": 0.0, "5808": 1.0, "5853": 0.25, "5862": 0.75, "5888": 0.5, "5903": 0.0, "5906": 1.0, "5940": 0.0, "5951": 0.6666666666666666, "5970": 0.3333333333333333, "5981": 1.0, "5993": 0.13333333333333333, "6002": 0.8181818181818182, "6004": 0.0, "6005": 0.38461538461538464, "6009": 1.0, "6041": 0.75, "6080": 0.5, "6110": 0.5, "6121": 0.5, "6122": 0.3333333333333333, "6131": 0.4166666666666667, "6133": 0.5, "6142": 0.5, "6146": 1.0, "6199": 0.5, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.6666666666666666, "6278": 1.0, "6395": 0.0, "6410": 1.0, "6420": 1.0, "6441": 1.0, "6467": 0.0, "6468": 0.5, "6479": 1.0, "6525": 0.6666666666666666, "6554": 0.5, "6562": 0.25, "6611": 1.0, "6612": 0.3333333333333333, "6625": 1.0, "6629": 0.0, "6635": 0.5, "6644": 0.0, "6647": 0.6666666666666666, "6668": 1.0, "6679": 0.6666666666666666, "6683": 1.0, "6713": 0.5, "6715": 1.0, "6746": 0.5, "6787": 0.0, "6792": 0.0, "6800": 1.0, "6803": 0.5, "6807": 1.0, "6814": 0.0, "6832": 1.0, "6835": 0.5, "6849": 1.0, "6862": 1.0, "6867": 0.2857142857142857, "6875": 1.0, "6890": 1.0, "6891": 0.6666666666666666, "6896": 0.0, "6901": 0.5, "6907": 0.0, "6909": 0.0, "6959": 0.5, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.5, "7080": 1.0, "7096": 1.0, "7098": 1.0, "7105": 1.0, "7109": 0.6666666666666666, "7124": 0.75, "7141": 0.0, "7145": 0.3333333333333333, "7178": 1.0, "7188": 1.0, "7205": 1.0, "7206": 0.0, "7218": 1.0, "7221": 0.0, "7269": 1.0, "7279": 0.75, "7295": 0.0, "7311": 1.0, "7326": 0.0, "7329": 1.0, "7344": 0.5, "7345": 0.3333333333333333, "7377": 1.0, "7431": 0.0, "7441": 0.6666666666666666, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.4, "7467": 1.0, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.3333333333333333, "7529": 0.5, "7533": 0.5, "7534": 0.3333333333333333, "7590": 1.0, "7592": 0.0, "7594": 1.0, "7622": 0.0, "7633": 0.25, "7674": 0.0, "7700": 0.25, "7702": 1.0, "7705": 0.6666666666666666, "7734": 0.5, "7747": 0.5, "7754": 1.0, "7758": 1.0, "7801": 1.0, "7803": 0.5, "7823": 0.6666666666666666, "7876": 1.0, "7879": 0.0, "7880": 0.5, "7911": 1.0, "7925": 0.16666666666666666, "7928": 0.25, "7936": 0.6666666666666666, "7992": 0.5, "8002": 0.6666666666666666, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 1.0, "8040": 1.0, "8072": 0.6666666666666666, "8079": 1.0, "8102": 0.25, "8116": 1.0, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 0.5, "8275": 1.0, "8296": 1.0, "8332": 1.0, "8351": 0.5, "8378": 0.3333333333333333, "8456": 0.6666666666666666, "8475": 0.7142857142857143, "8507": 0.0, "8512": 1.0, "8513": 0.0, "8532": 1.0, "8537": 1.0, "8539": 0.25, "8544": 0.5, "8592": 0.6666666666666666, "8632": 1.0, "8635": 0.0, "8702": 0.6666666666666666, "8779": 1.0, "8789": 0.6666666666666666, "8795": 1.0, "8832": 1.0, "8834": 0.3333333333333333, "8855": 0.0, "8874": 1.0, "8934": 1.0, "8937": 1.0, "8947": 0.5, "8959": 1.0, "8970": 1.0, "8974": 0.25, "8982": 0.0, "9060": 0.5, "9088": 0.0, "9108": 0.0, "9115": 1.0, "9126": 0.0, "9164": 0.3333333333333333, "9174": 0.0, "9188": 0.5, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.5, "9329": 1.0, "9332": 1.0, "9381": 1.0, "9385": 1.0, "9391": 0.75, "9403": 0.0, "9481": 1.0, "9487": 1.0, "9548": 1.0, "9556": 1.0, "9565": 1.0, "9598": 1.0, "9617": 0.6666666666666666, "9633": 0.5, "9643": 1.0, "9644": 1.0, "9646": 1.0, "9668": 0.6666666666666666, "9701": 0.0, "9733": 0.3333333333333333, "9735": 1.0, "9737": 1.0, "9771": 0.0, "9808": 0.3333333333333333, "9824": 1.0, "9871": 0.25, "9882": 0.0, "9925": 1.0, "9929": 1.0, "9961": 1.0, "9979": 0.5, "10034": 0.5, "10039": 1.0, "10109": 0.42857142857142855, "10122": 0.25, "10136": 0.6666666666666666, "10137": 1.0, "10152": 1.0, "10183": 0.5, "10213": 0.3333333333333333, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.3333333333333333, "10462": 0.0, "10482": 0.0, "10497": 0.5, "10526": 0.5, "10547": 0.0, "10558": 0.5, "10596": 1.0, "10601": 1.0, "10628": 1.0, "10639": 0.4, "10645": 1.0, "10674": 1.0, "10710": 1.0, "10734": 0.8333333333333334, "10792": 1.0, "10808": 1.0, "10809": 0.6666666666666666, "10812": 1.0, "10827": 0.2, "10845": 1.0, "10912": 1.0, "10932": 1.0, "10975": 0.0, "10979": 0.5, "10994": 1.0, "11039": 0.375, "11054": 1.0, "11088": 1.0}, "MRR@10": {"8": 1.0, "15": 0.16666666666666666, "18": 0.5, "26": 0.14285714285714285, "34": 0.0, "42": 1.0, "56": 0.14285714285714285, "68": 0.0, "89": 0.5, "90": 0.3333333333333333, "94": 1.0, "98": 1.0, "104": 0.5, "106": 0.25, "109": 1.0, "475": 0.5, "503": 0.0, "504": 0.0, "515": 1.0, "529": 0.0, "547": 0.0, "549": 0.14285714285714285, "559": 0.0, "570": 0.0, "585": 0.2, "588": 0.5, "594": 1.0, "603": 0.0, "604": 0.1111111111111111, "620": 0.058823529411764705, "622": 0.5, "659": 1.0, "672": 0.058823529411764705, "684": 0.0, "687": 0.08333333333333333, "689": 0.0, "691": 0.5, "699": 1.0, "701": 0.058823529411764705, "715": 0.0, "721": 1.0, "744": 0.16666666666666666, "750": 1.0, "753": 0.3333333333333333, "766": 0.125, "776": 1.0, "810": 0.1, "813": 0.1, "849": 0.125, "852": 1.0, "853": 0.07142857142857142, "858": 0.0, "859": 0.3333333333333333, "864": 1.0, "879": 0.5, "885": 0.08333333333333333, "895": 1.0, "904": 0.5, "928": 1.0, "929": 1.0, "932": 0.3333333333333333, "939": 0.3333333333333333, "945": 0.1111111111111111, "957": 1.0, "988": 0.0, "1074": 0.3333333333333333, "1085": 0.0, "1090": 0.16666666666666666, "1150": 0.0, "1157": 1.0, "1159": 0.0, "1198": 1.0, "1230": 0.0, "1281": 0.2, "1284": 1.0, "1297": 1.0, "1306": 0.5, "1309": 0.5, "1310": 1.0, "1321": 0.0, "1322": 0.3333333333333333, "1391": 0.125, "1393": 0.5, "1415": 1.0, "1416": 0.0, "1441": 1.0, "1451": 1.0, "1469": 1.0, "1530": 0.3333333333333333, "1670": 1.0, "1676": 1.0, "1736": 1.0, "1748": 1.0, "1783": 0.0, "1812": 0.16666666666666666, "1815": 1.0, "1819": 0.5, "1824": 0.08333333333333333, "1826": 1.0, "1832": 0.3333333333333333, "1871": 1.0, "1877": 1.0, "1889": 0.0, "1915": 1.0, "1920": 0.0, "1933": 1.0, "1948": 1.0, "1994": 0.5, "2010": 1.0, "2051": 0.0, "2070": 0.3333333333333333, "2075": 1.0, "2076": 0.3333333333333333, "2088": 0.0, "2108": 0.1111111111111111, "2118": 0.0, "2154": 0.0, "2181": 0.0, "2183": 1.0, "2204": 1.0, "2264": 0.5, "2296": 1.0, "2306": 1.0, "2316": 1.0, "2318": 0.125, "2330": 1.0, "2334": 0.5, "2348": 1.0, "2376": 1.0, "2383": 1.0, "2384": 0.5, "2385": 1.0, "2388": 1.0, "2395": 1.0, "2398": 0.05555555555555555, "2399": 1.0, "2400": 0.5, "2407": 0.5, "2416": 1.0, "2423": 1.0, "2443": 0.0, "2445": 1.0, "2460": 1.0, "2465": 0.0, "2472": 0.0, "2486": 1.0, "2498": 0.14285714285714285, "2513": 0.0, "2516": 1.0, "2549": 1.0, "2551": 1.0, "2568": 0.3333333333333333, "2579": 0.25, "2580": 1.0, "2587": 1.0, "2589": 1.0, "2590": 1.0, "2593": 1.0, "2598": 1.0, "2648": 1.0, "2676": 0.3333333333333333, "2685": 1.0, "2695": 1.0, "2713": 1.0, "2724": 1.0, "2737": 0.5, "2747": 0.3333333333333333, "2749": 1.0, "2790": 1.0, "2801": 1.0, "2856": 1.0, "2857": 1.0, "2880": 1.0, "2885": 0.0, "2891": 1.0, "2895": 0.0, "2903": 0.07692307692307693, "2923": 1.0, "2964": 0.05263157894736842, "2968": 0.5, "2994": 1.0, "3006": 0.3333333333333333, "3008": 0.3333333333333333, "3014": 1.0, "3033": 0.3333333333333333, "3039": 0.25, "3049": 0.3333333333333333, "3051": 1.0, "3067": 0.0, "3085": 0.09090909090909091, "3091": 1.0, "3103": 0.3333333333333333, "3115": 1.0, "3125": 0.5, "3148": 1.0, "3149": 0.0, "3177": 0.14285714285714285, "3179": 1.0, "3186": 1.0, "3189": 0.0, "3254": 1.0, "3264": 0.5, "3357": 1.0, "3369": 0.058823529411764705, "3394": 0.5, "3404": 1.0, "3405": 0.09090909090909091, "3446": 0.5, "3451": 0.05263157894736842, "3453": 0.5, "3480": 1.0, "3490": 0.0625, "3500": 1.0, "3503": 0.5, "3512": 1.0, "3528": 0.5, "3530": 0.3333333333333333, "3534": 0.0, "3566": 1.0, "3569": 0.3333333333333333, "3594": 0.07142857142857142, "3612": 0.058823529411764705, "3615": 0.2, "3625": 0.25, "3682": 0.14285714285714285, "3683": 1.0, "3694": 1.0, "3724": 0.0, "3735": 0.08333333333333333, "3759": 0.0, "3767": 0.3333333333333333, "3771": 0.14285714285714285, "3781": 1.0, "3789": 1.0, "3791": 0.5, "3801": 0.25, "3822": 0.0, "3829": 0.0, "3830": 0.5, "3837": 0.5, "3859": 0.0, "3875": 1.0, "3888": 0.0, "3909": 1.0, "3932": 0.0, "3934": 0.25, "3995": 0.0, "4007": 1.0, "4011": 1.0, "4019": 0.5, "4031": 0.0, "4037": 0.09090909090909091, "4047": 1.0, "4071": 0.0, "4084": 0.25, "4102": 0.0, "4103": 0.0, "4105": 0.0, "4116": 1.0, "4125": 0.0, "4142": 0.14285714285714285, "4153": 0.1, "4179": 0.5, "4188": 0.0, "4205": 0.3333333333333333, "4233": 1.0, "4265": 1.0, "4286": 1.0, "4289": 0.2, "4306": 1.0, "4312": 1.0, "4335": 1.0, "4339": 1.0, "4394": 0.0, "4409": 0.25, "4411": 1.0, "4414": 1.0, "4415": 0.0, "4433": 0.06666666666666667, "4447": 1.0, "4464": 0.25, "4465": 0.25, "4484": 0.5, "4499": 0.0, "4500": 0.0, "4504": 1.0, "4514": 1.0, "4523": 0.3333333333333333, "4539": 0.16666666666666666, "4571": 1.0, "4600": 0.5, "4605": 1.0, "4615": 0.5, "4640": 0.0, "4641": 0.2, "4678": 0.0, "4681": 1.0, "4700": 1.0, "4714": 1.0, "4756": 0.0, "4767": 0.0, "4775": 0.0, "4777": 0.0, "4785": 1.0, "4804": 1.0, "4813": 1.0, "4823": 1.0, "4827": 0.16666666666666666, "4837": 0.0, "4844": 1.0, "4845": 1.0, "4846": 0.0, "4863": 0.5, "4865": 0.0, "4920": 1.0, "4942": 0.07142857142857142, "4946": 1.0, "4955": 0.0, "4962": 1.0, "4968": 0.5, "4981": 1.0, "4999": 1.0, "5021": 0.0, "5030": 0.0, "5045": 0.07692307692307693, "5054": 0.0, "5061": 0.0, "5064": 1.0, "5067": 0.3333333333333333, "5080": 0.0, "5083": 0.2, "5085": 0.5, "5086": 0.5, "5090": 0.2, "5125": 1.0, "5134": 0.3333333333333333, "5150": 0.0, "5155": 1.0, "5172": 0.05555555555555555, "5178": 0.2, "5185": 0.0, "5196": 0.5, "5206": 0.0, "5228": 0.1111111111111111, "5231": 0.0, "5241": 1.0, "5254": 1.0, "5255": 1.0, "5264": 1.0, "5271": 0.25, "5331": 1.0, "5343": 0.08333333333333333, "5347": 1.0, "5356": 0.1, "5369": 0.0, "5374": 0.07692307692307693, "5380": 1.0, "5402": 0.0, "5410": 0.5, "5422": 0.25, "5427": 0.16666666666666666, "5460": 0.0, "5464": 0.14285714285714285, "5503": 0.5, "5505": 1.0, "5511": 0.16666666666666666, "5534": 0.3333333333333333, "5549": 0.0, "5585": 0.1, "5592": 1.0, "5616": 0.1, "5620": 0.0, "5646": 1.0, "5653": 1.0, "5683": 1.0, "5710": 0.0, "5741": 0.0, "5763": 0.3333333333333333, "5782": 1.0, "5790": 0.0, "5808": 0.3333333333333333, "5853": 0.3333333333333333, "5862": 0.3333333333333333, "5888": 0.3333333333333333, "5903": 0.0, "5906": 0.25, "5940": 0.0, "5951": 0.5, "5970": 0.25, "5981": 1.0, "5993": 1.0, "6002": 1.0, "6004": 0.0, "6005": 1.0, "6009": 0.07692307692307693, "6041": 1.0, "6080": 1.0, "6110": 1.0, "6121": 0.25, "6122": 1.0, "6131": 1.0, "6133": 1.0, "6142": 1.0, "6146": 1.0, "6199": 1.0, "6219": 1.0, "6221": 0.0, "6252": 0.0, "6262": 0.3333333333333333, "6278": 0.3333333333333333, "6395": 0.0, "6410": 0.16666666666666666, "6420": 1.0, "6441": 0.2, "6467": 0.0, "6468": 1.0, "6479": 1.0, "6525": 0.5, "6554": 1.0, "6562": 0.07142857142857142, "6611": 1.0, "6612": 0.14285714285714285, "6625": 0.08333333333333333, "6629": 0.0, "6635": 1.0, "6644": 0.0, "6647": 1.0, "6668": 1.0, "6679": 0.5, "6683": 1.0, "6713": 1.0, "6715": 1.0, "6746": 0.25, "6787": 0.0, "6792": 0.0, "6800": 0.2, "6803": 1.0, "6807": 1.0, "6814": 0.0, "6832": 0.16666666666666666, "6835": 1.0, "6849": 0.3333333333333333, "6862": 1.0, "6867": 0.2, "6875": 0.058823529411764705, "6890": 0.25, "6891": 1.0, "6896": 0.0, "6901": 0.1111111111111111, "6907": 0.0, "6909": 0.0, "6959": 0.25, "6985": 1.0, "7017": 0.0, "7068": 1.0, "7071": 0.05263157894736842, "7080": 1.0, "7096": 0.5, "7098": 1.0, "7105": 0.16666666666666666, "7109": 0.5, "7124": 0.5, "7141": 0.0, "7145": 0.1, "7178": 0.14285714285714285, "7188": 0.08333333333333333, "7205": 1.0, "7206": 0.0, "7218": 0.25, "7221": 0.0, "7269": 0.05, "7279": 0.25, "7295": 0.0, "7311": 0.05, "7326": 0.0, "7329": 0.25, "7344": 1.0, "7345": 0.07692307692307693, "7377": 1.0, "7431": 0.0, "7441": 0.16666666666666666, "7445": 0.0, "7448": 1.0, "7456": 0.0, "7463": 0.14285714285714285, "7467": 0.25, "7484": 1.0, "7509": 0.0, "7512": 0.0, "7513": 0.1111111111111111, "7529": 0.25, "7533": 0.05263157894736842, "7534": 1.0, "7590": 1.0, "7592": 0.0, "7594": 0.14285714285714285, "7622": 0.0, "7633": 1.0, "7674": 0.0, "7700": 0.08333333333333333, "7702": 0.5, "7705": 0.3333333333333333, "7734": 1.0, "7747": 0.14285714285714285, "7754": 0.08333333333333333, "7758": 0.3333333333333333, "7801": 1.0, "7803": 0.1111111111111111, "7823": 1.0, "7876": 0.2, "7879": 0.0, "7880": 0.25, "7911": 1.0, "7925": 0.5, "7928": 0.5, "7936": 0.3333333333333333, "7992": 0.5, "8002": 0.5, "8005": 0.0, "8013": 0.0, "8017": 1.0, "8034": 0.07692307692307693, "8040": 1.0, "8072": 0.5, "8079": 1.0, "8102": 0.07142857142857142, "8116": 0.25, "8121": 1.0, "8202": 0.0, "8230": 0.0, "8247": 0.0, "8271": 1.0, "8275": 0.25, "8296": 1.0, "8332": 1.0, "8351": 1.0, "8378": 0.1, "8456": 1.0, "8475": 1.0, "8507": 0.0, "8512": 0.3333333333333333, "8513": 0.0, "8532": 1.0, "8537": 0.125, "8539": 0.05263157894736842, "8544": 1.0, "8592": 0.3333333333333333, "8632": 0.14285714285714285, "8635": 0.0, "8702": 1.0, "8779": 0.09090909090909091, "8789": 1.0, "8795": 1.0, "8832": 1.0, "8834": 0.5, "8855": 0.0, "8874": 0.2, "8934": 1.0, "8937": 0.16666666666666666, "8947": 0.2, "8959": 0.25, "8970": 1.0, "8974": 1.0, "8982": 0.0, "9060": 1.0, "9088": 0.0, "9108": 0.0, "9115": 0.25, "9126": 0.0, "9164": 0.125, "9174": 0.0, "9188": 1.0, "9245": 0.0, "9275": 0.0, "9291": 0.0, "9296": 0.125, "9329": 1.0, "9332": 0.16666666666666666, "9381": 0.2, "9385": 0.16666666666666666, "9391": 0.3333333333333333, "9403": 0.0, "9481": 0.5, "9487": 0.07692307692307693, "9548": 0.2, "9556": 1.0, "9565": 1.0, "9598": 0.125, "9617": 0.16666666666666666, "9633": 1.0, "9643": 0.3333333333333333, "9644": 0.5, "9646": 0.2, "9668": 1.0, "9701": 0.0, "9733": 0.16666666666666666, "9735": 0.14285714285714285, "9737": 1.0, "9771": 0.0, "9808": 0.5, "9824": 1.0, "9871": 0.5, "9882": 0.0, "9925": 1.0, "9929": 0.3333333333333333, "9961": 0.1, "9979": 1.0, "10034": 0.25, "10039": 0.5, "10109": 1.0, "10122": 0.3333333333333333, "10136": 0.14285714285714285, "10137": 0.1, "10152": 1.0, "10183": 0.16666666666666666, "10213": 1.0, "10246": 0.0, "10267": 0.0, "10414": 1.0, "10447": 0.25, "10462": 0.0, "10482": 0.0, "10497": 1.0, "10526": 0.125, "10547": 0.0, "10558": 0.07142857142857142, "10596": 1.0, "10601": 1.0, "10628": 0.1, "10639": 0.5, "10645": 0.5, "10674": 1.0, "10710": 0.14285714285714285, "10734": 1.0, "10792": 0.14285714285714285, "10808": 0.14285714285714285, "10809": 0.5, "10812": 1.0, "10827": 0.2, "10845": 1.0, "10912": 0.058823529411764705, "10932": 1.0, "10975": 0.0, "10979": 1.0, "10994": 0.3333333333333333, "11039": 0.3333333333333333, "11054": 1.0, "11088": 0.14285714285714285}}} diff --git a/evals/benchmark/results/fiqa/qrels.json b/evals/benchmark/results/fiqa/qrels.json new file mode 100644 index 000000000..64af834e7 --- /dev/null +++ b/evals/benchmark/results/fiqa/qrels.json @@ -0,0 +1 @@ +{"8": {"566392": 1, "65404": 1}, "15": {"325273": 1}, "18": {"88124": 1}, "26": {"285255": 1, "350819": 1}, "34": {"599545": 1}, "42": {"272709": 1, "327263": 1, "331981": 1}, "56": {"572690": 1}, "68": {"19183": 1}, "89": {"413229": 1, "590102": 1, "268026": 1, "248624": 1, "508754": 1, "64556": 1}, "90": {"31793": 1}, "94": {"245447": 1}, "98": {"575929": 1, "527522": 1}, "104": {"575869": 1, "523158": 1}, "106": {"76695": 1}, "109": {"73427": 1}, "475": {"366761": 1}, "503": {"367641": 1}, "504": {"500755": 1, "344203": 1, "498751": 1}, "515": {"372909": 1}, "529": {"510701": 1}, "547": {"6349": 1, "278629": 1}, "549": {"214024": 1}, "559": {"246459": 1}, "570": {"363591": 1}, "585": {"140226": 1, "552375": 1}, "588": {"570546": 1, "203710": 1}, "594": {"377322": 1, "534059": 1}, "603": {"456440": 1}, "604": {"451443": 1, "231947": 1, "261622": 1}, "620": {"331332": 1, "417301": 1, "189303": 1, "180673": 1, "487067": 1}, "622": {"179756": 1, "369239": 1}, "659": {"13139": 1, "230908": 1, "584685": 1, "439467": 1, "264297": 1, "235925": 1, "168796": 1, "120279": 1, "365240": 1, "449079": 1}, "672": {"40966": 1, "563025": 1}, "684": {"441120": 1}, "687": {"146021": 1, "268992": 1}, "689": {"411044": 1}, "691": {"395912": 1}, "699": {"107092": 1}, "701": {"288537": 1, "339488": 1, "389446": 1}, "715": {"579763": 1, "546538": 1, "187404": 1}, "721": {"496225": 1}, "744": {"490443": 1, "566480": 1, "78176": 1}, "750": {"419768": 1, "33602": 1}, "753": {"466718": 1, "243503": 1}, "766": {"2996": 1, "387218": 1, "550172": 1}, "776": {"467044": 1, "583640": 1, "597247": 1, "124027": 1, "592680": 1, "10440": 1, "127263": 1, "332373": 1, "220127": 1, "597880": 1, "496899": 1, "591516": 1}, "810": {"30596": 1}, "813": {"106501": 1, "436701": 1}, "849": {"557186": 1}, "852": {"245702": 1, "531288": 1, "431735": 1}, "853": {"260795": 1, "147080": 1}, "858": {"146632": 1, "45185": 1, "122485": 1, "278450": 1}, "859": {"449630": 1, "18749": 1}, "864": {"472924": 1, "30142": 1, "211364": 1, "24890": 1, "566337": 1, "152072": 1}, "879": {"366830": 1}, "885": {"337165": 1, "409184": 1}, "895": {"416268": 1}, "904": {"415514": 1}, "928": {"349866": 1, "11572": 1}, "929": {"367754": 1}, "932": {"48722": 1, "387010": 1}, "939": {"392434": 1}, "945": {"352052": 1}, "957": {"446870": 1, "321500": 1, "540334": 1}, "988": {"226053": 1, "107688": 1}, "1074": {"443960": 1}, "1085": {"467737": 1, "393710": 1}, "1090": {"518896": 1, "203091": 1}, "1150": {"531698": 1, "43603": 1, "353369": 1, "19936": 1}, "1157": {"272425": 1, "405777": 1}, "1159": {"496064": 1}, "1198": {"56718": 1}, "1230": {"191649": 1}, "1281": {"361415": 1}, "1284": {"342411": 1}, "1297": {"171761": 1}, "1306": {"594206": 1, "484437": 1, "204075": 1, "167684": 1}, "1309": {"471630": 1, "156162": 1, "489401": 1, "438869": 1}, "1310": {"535651": 1}, "1321": {"216456": 1, "292065": 1}, "1322": {"399418": 1, "114231": 1, "115552": 1, "64138": 1}, "1391": {"266229": 1, "562176": 1, "440745": 1}, "1393": {"220022": 1, "589398": 1, "532932": 1, "352838": 1, "539133": 1, "352640": 1}, "1415": {"387165": 1, "306144": 1, "66246": 1, "393953": 1}, "1416": {"326717": 1}, "1441": {"178": 1, "414940": 1, "416727": 1, "471123": 1}, "1451": {"538005": 1}, "1469": {"444589": 1}, "1530": {"28764": 1, "313361": 1, "219425": 1}, "1670": {"290887": 1, "393898": 1, "493792": 1}, "1676": {"77245": 1, "394871": 1, "58937": 1, "234436": 1}, "1736": {"399406": 1, "396933": 1, "25543": 1, "562896": 1, "443419": 1}, "1748": {"18001": 1, "528564": 1, "576295": 1}, "1783": {"332314": 1}, "1812": {"530570": 1, "219274": 1}, "1815": {"446928": 1}, "1819": {"376499": 1, "66058": 1, "212713": 1, "267362": 1, "250285": 1, "220691": 1}, "1824": {"244808": 1}, "1826": {"401731": 1, "8057": 1, "181787": 1, "583788": 1}, "1832": {"590276": 1}, "1871": {"175524": 1, "364112": 1, "244185": 1, "444162": 1}, "1877": {"178697": 1}, "1889": {"388713": 1}, "1915": {"318266": 1, "433801": 1}, "1920": {"269943": 1}, "1933": {"183612": 1}, "1948": {"467509": 1}, "1994": {"434846": 1, "565157": 1, "51491": 1, "243356": 1, "156640": 1}, "2010": {"44256": 1, "233877": 1, "174034": 1}, "2051": {"558042": 1}, "2070": {"136438": 1, "363678": 1, "30253": 1}, "2075": {"170042": 1, "44417": 1, "260983": 1, "359580": 1, "60459": 1, "523393": 1, "519619": 1, "301866": 1, "14967": 1}, "2076": {"184646": 1, "278824": 1}, "2088": {"399875": 1, "599524": 1}, "2108": {"525200": 1, "309171": 1, "155389": 1}, "2118": {"411061": 1}, "2154": {"476632": 1}, "2181": {"376631": 1, "397329": 1}, "2183": {"132678": 1, "124427": 1, "571625": 1, "498146": 1, "24994": 1, "24344": 1}, "2204": {"280056": 1, "4066": 1, "174363": 1, "424523": 1, "271514": 1, "50809": 1, "374030": 1, "83922": 1}, "2264": {"534454": 1, "412819": 1}, "2296": {"396853": 1, "106424": 1, "400009": 1, "119298": 1, "83330": 1, "366594": 1, "279897": 1, "253563": 1, "130850": 1}, "2306": {"581889": 1, "315875": 1}, "2316": {"482343": 1, "348955": 1}, "2318": {"428533": 1}, "2330": {"478514": 1, "104221": 1}, "2334": {"228403": 1, "2304": 1, "150650": 1}, "2348": {"211867": 1, "247486": 1, "381757": 1, "447619": 1, "566573": 1, "410166": 1, "211622": 1, "474234": 1, "306430": 1, "268261": 1, "352271": 1, "543714": 1, "146441": 1, "265874": 1, "134864": 1}, "2376": {"64263": 1, "549833": 1, "91545": 1, "324411": 1, "114417": 1, "407455": 1, "402249": 1, "244961": 1}, "2383": {"17215": 1, "232199": 1}, "2384": {"435835": 1}, "2385": {"407654": 1, "373059": 1}, "2388": {"104988": 1}, "2395": {"179066": 1}, "2398": {"118730": 1, "509391": 1, "363810": 1, "224654": 1, "590489": 1}, "2399": {"343489": 1}, "2400": {"456470": 1, "1198": 1, "564271": 1}, "2407": {"294327": 1, "2064": 1, "319734": 1, "173929": 1}, "2416": {"162612": 1, "48569": 1, "105129": 1, "471247": 1}, "2423": {"81106": 1, "396127": 1, "553288": 1, "551954": 1, "529444": 1, "305946": 1, "538023": 1}, "2443": {"305579": 1}, "2445": {"431685": 1}, "2460": {"584337": 1, "549180": 1, "530037": 1}, "2465": {"570680": 1, "81046": 1, "546509": 1}, "2472": {"401125": 1, "370334": 1, "307315": 1}, "2486": {"487791": 1}, "2498": {"365456": 1}, "2513": {"82344": 1}, "2516": {"505678": 1, "199508": 1, "446340": 1, "566602": 1}, "2549": {"58451": 1, "21103": 1}, "2551": {"413832": 1, "450742": 1, "143100": 1}, "2568": {"139047": 1, "388798": 1, "296769": 1, "346042": 1, "108739": 1, "590082": 1, "127353": 1}, "2579": {"197052": 1, "524471": 1, "432808": 1, "191977": 1, "280838": 1, "432020": 1}, "2580": {"344118": 1, "503934": 1, "11988": 1}, "2587": {"89326": 1}, "2589": {"43961": 1, "450577": 1, "468144": 1}, "2590": {"296528": 1, "209493": 1, "589625": 1}, "2593": {"147343": 1, "231614": 1, "528132": 1}, "2598": {"376126": 1, "593029": 1}, "2648": {"307120": 1}, "2676": {"55666": 1}, "2685": {"384532": 1, "154113": 1, "370300": 1, "37900": 1, "382005": 1, "303293": 1, "594182": 1, "468923": 1}, "2695": {"305904": 1}, "2713": {"29372": 1, "388147": 1}, "2724": {"491472": 1, "32172": 1}, "2737": {"426678": 1}, "2747": {"540571": 1}, "2749": {"587192": 1, "207285": 1}, "2790": {"423403": 1, "279329": 1, "27268": 1, "472484": 1, "100483": 1, "469125": 1, "4612": 1}, "2801": {"398856": 1, "475541": 1}, "2856": {"231727": 1, "213331": 1, "110848": 1, "342212": 1}, "2857": {"233732": 1, "501384": 1, "60175": 1, "295864": 1}, "2880": {"533791": 1}, "2885": {"367360": 1, "414692": 1, "359579": 1, "85229": 1, "454810": 1}, "2891": {"31117": 1}, "2895": {"521996": 1, "328691": 1}, "2903": {"527776": 1}, "2923": {"82744": 1, "161667": 1, "46381": 1, "390089": 1, "264271": 1}, "2964": {"302409": 1, "95116": 1}, "2968": {"242124": 1, "388646": 1}, "2994": {"419319": 1, "318491": 1, "569145": 1}, "3006": {"127838": 1, "512096": 1, "269851": 1, "403137": 1, "568473": 1, "328300": 1}, "3008": {"231688": 1, "180192": 1, "407401": 1, "323406": 1}, "3014": {"341399": 1, "273282": 1}, "3033": {"571430": 1, "265866": 1}, "3039": {"136804": 1}, "3049": {"127974": 1, "88477": 1, "582864": 1, "450808": 1}, "3051": {"232063": 1, "591940": 1}, "3067": {"406156": 1, "517299": 1}, "3085": {"248619": 1, "527010": 1}, "3091": {"159936": 1, "108302": 1, "366448": 1, "187739": 1, "534099": 1, "274870": 1, "95778": 1}, "3103": {"176596": 1}, "3115": {"129364": 1, "183869": 1, "234950": 1, "389028": 1, "233562": 1, "316794": 1}, "3125": {"89008": 1}, "3148": {"178127": 1, "584305": 1, "438000": 1, "92888": 1, "172855": 1}, "3149": {"490223": 1, "98112": 1}, "3177": {"17208": 1, "536120": 1, "268289": 1}, "3179": {"150219": 1, "385073": 1}, "3186": {"555486": 1, "545421": 1}, "3189": {"225395": 1, "272840": 1}, "3254": {"484891": 1, "443511": 1, "475756": 1}, "3264": {"598807": 1, "134764": 1, "486525": 1, "260383": 1}, "3357": {"209974": 1}, "3369": {"371886": 1, "163834": 1, "231012": 1, "145716": 1, "411910": 1, "395840": 1}, "3394": {"445971": 1, "342258": 1, "129319": 1, "570664": 1}, "3404": {"442110": 1, "556976": 1, "160301": 1, "488574": 1, "498834": 1, "395483": 1, "277583": 1}, "3405": {"495467": 1}, "3446": {"366685": 1, "211839": 1, "511386": 1, "236899": 1, "323498": 1}, "3451": {"26292": 1, "192307": 1, "276883": 1, "588448": 1, "165970": 1, "490170": 1, "99132": 1}, "3453": {"418626": 1, "233635": 1}, "3480": {"53468": 1, "418352": 1, "221015": 1, "75568": 1}, "3490": {"420529": 1}, "3500": {"273187": 1, "141935": 1, "557478": 1, "174019": 1, "71424": 1, "565409": 1}, "3503": {"345294": 1, "297764": 1, "63091": 1, "177563": 1, "482077": 1, "360621": 1, "89509": 1}, "3512": {"466835": 1, "115042": 1}, "3528": {"345697": 1}, "3530": {"189190": 1, "184299": 1, "239998": 1}, "3534": {"340329": 1}, "3566": {"152286": 1, "99943": 1, "307424": 1}, "3569": {"450135": 1}, "3594": {"246882": 1, "554171": 1, "525247": 1, "490294": 1}, "3612": {"402726": 1, "584291": 1, "259625": 1, "212687": 1}, "3615": {"518379": 1, "335991": 1}, "3625": {"500751": 1, "384469": 1, "414295": 1}, "3682": {"356161": 1, "329662": 1}, "3683": {"276975": 1, "185909": 1, "454501": 1, "565016": 1, "522713": 1, "105973": 1}, "3694": {"282442": 1, "204747": 1}, "3724": {"302512": 1, "273497": 1, "309200": 1, "508921": 1, "434190": 1, "279570": 1, "497216": 1, "552887": 1, "341146": 1, "199970": 1}, "3735": {"330041": 1, "266900": 1, "314478": 1}, "3759": {"527966": 1, "67167": 1, "522358": 1}, "3767": {"368679": 1, "153922": 1, "392060": 1, "520395": 1, "320246": 1}, "3771": {"521712": 1, "128471": 1, "488948": 1, "198349": 1, "217683": 1, "49601": 1}, "3781": {"131959": 1, "413328": 1}, "3789": {"459724": 1, "274573": 1, "492856": 1, "571131": 1}, "3791": {"327432": 1, "577201": 1, "212222": 1}, "3801": {"390529": 1, "307776": 1}, "3822": {"305907": 1, "385090": 1, "418900": 1, "308837": 1}, "3829": {"523850": 1, "291438": 1}, "3830": {"54952": 1}, "3837": {"345533": 1}, "3859": {"230261": 1}, "3875": {"136315": 1, "249859": 1}, "3888": {"307083": 1, "319213": 1, "239632": 1}, "3909": {"404352": 1, "200690": 1, "312248": 1, "404356": 1, "374400": 1, "61586": 1, "514003": 1, "193459": 1, "245616": 1, "353028": 1}, "3932": {"107697": 1, "527443": 1}, "3934": {"457034": 1, "122952": 1, "209604": 1, "201769": 1}, "3995": {"427032": 1, "278734": 1, "297900": 1, "506909": 1, "230208": 1}, "4007": {"556220": 1, "388704": 1, "521657": 1}, "4011": {"136367": 1, "470": 1, "67699": 1}, "4019": {"287991": 1, "379948": 1, "332749": 1, "6881": 1, "125477": 1}, "4031": {"115741": 1}, "4037": {"10275": 1, "298099": 1}, "4047": {"417840": 1, "452175": 1, "77652": 1, "324779": 1}, "4071": {"129875": 1}, "4084": {"250028": 1, "308633": 1}, "4102": {"554734": 1, "241101": 1, "39115": 1, "448699": 1}, "4103": {"440270": 1}, "4105": {"166412": 1, "25096": 1}, "4116": {"234674": 1, "109149": 1}, "4125": {"344648": 1, "72046": 1}, "4142": {"586930": 1, "544020": 1, "203926": 1, "61346": 1}, "4153": {"459596": 1}, "4179": {"278196": 1}, "4188": {"468108": 1}, "4205": {"79777": 1, "134239": 1, "540451": 1}, "4233": {"16270": 1, "590836": 1, "154886": 1, "31377": 1, "187129": 1}, "4265": {"392163": 1, "293111": 1, "140947": 1, "349710": 1, "157553": 1, "93881": 1}, "4286": {"70460": 1, "566069": 1}, "4289": {"288330": 1, "24881": 1}, "4306": {"450694": 1, "221435": 1, "233718": 1, "61819": 1}, "4312": {"116647": 1, "222639": 1, "135845": 1, "167950": 1, "282435": 1, "399149": 1, "507284": 1}, "4335": {"318583": 1, "357013": 1, "322246": 1, "484148": 1}, "4339": {"386531": 1, "488615": 1}, "4394": {"336045": 1, "441582": 1}, "4409": {"426676": 1, "102326": 1, "115066": 1, "499128": 1, "97925": 1, "100306": 1, "245903": 1, "147439": 1, "102088": 1, "360682": 1}, "4411": {"79903": 1, "68239": 1, "116545": 1}, "4414": {"235522": 1, "367137": 1}, "4415": {"238234": 1, "147646": 1, "414188": 1, "67676": 1}, "4433": {"146181": 1, "301580": 1}, "4447": {"286226": 1}, "4464": {"102904": 1}, "4465": {"376575": 1, "344041": 1, "299242": 1}, "4484": {"33990": 1}, "4499": {"323363": 1, "513818": 1, "76996": 1}, "4500": {"163048": 1, "533623": 1}, "4504": {"2981": 1, "566458": 1}, "4514": {"352485": 1, "69485": 1, "337764": 1, "156211": 1, "426270": 1, "209804": 1}, "4523": {"594257": 1, "119165": 1, "393009": 1, "129255": 1, "66626": 1}, "4539": {"370879": 1, "275925": 1}, "4571": {"496857": 1, "282392": 1, "213824": 1}, "4600": {"482415": 1}, "4605": {"41312": 1, "313306": 1, "453941": 1, "504661": 1, "229310": 1, "210759": 1, "400826": 1}, "4615": {"496427": 1, "261900": 1, "262934": 1, "69523": 1, "421365": 1, "596196": 1}, "4640": {"322314": 1, "101369": 1, "540539": 1}, "4641": {"44594": 1, "406219": 1, "319954": 1, "397358": 1, "88327": 1}, "4678": {"305153": 1}, "4681": {"181909": 1, "587959": 1}, "4700": {"345137": 1}, "4714": {"505057": 1, "584450": 1, "450819": 1, "324833": 1}, "4756": {"340254": 1}, "4767": {"280805": 1, "568670": 1, "224057": 1, "22804": 1, "125986": 1}, "4775": {"72021": 1}, "4777": {"590710": 1}, "4785": {"37517": 1}, "4804": {"583651": 1, "104395": 1}, "4813": {"98356": 1}, "4823": {"104726": 1, "362919": 1, "561929": 1}, "4827": {"495791": 1}, "4837": {"531841": 1, "20958": 1}, "4844": {"275257": 1, "192193": 1}, "4845": {"40424": 1, "339928": 1, "41625": 1}, "4846": {"151104": 1, "323749": 1}, "4863": {"87398": 1, "198465": 1, "290562": 1}, "4865": {"100485": 1}, "4920": {"306815": 1, "228341": 1, "82294": 1}, "4942": {"357685": 1}, "4946": {"121690": 1}, "4955": {"581318": 1}, "4962": {"158363": 1, "599925": 1}, "4968": {"387022": 1}, "4981": {"45218": 1, "247894": 1, "102684": 1}, "4999": {"314898": 1, "338803": 1, "9938": 1, "46211": 1}, "5021": {"589285": 1}, "5030": {"215540": 1}, "5045": {"264554": 1}, "5054": {"28119": 1}, "5061": {"23747": 1}, "5064": {"2003": 1, "192811": 1}, "5067": {"547301": 1}, "5080": {"256055": 1}, "5083": {"305509": 1, "138845": 1}, "5085": {"102436": 1}, "5086": {"96606": 1}, "5090": {"436493": 1, "12988": 1}, "5125": {"318728": 1}, "5134": {"158523": 1, "206727": 1}, "5150": {"229937": 1}, "5155": {"462892": 1}, "5172": {"529418": 1, "539165": 1}, "5178": {"240261": 1, "561056": 1, "111815": 1, "393833": 1, "39819": 1}, "5185": {"210236": 1, "317354": 1}, "5196": {"172128": 1, "565745": 1, "114829": 1}, "5206": {"563030": 1, "287157": 1, "28230": 1, "201982": 1, "117276": 1, "300660": 1}, "5228": {"232451": 1}, "5231": {"146188": 1}, "5241": {"376123": 1, "234286": 1, "322157": 1, "344740": 1, "27489": 1}, "5254": {"392851": 1, "402466": 1}, "5255": {"75112": 1, "330453": 1, "412830": 1}, "5264": {"371720": 1, "52579": 1, "505694": 1, "431814": 1, "576564": 1}, "5271": {"58599": 1, "221439": 1, "67663": 1}, "5331": {"200784": 1}, "5343": {"2860": 1}, "5347": {"352930": 1}, "5356": {"279785": 1, "71553": 1, "381362": 1, "312405": 1, "562259": 1}, "5369": {"171339": 1, "44223": 1}, "5374": {"152688": 1}, "5380": {"549254": 1}, "5402": {"491350": 1, "227485": 1}, "5410": {"13975": 1, "507813": 1, "368802": 1}, "5422": {"273906": 1, "79517": 1, "151973": 1, "13513": 1}, "5427": {"200603": 1, "323284": 1}, "5460": {"93248": 1, "184337": 1, "21174": 1, "108514": 1, "463885": 1}, "5464": {"350399": 1, "294549": 1, "86691": 1}, "5503": {"146277": 1, "64279": 1}, "5505": {"100387": 1}, "5511": {"169893": 1, "107898": 1, "529123": 1, "560325": 1, "478426": 1, "383193": 1, "114303": 1, "278699": 1, "12746": 1, "51873": 1}, "5534": {"423272": 1, "421136": 1}, "5549": {"286227": 1, "309361": 1}, "5585": {"300117": 1}, "5592": {"355241": 1}, "5616": {"391291": 1}, "5620": {"448784": 1, "171510": 1, "329552": 1, "548740": 1}, "5646": {"200972": 1, "426510": 1, "516413": 1}, "5653": {"588247": 1}, "5683": {"173026": 1, "448690": 1}, "5710": {"232311": 1}, "5741": {"259081": 1, "25943": 1}, "5763": {"89964": 1, "515361": 1, "462019": 1}, "5782": {"172084": 1, "379891": 1, "319773": 1, "448614": 1, "595455": 1}, "5790": {"134794": 1}, "5808": {"251824": 1}, "5853": {"284318": 1, "476663": 1, "160105": 1, "495699": 1, "439459": 1, "473647": 1, "431811": 1, "424598": 1}, "5862": {"130209": 1, "269898": 1, "170141": 1, "562511": 1}, "5888": {"336792": 1, "540806": 1}, "5903": {"231863": 1}, "5906": {"409907": 1, "15172": 1}, "5940": {"486243": 1, "433827": 1, "93936": 1}, "5951": {"418034": 1, "259777": 1, "31663": 1, "151774": 1, "374480": 1, "473765": 1, "298065": 1, "596834": 1, "497260": 1}, "5970": {"181608": 1, "546568": 1, "168453": 1}, "5981": {"564554": 1, "245532": 1, "171483": 1, "286182": 1, "210439": 1}, "5993": {"94373": 1, "367375": 1, "287571": 1, "224918": 1, "128574": 1, "431212": 1, "272866": 1, "230215": 1, "55084": 1, "160193": 1, "5827": 1, "352638": 1, "63690": 1, "426120": 1, "63501": 1}, "6002": {"404605": 1, "273501": 1, "390642": 1, "359862": 1, "516848": 1, "391819": 1, "34389": 1, "233472": 1, "519346": 1, "593434": 1, "154181": 1}, "6004": {"149555": 1}, "6005": {"135415": 1, "478457": 1, "345895": 1, "73310": 1, "384626": 1, "176498": 1, "149500": 1, "572272": 1, "414288": 1, "507544": 1, "390689": 1, "414534": 1, "270856": 1}, "6009": {"228983": 1}, "6041": {"111091": 1, "425020": 1, "241308": 1, "81655": 1}, "6080": {"164513": 1, "22856": 1}, "6110": {"118039": 1, "331850": 1, "188531": 1, "233379": 1, "94117": 1, "259706": 1, "320450": 1, "226496": 1}, "6121": {"394460": 1, "289231": 1}, "6122": {"496166": 1, "169824": 1, "44344": 1}, "6131": {"326094": 1, "365263": 1, "218088": 1, "252534": 1, "381720": 1, "235452": 1, "170204": 1, "416679": 1, "334111": 1, "2460": 1, "368806": 1, "258465": 1}, "6133": {"7733": 1, "415705": 1}, "6142": {"155461": 1, "209863": 1}, "6146": {"7403": 1, "160125": 1}, "6199": {"414693": 1, "584273": 1, "239214": 1, "169921": 1}, "6219": {"48946": 1, "146924": 1}, "6221": {"128698": 1, "470716": 1, "257248": 1, "519675": 1, "76414": 1, "169688": 1, "455614": 1, "115717": 1}, "6252": {"394551": 1, "160932": 1, "293624": 1, "233294": 1, "243268": 1, "379487": 1, "62868": 1}, "6262": {"34538": 1, "26799": 1, "390877": 1}, "6278": {"138511": 1, "63698": 1}, "6395": {"166227": 1}, "6410": {"471723": 1}, "6420": {"565568": 1, "127452": 1}, "6441": {"254279": 1}, "6467": {"453256": 1, "66834": 1, "23217": 1, "346641": 1, "367313": 1}, "6468": {"548970": 1, "332069": 1}, "6479": {"272008": 1, "404339": 1}, "6525": {"181985": 1, "98150": 1, "106541": 1}, "6554": {"583203": 1, "22469": 1}, "6562": {"135675": 1, "582414": 1, "561344": 1, "501157": 1}, "6611": {"293679": 1, "198764": 1}, "6612": {"254454": 1, "205522": 1, "322900": 1}, "6625": {"88892": 1}, "6629": {"444405": 1}, "6635": {"102449": 1, "587137": 1, "3656": 1, "156358": 1}, "6644": {"175035": 1}, "6647": {"69790": 1, "428017": 1, "462265": 1}, "6668": {"275902": 1, "457294": 1}, "6679": {"242298": 1, "581672": 1, "358492": 1}, "6683": {"592187": 1, "77631": 1}, "6713": {"147853": 1, "357571": 1}, "6715": {"568526": 1, "13732": 1}, "6746": {"210887": 1, "333674": 1}, "6787": {"587120": 1}, "6792": {"485973": 1}, "6800": {"35191": 1}, "6803": {"193012": 1, "544053": 1, "142726": 1, "42620": 1}, "6807": {"332567": 1}, "6814": {"340214": 1, "223206": 1}, "6832": {"377853": 1, "573874": 1}, "6835": {"149305": 1, "102243": 1}, "6849": {"44601": 1}, "6862": {"386796": 1}, "6867": {"443804": 1, "466143": 1, "878": 1, "540799": 1, "502607": 1, "445258": 1, "538750": 1}, "6875": {"224392": 1}, "6890": {"558703": 1, "240196": 1}, "6891": {"277217": 1, "463837": 1, "474351": 1}, "6896": {"251704": 1}, "6901": {"388571": 1, "254474": 1}, "6907": {"251604": 1}, "6909": {"127012": 1}, "6959": {"208331": 1, "205010": 1}, "6985": {"456436": 1}, "7017": {"28271": 1}, "7068": {"61864": 1, "10797": 1}, "7071": {"124230": 1, "238629": 1}, "7080": {"517577": 1, "596518": 1}, "7096": {"482238": 1}, "7098": {"34139": 1, "266898": 1}, "7105": {"72652": 1}, "7109": {"257185": 1, "565501": 1, "447781": 1}, "7124": {"74615": 1, "154725": 1, "558617": 1, "526110": 1}, "7141": {"255927": 1, "132288": 1}, "7145": {"81458": 1, "17923": 1, "116865": 1}, "7178": {"401329": 1}, "7188": {"305346": 1}, "7205": {"159966": 1}, "7206": {"441155": 1, "532211": 1, "553066": 1}, "7218": {"32290": 1}, "7221": {"186578": 1}, "7269": {"486696": 1}, "7279": {"476834": 1, "322456": 1, "81483": 1, "464502": 1}, "7295": {"244749": 1}, "7311": {"323768": 1}, "7326": {"584295": 1}, "7329": {"118520": 1, "325330": 1}, "7344": {"14368": 1, "108403": 1}, "7345": {"508821": 1, "237645": 1, "527080": 1}, "7377": {"579557": 1}, "7431": {"372921": 1}, "7441": {"537418": 1, "514500": 1, "117576": 1}, "7445": {"153178": 1, "104343": 1, "296231": 1}, "7448": {"418150": 1}, "7456": {"65120": 1}, "7463": {"577951": 1, "582005": 1, "105634": 1, "305287": 1, "5152": 1}, "7467": {"193312": 1}, "7484": {"444562": 1}, "7509": {"178303": 1}, "7512": {"191060": 1}, "7513": {"115372": 1, "273861": 1, "567362": 1}, "7529": {"66607": 1, "293626": 1}, "7533": {"162668": 1, "93853": 1}, "7534": {"175821": 1, "89714": 1, "358125": 1}, "7590": {"105165": 1, "222505": 1}, "7592": {"273947": 1}, "7594": {"573899": 1}, "7622": {"253369": 1, "378594": 1}, "7633": {"494727": 1, "442048": 1, "122996": 1, "197839": 1}, "7674": {"193318": 1, "519390": 1, "494295": 1}, "7700": {"273761": 1, "507468": 1, "2653": 1, "179328": 1}, "7702": {"387277": 1, "558924": 1}, "7705": {"539263": 1, "195191": 1, "377429": 1}, "7734": {"226070": 1, "433905": 1}, "7747": {"97729": 1, "296420": 1}, "7754": {"197527": 1}, "7758": {"574327": 1}, "7801": {"252574": 1, "427592": 1}, "7803": {"565926": 1, "157504": 1}, "7823": {"105666": 1, "583549": 1, "451196": 1}, "7876": {"533779": 1}, "7879": {"102029": 1, "372551": 1, "421285": 1}, "7880": {"272790": 1, "85319": 1}, "7911": {"57711": 1, "131996": 1}, "7925": {"318185": 1, "251100": 1, "503912": 1, "310636": 1, "402482": 1, "438974": 1}, "7928": {"480967": 1, "118633": 1, "501504": 1, "499811": 1}, "7936": {"293767": 1, "363043": 1, "336541": 1}, "7992": {"559654": 1, "264476": 1}, "8002": {"118786": 1, "265159": 1, "34767": 1}, "8005": {"48800": 1}, "8013": {"496159": 1, "224231": 1}, "8017": {"97836": 1}, "8034": {"597351": 1}, "8040": {"523058": 1}, "8072": {"394244": 1, "169062": 1, "31465": 1}, "8079": {"108040": 1, "57138": 1, "95278": 1}, "8102": {"552707": 1, "557877": 1, "378173": 1, "90294": 1}, "8116": {"236482": 1, "433032": 1}, "8121": {"432424": 1}, "8202": {"513258": 1, "93971": 1}, "8230": {"319599": 1, "363204": 1}, "8247": {"42521": 1, "475170": 1, "321114": 1, "465313": 1}, "8271": {"415511": 1, "376709": 1}, "8275": {"294867": 1}, "8296": {"78675": 1, "79998": 1}, "8332": {"328794": 1, "232261": 1, "445526": 1}, "8351": {"273142": 1, "472516": 1}, "8378": {"178684": 1, "102237": 1, "125298": 1}, "8456": {"257853": 1, "510875": 1, "486333": 1}, "8475": {"494655": 1, "240562": 1, "293320": 1, "104793": 1, "437427": 1, "444107": 1, "481728": 1}, "8507": {"509819": 1, "370995": 1, "351615": 1}, "8512": {"32811": 1}, "8513": {"270573": 1}, "8532": {"416307": 1, "27401": 1, "320101": 1}, "8537": {"6574": 1}, "8539": {"218728": 1, "292609": 1, "196304": 1, "396038": 1}, "8544": {"214003": 1, "267113": 1}, "8592": {"340264": 1, "441718": 1, "261258": 1}, "8632": {"43497": 1, "213976": 1}, "8635": {"67107": 1, "240215": 1}, "8702": {"135363": 1, "345410": 1, "157759": 1}, "8779": {"180571": 1, "209730": 1}, "8789": {"87349": 1, "70853": 1, "41912": 1}, "8795": {"57297": 1}, "8832": {"94689": 1}, "8834": {"133644": 1, "521095": 1, "12232": 1, "197151": 1, "203139": 1, "569303": 1}, "8855": {"208165": 1, "474296": 1, "312821": 1}, "8874": {"403025": 1}, "8934": {"244334": 1, "116675": 1}, "8937": {"469888": 1}, "8947": {"455398": 1, "461592": 1}, "8959": {"250164": 1}, "8970": {"187583": 1, "96121": 1}, "8974": {"140738": 1, "216365": 1, "523331": 1, "134931": 1, "356595": 1, "170625": 1, "10476": 1, "478242": 1}, "8982": {"454610": 1, "200360": 1}, "9060": {"511093": 1, "40447": 1}, "9088": {"569461": 1, "561377": 1}, "9108": {"272021": 1, "472585": 1}, "9115": {"158520": 1, "207325": 1, "422467": 1}, "9126": {"514831": 1}, "9164": {"305274": 1, "365298": 1, "263390": 1}, "9174": {"431652": 1, "535317": 1, "160218": 1, "544576": 1, "405217": 1}, "9188": {"265167": 1, "580802": 1}, "9245": {"194561": 1}, "9275": {"338754": 1, "14364": 1}, "9291": {"96926": 1, "480315": 1}, "9296": {"435746": 1, "206744": 1}, "9329": {"523913": 1, "326991": 1}, "9332": {"271766": 1}, "9381": {"408123": 1, "384983": 1}, "9385": {"403755": 1}, "9391": {"136515": 1, "283202": 1, "248158": 1, "503637": 1}, "9403": {"6666": 1, "328086": 1, "345199": 1}, "9481": {"30417": 1, "402240": 1}, "9487": {"165544": 1}, "9548": {"297290": 1}, "9556": {"37040": 1}, "9565": {"292559": 1, "478291": 1}, "9598": {"379311": 1}, "9617": {"408524": 1, "364735": 1, "314008": 1}, "9633": {"534418": 1, "585447": 1}, "9643": {"112208": 1}, "9644": {"194605": 1}, "9646": {"556191": 1, "272784": 1}, "9668": {"13260": 1, "42438": 1, "111768": 1}, "9701": {"387141": 1, "357739": 1}, "9733": {"526073": 1, "110163": 1, "38655": 1}, "9735": {"392317": 1}, "9737": {"245082": 1, "419897": 1, "99131": 1}, "9771": {"263955": 1, "28740": 1}, "9808": {"40702": 1, "557582": 1, "431946": 1}, "9824": {"574777": 1}, "9871": {"448890": 1, "76562": 1, "40051": 1, "170594": 1}, "9882": {"65702": 1}, "9925": {"374309": 1, "289120": 1}, "9929": {"266323": 1, "568006": 1}, "9961": {"21311": 1}, "9979": {"538237": 1, "35633": 1, "337243": 1, "35369": 1}, "10034": {"480749": 1, "181942": 1}, "10039": {"67785": 1}, "10109": {"28314": 1, "506374": 1, "93231": 1, "156029": 1, "406974": 1, "499849": 1, "566591": 1}, "10122": {"570787": 1, "139368": 1, "273718": 1, "259084": 1}, "10136": {"526115": 1, "152316": 1, "290930": 1}, "10137": {"57168": 1}, "10152": {"113585": 1}, "10183": {"66858": 1, "314342": 1}, "10213": {"380942": 1, "270221": 1, "545712": 1}, "10246": {"512984": 1, "77573": 1}, "10267": {"17652": 1, "457108": 1, "424511": 1, "328556": 1, "460398": 1}, "10414": {"79807": 1, "303325": 1}, "10447": {"382236": 1, "152096": 1, "300721": 1}, "10462": {"8266": 1, "11378": 1, "35680": 1, "437879": 1, "204035": 1, "581204": 1}, "10482": {"549072": 1}, "10497": {"304284": 1, "147765": 1, "34913": 1, "398622": 1, "575729": 1, "31483": 1, "159880": 1, "71898": 1, "196423": 1, "445230": 1}, "10526": {"39185": 1, "283008": 1}, "10547": {"571306": 1}, "10558": {"222485": 1, "483268": 1}, "10596": {"284235": 1, "316838": 1, "208070": 1, "164008": 1, "413041": 1}, "10601": {"568064": 1}, "10628": {"588116": 1}, "10639": {"431799": 1, "495774": 1, "278453": 1, "163353": 1, "187039": 1}, "10645": {"588607": 1, "22221": 1}, "10674": {"99857": 1, "257241": 1, "543589": 1, "3095": 1}, "10710": {"6771": 1}, "10734": {"589970": 1, "263481": 1, "470289": 1, "225718": 1, "426278": 1, "62882": 1}, "10792": {"195044": 1, "374956": 1}, "10808": {"28291": 1, "487052": 1, "484599": 1}, "10809": {"103362": 1, "286141": 1, "242524": 1}, "10812": {"451935": 1, "46737": 1, "314455": 1, "342756": 1}, "10827": {"160786": 1, "42301": 1, "7748": 1, "107554": 1, "95282": 1}, "10845": {"191588": 1}, "10912": {"518721": 1}, "10932": {"104134": 1}, "10975": {"61022": 1}, "10979": {"148728": 1, "164001": 1, "362762": 1, "121158": 1}, "10994": {"496820": 1, "212394": 1}, "11039": {"53544": 1, "202768": 1, "353625": 1, "293531": 1, "249063": 1, "79363": 1, "330058": 1, "91183": 1}, "11054": {"155053": 1, "321015": 1}, "11088": {"437100": 1}} \ No newline at end of file diff --git a/evals/benchmark/results/fiqa/run-dense.trec.gz b/evals/benchmark/results/fiqa/run-dense.trec.gz new file mode 100644 index 000000000..4bcbb2395 Binary files /dev/null and b/evals/benchmark/results/fiqa/run-dense.trec.gz differ diff --git a/evals/benchmark/results/fiqa/run-w0.25.trec.gz b/evals/benchmark/results/fiqa/run-w0.25.trec.gz new file mode 100644 index 000000000..f77267400 Binary files /dev/null and b/evals/benchmark/results/fiqa/run-w0.25.trec.gz differ diff --git a/evals/benchmark/results/fiqa/run-w0.5.trec.gz b/evals/benchmark/results/fiqa/run-w0.5.trec.gz new file mode 100644 index 000000000..cb9e813e9 Binary files /dev/null and b/evals/benchmark/results/fiqa/run-w0.5.trec.gz differ diff --git a/evals/benchmark/results/fiqa/run-w0.75.trec.gz b/evals/benchmark/results/fiqa/run-w0.75.trec.gz new file mode 100644 index 000000000..8b8fd4f63 Binary files /dev/null and b/evals/benchmark/results/fiqa/run-w0.75.trec.gz differ diff --git a/evals/benchmark/results/fiqa/run-w1.0.trec.gz b/evals/benchmark/results/fiqa/run-w1.0.trec.gz new file mode 100644 index 000000000..af3d9b25e Binary files /dev/null and b/evals/benchmark/results/fiqa/run-w1.0.trec.gz differ diff --git a/evals/benchmark/results/fiqa/stats-w0.25-vs-dense.jsonl b/evals/benchmark/results/fiqa/stats-w0.25-vs-dense.jsonl new file mode 100644 index 000000000..1656c8b26 --- /dev/null +++ b/evals/benchmark/results/fiqa/stats-w0.25-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w0.25", "arm_b": "dense"} +{"row_type": "ir", "dataset": "fiqa", "metric": "nDCG@10", "n": 648, "mean_a": 0.4363178119525206, "mean_b": 0.45978106466228363, "mean_diff": 0.023463252709763096, "ci_low": 0.011821418732420208, "ci_high": 0.03564233972463353, "p_value": 0.00039996000399960006, "significant": true} +{"row_type": "ir", "dataset": "fiqa", "metric": "Recall@20", "n": 648, "mean_a": 0.6186453678351826, "mean_b": 0.6186453678351826, "mean_diff": 0.0, "ci_low": 0.0, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "fiqa", "metric": "MRR@10", "n": 648, "mean_a": 0.5093296382359778, "mean_b": 0.5447689828844154, "mean_diff": 0.03543934464843764, "ci_low": 0.017134746294228436, "ci_high": 0.05425535648253711, "p_value": 0.0004999500049995, "significant": true} diff --git a/evals/benchmark/results/fiqa/stats-w0.5-vs-dense.jsonl b/evals/benchmark/results/fiqa/stats-w0.5-vs-dense.jsonl new file mode 100644 index 000000000..d08667b9d --- /dev/null +++ b/evals/benchmark/results/fiqa/stats-w0.5-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w0.5", "arm_b": "dense"} +{"row_type": "ir", "dataset": "fiqa", "metric": "nDCG@10", "n": 648, "mean_a": 0.43334553413585447, "mean_b": 0.45978106466228363, "mean_diff": 0.026435530526429246, "ci_low": 0.013750582213694486, "ci_high": 0.039414787702170946, "p_value": 0.00039996000399960006, "significant": true} +{"row_type": "ir", "dataset": "fiqa", "metric": "Recall@20", "n": 648, "mean_a": 0.6186453678351826, "mean_b": 0.6186453678351826, "mean_diff": 0.0, "ci_low": 0.0, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "fiqa", "metric": "MRR@10", "n": 648, "mean_a": 0.5052615655336459, "mean_b": 0.5447689828844154, "mean_diff": 0.039507417350769604, "ci_low": 0.01943150819023354, "ci_high": 0.059518900213452144, "p_value": 0.0004999500049995, "significant": true} diff --git a/evals/benchmark/results/fiqa/stats-w0.75-vs-dense.jsonl b/evals/benchmark/results/fiqa/stats-w0.75-vs-dense.jsonl new file mode 100644 index 000000000..b893e1676 --- /dev/null +++ b/evals/benchmark/results/fiqa/stats-w0.75-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w0.75", "arm_b": "dense"} +{"row_type": "ir", "dataset": "fiqa", "metric": "nDCG@10", "n": 648, "mean_a": 0.428901545232448, "mean_b": 0.45978106466228363, "mean_diff": 0.03087951942983565, "ci_low": 0.017526607126724953, "ci_high": 0.04435456873743836, "p_value": 0.00029997000299970003, "significant": true} +{"row_type": "ir", "dataset": "fiqa", "metric": "Recall@20", "n": 648, "mean_a": 0.6186453678351826, "mean_b": 0.6186453678351826, "mean_diff": 0.0, "ci_low": 0.0, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "fiqa", "metric": "MRR@10", "n": 648, "mean_a": 0.49862631977802974, "mean_b": 0.5447689828844154, "mean_diff": 0.046142663106385734, "ci_low": 0.024747511245256454, "ci_high": 0.06754186994436374, "p_value": 0.00029997000299970003, "significant": true} diff --git a/evals/benchmark/results/fiqa/stats-w1.0-vs-dense.jsonl b/evals/benchmark/results/fiqa/stats-w1.0-vs-dense.jsonl new file mode 100644 index 000000000..78b0d7a8b --- /dev/null +++ b/evals/benchmark/results/fiqa/stats-w1.0-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w1.0", "arm_b": "dense"} +{"row_type": "ir", "dataset": "fiqa", "metric": "nDCG@10", "n": 648, "mean_a": 0.4033465313238571, "mean_b": 0.45978106466228363, "mean_diff": 0.056434533338426525, "ci_low": 0.041732850176846754, "ci_high": 0.0715287296803379, "p_value": 9.999000099990002e-05, "significant": true} +{"row_type": "ir", "dataset": "fiqa", "metric": "Recall@20", "n": 648, "mean_a": 0.5802359206294392, "mean_b": 0.6186453678351826, "mean_diff": 0.038409447205743505, "ci_low": 0.0240065767351647, "ci_high": 0.05309461875144745, "p_value": 9.999000099990002e-05, "significant": true} +{"row_type": "ir", "dataset": "fiqa", "metric": "MRR@10", "n": 648, "mean_a": 0.4776728099076812, "mean_b": 0.5447689828844154, "mean_diff": 0.06709617297673427, "ci_low": 0.04504043701411227, "ci_high": 0.08989294441393272, "p_value": 9.999000099990002e-05, "significant": true} diff --git a/evals/benchmark/results/manifest.frozen.json b/evals/benchmark/results/manifest.frozen.json new file mode 100644 index 000000000..5bda42167 --- /dev/null +++ b/evals/benchmark/results/manifest.frozen.json @@ -0,0 +1,87 @@ +{ + "arms": [ + { + "config": { + "intent_llm": true, + "layout_detection": true, + "neighbor_expansion": 2, + "table_extraction": true, + "title_search": true + }, + "description": "test/retrieval-parity with every parity feature enabled", + "name": "lilbee-parity", + "system": "lilbee" + }, + { + "config": { + "pipeline": "deepdoc" + }, + "description": "RAGFlow DeepDoc pipeline at its defaults", + "name": "ragflow-default", + "system": "ragflow" + } + ], + "datasets": [ + { + "label_kind": "native", + "loader": "scifact", + "name": "scifact", + "split": "test" + }, + { + "label_kind": "native", + "loader": "fiqa", + "name": "fiqa", + "split": "test" + }, + { + "label_kind": "native", + "loader": "nfcorpus", + "name": "nfcorpus", + "split": "test" + }, + { + "label_kind": "native", + "loader": "hotpotqa", + "name": "hotpotqa", + "split": "test" + }, + { + "label_kind": "native", + "loader": "trec-covid", + "name": "trec-covid", + "split": "test" + }, + { + "label_kind": "derived", + "loader": "tatdqa", + "name": "tat-dqa", + "split": "test" + }, + { + "label_kind": "derived", + "loader": "ottqa", + "name": "ott-qa", + "split": "dev" + } + ], + "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", + "metrics": [ + "nDCG@10", + "Recall@20", + "MRR@10" + ], + "models": { + "embedder": "qwen3-embedding", + "generator": "qwen2.5-72b-instruct", + "generator_swap": "mistral-large-2411", + "judge": "fable-5" + }, + "run_id": "parity-2026-07", + "stats": { + "alpha": 0.05, + "bootstrap_resamples": 10000, + "seed": 20260714 + }, + "temperature": 0.0 +} diff --git a/evals/benchmark/results/nfcorpus/ir-dense.jsonl b/evals/benchmark/results/nfcorpus/ir-dense.jsonl new file mode 100644 index 000000000..b591f3163 --- /dev/null +++ b/evals/benchmark/results/nfcorpus/ir-dense.jsonl @@ -0,0 +1 @@ +{"dataset": "nfcorpus", "run_tag": "dense", "aggregated": {"nDCG@10": 0.3666, "Recall@20": 0.2135, "MRR@10": 0.5918}, "per_query": {"nDCG@10": {"PLAIN-2": 0.8291570719562573, "PLAIN-12": 0.2970921024690982, "PLAIN-23": 0.528173471047711, "PLAIN-33": 0.35992808417710315, "PLAIN-44": 0.42763897717969646, "PLAIN-56": 0.8281497384091, "PLAIN-68": 0.4509737963519493, "PLAIN-78": 0.06370866775185362, "PLAIN-91": 0.5734966058043384, "PLAIN-102": 0.2021553370411397, "PLAIN-112": 0.674411193614723, "PLAIN-123": 0.05487565949361938, "PLAIN-133": 0.2669507571941387, "PLAIN-143": 0.6347240446515069, "PLAIN-153": 0.9478557351534421, "PLAIN-165": 0.5494506864770778, "PLAIN-175": 0.42612495998954475, "PLAIN-186": 0.22009176629808017, "PLAIN-196": 0.22009176629808017, "PLAIN-207": 0.45186484096594276, "PLAIN-217": 0.28762634971412726, "PLAIN-227": 0.0, "PLAIN-238": 0.0, "PLAIN-248": 0.2710884709729799, "PLAIN-259": 0.3589542101716347, "PLAIN-270": 0.31488013066763093, "PLAIN-280": 0.23440650992329987, "PLAIN-291": 0.5173633627401372, "PLAIN-307": 0.9325210919548239, "PLAIN-320": 0.0, "PLAIN-332": 0.6131471927654584, "PLAIN-344": 0.5106141061027615, "PLAIN-358": 0.0, "PLAIN-371": 0.38009376671593426, "PLAIN-383": 0.16958010263680806, "PLAIN-395": 0.0, "PLAIN-407": 0.6387878864795979, "PLAIN-418": 0.8670870086853021, "PLAIN-430": 0.6174886499090687, "PLAIN-441": 0.777747077360283, "PLAIN-457": 0.0, "PLAIN-468": 0.4318845862516251, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 0.5107164012496701, "PLAIN-510": 0.0, "PLAIN-520": 0.0, "PLAIN-531": 0.6114735274236563, "PLAIN-541": 0.09546043308946733, "PLAIN-551": 0.7903864795495061, "PLAIN-561": 0.43735247915031, "PLAIN-571": 0.2162957183053307, "PLAIN-583": 0.22009176629808017, "PLAIN-593": 0.46927872602275644, "PLAIN-603": 0.0, "PLAIN-613": 0.11004588314904008, "PLAIN-623": 0.33013764944712026, "PLAIN-634": 0.424926013816671, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.4519792040377807, "PLAIN-711": 0.7353797772725617, "PLAIN-721": 1.0, "PLAIN-731": 0.7917063341896681, "PLAIN-741": 0.8701249883466593, "PLAIN-751": 0.0, "PLAIN-761": 0.15130120674940672, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.644824486427155, "PLAIN-806": 0.8603818544462509, "PLAIN-817": 0.43067655807339306, "PLAIN-827": 0.6758704024811183, "PLAIN-838": 0.7767471075223497, "PLAIN-850": 0.6530488081313035, "PLAIN-872": 0.0, "PLAIN-882": 0.43231813227099475, "PLAIN-892": 0.3557772116892465, "PLAIN-902": 0.5294362295028773, "PLAIN-913": 0.7967610662472993, "PLAIN-924": 0.30523488393970116, "PLAIN-934": 0.611755947233009, "PLAIN-946": 0.22009176629808017, "PLAIN-956": 0.7948833326720564, "PLAIN-966": 0.0, "PLAIN-977": 0.46927872602275644, "PLAIN-987": 0.0, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.4263100803427562, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.07839826897867533, "PLAIN-1066": 0.0, "PLAIN-1088": 0.5531464700081437, "PLAIN-1098": 0.0, "PLAIN-1109": 0.762416491594312, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.22009176629808017, "PLAIN-1151": 0.7471589728148227, "PLAIN-1161": 0.0, "PLAIN-1172": 0.22009176629808017, "PLAIN-1183": 0.16421958630632802, "PLAIN-1193": 0.06362078819895171, "PLAIN-1203": 0.48471198902551854, "PLAIN-1214": 0.0, "PLAIN-1225": 0.7564477297898766, "PLAIN-1236": 0.3391602052736161, "PLAIN-1249": 0.0, "PLAIN-1262": 0.4690000933206748, "PLAIN-1275": 0.727329844310522, "PLAIN-1288": 0.15457433957839825, "PLAIN-1299": 0.3589542101716347, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5842273861585908, "PLAIN-1331": 0.0, "PLAIN-1342": 0.5423640154200349, "PLAIN-1353": 0.0, "PLAIN-1363": 0.08514311764162098, "PLAIN-1374": 0.2051166673279436, "PLAIN-1387": 0.7788065869851872, "PLAIN-1398": 1.0, "PLAIN-1409": 0.513528549750033, "PLAIN-1419": 0.4690000933206748, "PLAIN-1429": 0.0, "PLAIN-1441": 0.5684579545875155, "PLAIN-1453": 0.3241295975188817, "PLAIN-1463": 0.4351098955930419, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.30523488393970116, "PLAIN-1516": 0.0, "PLAIN-1527": 0.9216017310213247, "PLAIN-1537": 0.6618313225363274, "PLAIN-1547": 0.0, "PLAIN-1557": 0.2489083270225946, "PLAIN-1568": 1.0, "PLAIN-1579": 0.3391602052736161, "PLAIN-1590": 0.06943122193677727, "PLAIN-1601": 0.7222954974312066, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.7799082337019199, "PLAIN-1645": 0.0, "PLAIN-1656": 0.0, "PLAIN-1667": 0.725452110735279, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 0.07336392209936005, "PLAIN-1731": 0.7653606369886217, "PLAIN-1741": 1.0, "PLAIN-1752": 0.0, "PLAIN-1762": 0.5026317885449211, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 0.6131471927654584, "PLAIN-1805": 1.0, "PLAIN-1817": 0.3589542101716347, "PLAIN-1837": 1.0, "PLAIN-1847": 0.06943122193677727, "PLAIN-1857": 0.6207622843987103, "PLAIN-1867": 0.0979129253499678, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.727329844310522, "PLAIN-1919": 0.8630152897016883, "PLAIN-1929": 0.2863459897524692, "PLAIN-1940": 0.3589542101716347, "PLAIN-1950": 0.21726071285222986, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.9266360779006401, "PLAIN-1995": 0.0, "PLAIN-2009": 0.31442295467135667, "PLAIN-2019": 0.4632423659320675, "PLAIN-2030": 0.0, "PLAIN-2040": 0.5141863642577489, "PLAIN-2051": 0.7538142945344393, "PLAIN-2061": 0.9216017310213247, "PLAIN-2071": 0.5103515512676448, "PLAIN-2081": 1.0, "PLAIN-2092": 0.6489315753318465, "PLAIN-2102": 0.8263333286520083, "PLAIN-2113": 0.0, "PLAIN-2124": 0.6906478832608419, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.644824486427155, "PLAIN-2167": 0.0, "PLAIN-2177": 0.5766882048947064, "PLAIN-2187": 0.10122860821230018, "PLAIN-2197": 0.8643145546088337, "PLAIN-2209": 0.0, "PLAIN-2220": 0.5526193270751085, "PLAIN-2230": 0.0, "PLAIN-2240": 0.43735247915031, "PLAIN-2250": 0.0, "PLAIN-2261": 0.6489315753318465, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.10699313236726378, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.45449827622138, "PLAIN-2343": 0.0, "PLAIN-2354": 0.48522855511632257, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.3932783996463063, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.5115716521414854, "PLAIN-2440": 0.09478836436955078, "PLAIN-2450": 0.22009176629808017, "PLAIN-2460": 0.15081553897217942, "PLAIN-2470": 0.6426210812570098, "PLAIN-2480": 0.19682836491904007, "PLAIN-2490": 0.311877917906475, "PLAIN-2500": 0.11974534227438745, "PLAIN-2510": 0.6402900881481708, "PLAIN-2520": 0.2536190390062209, "PLAIN-2530": 0.9652843890316114, "PLAIN-2540": 0.3620713465858444, "PLAIN-2550": 0.0, "PLAIN-2560": 0.9266360779006401, "PLAIN-2570": 0.4260948308642638, "PLAIN-2580": 0.2996484034259831, "PLAIN-2590": 0.17764036563972363, "PLAIN-2600": 0.6232826076301087, "PLAIN-2610": 0.4788129387816444, "PLAIN-2620": 0.4472723157606293, "PLAIN-2630": 0.8167922648045142, "PLAIN-2640": 0.5243071773297889, "PLAIN-2650": 0.70844095012329, "PLAIN-2660": 0.2863459897524692, "PLAIN-2670": 0.35301686087887274, "PLAIN-2680": 0.4236154895010551, "PLAIN-2690": 0.5815631499093725, "PLAIN-2700": 0.27887574473268145, "PLAIN-2710": 0.6779733527921183, "PLAIN-2720": 0.22009176629808017, "PLAIN-2730": 0.6433495827340141, "PLAIN-2740": 0.6706555576145379, "PLAIN-2750": 0.5644487757556916, "PLAIN-2760": 0.26612017649871916, "PLAIN-2770": 0.29666344146914797, "PLAIN-2780": 0.4786771405648691, "PLAIN-2790": 0.7482999096553582, "PLAIN-2800": 0.057959913024345, "PLAIN-2810": 0.0, "PLAIN-2820": 0.15619484341784115, "PLAIN-2830": 0.6387878864795979, "PLAIN-2840": 0.0, "PLAIN-2850": 0.38062652754129855, "PLAIN-2860": 1.0, "PLAIN-2870": 0.09797563950575583, "PLAIN-2880": 0.0, "PLAIN-2890": 0.1803895185407797, "PLAIN-2900": 0.17759747898211195, "PLAIN-2910": 0.3239134396887189, "PLAIN-2920": 0.0, "PLAIN-2930": 0.43056322872474745, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.3273949503887395, "PLAIN-2970": 0.38062652754129855, "PLAIN-2981": 0.43122038713502525, "PLAIN-2991": 0.38685280723454163, "PLAIN-3001": 0.38685280723454163, "PLAIN-3014": 0.0, "PLAIN-3026": 0.6309297535714575, "PLAIN-3037": 0.0, "PLAIN-3053": 0.7653606369886217, "PLAIN-3063": 0.0, "PLAIN-3074": 1.0, "PLAIN-3085": 0.5012658353418871, "PLAIN-3097": 0.4770382338730848, "PLAIN-3116": 0.0, "PLAIN-3131": 0.42598177852341024, "PLAIN-3141": 0.1290515430196899, "PLAIN-3151": 0.7320357309750318, "PLAIN-3161": 0.5396140508112093, "PLAIN-3171": 0.07768953695765952, "PLAIN-3181": 0.07839826897867533, "PLAIN-3191": 0.2814853444092611, "PLAIN-3201": 0.4209088765951527, "PLAIN-3211": 0.0, "PLAIN-3221": 0.05430265598415857, "PLAIN-3231": 0.0, "PLAIN-3241": 0.4384685740392189, "PLAIN-3251": 0.1803895185407797, "PLAIN-3261": 0.11591982604869, "PLAIN-3271": 0.0, "PLAIN-3281": 0.16195671984435944, "PLAIN-3292": 0.6110511850146623, "PLAIN-3302": 0.0, "PLAIN-3312": 0.2763339555934507, "PLAIN-3322": 0.07336392209936005, "PLAIN-3332": 0.7601167308693849, "PLAIN-3342": 0.7039180890341347, "PLAIN-3352": 0.0, "PLAIN-3362": 0.42142110103371916, "PLAIN-3372": 0.3607790370815594, "PLAIN-3382": 0.5726085298924031, "PLAIN-3392": 0.06362078819895171, "PLAIN-3402": 0.0, "PLAIN-3412": 0.12973219059467986, "PLAIN-3422": 0.039229421860310865, "PLAIN-3432": 0.18340980524840012, "PLAIN-3442": 0.14074267220463055, "PLAIN-3452": 0.3865668514110075, "PLAIN-3462": 0.5847192329112588, "PLAIN-3472": 0.08421358772610596}, "Recall@20": {"PLAIN-2": 0.4166666666666667, "PLAIN-12": 0.13333333333333333, "PLAIN-23": 0.06666666666666667, "PLAIN-33": 0.125, "PLAIN-44": 0.05172413793103448, "PLAIN-56": 0.2, "PLAIN-68": 0.07272727272727272, "PLAIN-78": 0.016129032258064516, "PLAIN-91": 0.15625, "PLAIN-102": 0.08333333333333333, "PLAIN-112": 0.08928571428571429, "PLAIN-123": 0.0196078431372549, "PLAIN-133": 0.027777777777777776, "PLAIN-143": 0.17647058823529413, "PLAIN-153": 0.23404255319148937, "PLAIN-165": 0.2222222222222222, "PLAIN-175": 0.10810810810810811, "PLAIN-186": 0.041666666666666664, "PLAIN-196": 0.014492753623188406, "PLAIN-207": 0.14545454545454545, "PLAIN-217": 0.16666666666666666, "PLAIN-227": 0.025, "PLAIN-238": 0.0, "PLAIN-248": 0.16129032258064516, "PLAIN-259": 0.21428571428571427, "PLAIN-270": 0.057692307692307696, "PLAIN-280": 0.25, "PLAIN-291": 0.3076923076923077, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.5, "PLAIN-344": 0.11475409836065574, "PLAIN-358": 0.3333333333333333, "PLAIN-371": 0.5, "PLAIN-383": 0.4, "PLAIN-395": 0.0, "PLAIN-407": 0.3333333333333333, "PLAIN-418": 0.6666666666666666, "PLAIN-430": 0.375, "PLAIN-441": 0.8, "PLAIN-457": 0.0, "PLAIN-468": 0.2, "PLAIN-478": 0.0, "PLAIN-488": 0.7333333333333333, "PLAIN-499": 0.11764705882352941, "PLAIN-510": 0.16666666666666666, "PLAIN-520": 0.0, "PLAIN-531": 0.03896103896103896, "PLAIN-541": 0.16666666666666666, "PLAIN-551": 1.0, "PLAIN-561": 0.12, "PLAIN-571": 0.42857142857142855, "PLAIN-583": 0.022727272727272728, "PLAIN-593": 0.3333333333333333, "PLAIN-603": 0.0, "PLAIN-613": 0.1, "PLAIN-623": 0.12195121951219512, "PLAIN-634": 0.14285714285714285, "PLAIN-645": 0.0, "PLAIN-660": 0.03368421052631579, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.375, "PLAIN-711": 0.21052631578947367, "PLAIN-721": 0.68, "PLAIN-731": 0.2037037037037037, "PLAIN-741": 0.47368421052631576, "PLAIN-751": 0.0, "PLAIN-761": 0.16666666666666666, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.5, "PLAIN-806": 0.13541666666666666, "PLAIN-817": 1.0, "PLAIN-827": 0.040983606557377046, "PLAIN-838": 0.6666666666666666, "PLAIN-850": 0.19298245614035087, "PLAIN-872": 0.0, "PLAIN-882": 0.2, "PLAIN-892": 0.06521739130434782, "PLAIN-902": 0.5, "PLAIN-913": 0.23529411764705882, "PLAIN-924": 0.0625, "PLAIN-934": 0.09900990099009901, "PLAIN-946": 0.03225806451612903, "PLAIN-956": 0.05825242718446602, "PLAIN-966": 0.0, "PLAIN-977": 0.3333333333333333, "PLAIN-987": 0.3333333333333333, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.1, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.010638297872340425, "PLAIN-1066": 0.0, "PLAIN-1088": 0.4, "PLAIN-1098": 0.0, "PLAIN-1109": 0.11224489795918367, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.0625, "PLAIN-1151": 0.06535947712418301, "PLAIN-1161": 0.0, "PLAIN-1172": 0.05263157894736842, "PLAIN-1183": 0.08, "PLAIN-1193": 0.05555555555555555, "PLAIN-1203": 0.3333333333333333, "PLAIN-1214": 0.058823529411764705, "PLAIN-1225": 0.3333333333333333, "PLAIN-1236": 0.2, "PLAIN-1249": 0.0, "PLAIN-1262": 0.3, "PLAIN-1275": 0.16363636363636364, "PLAIN-1288": 0.02040816326530612, "PLAIN-1299": 0.03278688524590164, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5, "PLAIN-1331": 0.0, "PLAIN-1342": 0.3125, "PLAIN-1353": 0.0, "PLAIN-1363": 0.09090909090909091, "PLAIN-1374": 0.07692307692307693, "PLAIN-1387": 0.5833333333333334, "PLAIN-1398": 0.1553398058252427, "PLAIN-1409": 0.02766798418972332, "PLAIN-1419": 0.06153846153846154, "PLAIN-1429": 0.0, "PLAIN-1441": 0.04838709677419355, "PLAIN-1453": 0.044897959183673466, "PLAIN-1463": 0.20833333333333334, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.13333333333333333, "PLAIN-1516": 0.0, "PLAIN-1527": 0.06930693069306931, "PLAIN-1537": 0.13333333333333333, "PLAIN-1547": 0.0, "PLAIN-1557": 0.07692307692307693, "PLAIN-1568": 1.0, "PLAIN-1579": 0.2, "PLAIN-1590": 0.05555555555555555, "PLAIN-1601": 0.0945945945945946, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.030162412993039442, "PLAIN-1645": 0.16666666666666666, "PLAIN-1656": 0.03571428571428571, "PLAIN-1667": 0.06918238993710692, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 0.9375, "PLAIN-1721": 0.03225806451612903, "PLAIN-1731": 0.6666666666666666, "PLAIN-1741": 0.04523809523809524, "PLAIN-1752": 1.0, "PLAIN-1762": 0.5, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 0.13793103448275862, "PLAIN-1817": 0.2222222222222222, "PLAIN-1837": 0.08673469387755102, "PLAIN-1847": 0.18181818181818182, "PLAIN-1857": 0.24242424242424243, "PLAIN-1867": 0.14285714285714285, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.023758099352051837, "PLAIN-1919": 0.3333333333333333, "PLAIN-1929": 0.3, "PLAIN-1940": 0.16666666666666666, "PLAIN-1950": 0.23076923076923078, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.28125, "PLAIN-1995": 0.09090909090909091, "PLAIN-2009": 0.375, "PLAIN-2019": 0.6666666666666666, "PLAIN-2030": 0.0, "PLAIN-2040": 0.05303030303030303, "PLAIN-2051": 0.030985915492957747, "PLAIN-2061": 0.04390243902439024, "PLAIN-2071": 0.15555555555555556, "PLAIN-2081": 1.0, "PLAIN-2092": 0.5, "PLAIN-2102": 0.030878859857482184, "PLAIN-2113": 0.0, "PLAIN-2124": 0.35294117647058826, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.5, "PLAIN-2167": 0.0, "PLAIN-2177": 0.20833333333333334, "PLAIN-2187": 0.1111111111111111, "PLAIN-2197": 0.1724137931034483, "PLAIN-2209": 0.0, "PLAIN-2220": 0.4444444444444444, "PLAIN-2230": 0.0, "PLAIN-2240": 0.36363636363636365, "PLAIN-2250": 0.0, "PLAIN-2261": 0.11666666666666667, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.4, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.09278350515463918, "PLAIN-2343": 0.0, "PLAIN-2354": 0.4, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.3684210526315789, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.13043478260869565, "PLAIN-2440": 0.047619047619047616, "PLAIN-2450": 0.07894736842105263, "PLAIN-2460": 0.0967741935483871, "PLAIN-2470": 0.1643835616438356, "PLAIN-2480": 0.16666666666666666, "PLAIN-2490": 0.13043478260869565, "PLAIN-2500": 0.42857142857142855, "PLAIN-2510": 0.21739130434782608, "PLAIN-2520": 0.0821917808219178, "PLAIN-2530": 0.20833333333333334, "PLAIN-2540": 0.38095238095238093, "PLAIN-2550": 0.04, "PLAIN-2560": 0.47619047619047616, "PLAIN-2570": 0.075, "PLAIN-2580": 0.08695652173913043, "PLAIN-2590": 0.07936507936507936, "PLAIN-2600": 0.4, "PLAIN-2610": 0.11475409836065574, "PLAIN-2620": 0.18181818181818182, "PLAIN-2630": 0.1875, "PLAIN-2640": 0.22916666666666666, "PLAIN-2650": 0.2413793103448276, "PLAIN-2660": 0.14814814814814814, "PLAIN-2670": 0.05714285714285714, "PLAIN-2680": 0.35714285714285715, "PLAIN-2690": 0.20930232558139536, "PLAIN-2700": 0.2, "PLAIN-2710": 0.23809523809523808, "PLAIN-2720": 0.05555555555555555, "PLAIN-2730": 0.20689655172413793, "PLAIN-2740": 0.12, "PLAIN-2750": 0.15254237288135594, "PLAIN-2760": 0.25, "PLAIN-2770": 0.21951219512195122, "PLAIN-2780": 0.3181818181818182, "PLAIN-2790": 0.15254237288135594, "PLAIN-2800": 0.08333333333333333, "PLAIN-2810": 0.125, "PLAIN-2820": 0.125, "PLAIN-2830": 0.3333333333333333, "PLAIN-2840": 0.0, "PLAIN-2850": 0.1111111111111111, "PLAIN-2860": 1.0, "PLAIN-2870": 0.2, "PLAIN-2880": 0.0, "PLAIN-2890": 0.034482758620689655, "PLAIN-2900": 0.08333333333333333, "PLAIN-2910": 0.06666666666666667, "PLAIN-2920": 0.0, "PLAIN-2930": 0.14285714285714285, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.5, "PLAIN-2970": 0.4444444444444444, "PLAIN-2981": 0.14285714285714285, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.6666666666666666, "PLAIN-3063": 0.25, "PLAIN-3074": 1.0, "PLAIN-3085": 1.0, "PLAIN-3097": 0.5, "PLAIN-3116": 0.0, "PLAIN-3131": 0.3333333333333333, "PLAIN-3141": 0.029411764705882353, "PLAIN-3151": 0.5, "PLAIN-3161": 0.13953488372093023, "PLAIN-3171": 0.029411764705882353, "PLAIN-3181": 0.037037037037037035, "PLAIN-3191": 0.1875, "PLAIN-3201": 0.09375, "PLAIN-3211": 0.0, "PLAIN-3221": 0.125, "PLAIN-3231": 0.25, "PLAIN-3241": 0.06896551724137931, "PLAIN-3251": 0.09523809523809523, "PLAIN-3261": 0.07692307692307693, "PLAIN-3271": 0.0, "PLAIN-3281": 0.038461538461538464, "PLAIN-3292": 0.7142857142857143, "PLAIN-3302": 0.08108108108108109, "PLAIN-3312": 0.5384615384615384, "PLAIN-3322": 0.02531645569620253, "PLAIN-3332": 0.6428571428571429, "PLAIN-3342": 0.6666666666666666, "PLAIN-3352": 0.02564102564102564, "PLAIN-3362": 0.18181818181818182, "PLAIN-3372": 0.07142857142857142, "PLAIN-3382": 0.5714285714285714, "PLAIN-3392": 0.13043478260869565, "PLAIN-3402": 0.07142857142857142, "PLAIN-3412": 0.061224489795918366, "PLAIN-3422": 0.0967741935483871, "PLAIN-3432": 0.09523809523809523, "PLAIN-3442": 0.034482758620689655, "PLAIN-3452": 0.29411764705882354, "PLAIN-3462": 0.1875, "PLAIN-3472": 0.058823529411764705}, "MRR@10": {"PLAIN-2": 1.0, "PLAIN-12": 1.0, "PLAIN-23": 1.0, "PLAIN-33": 1.0, "PLAIN-44": 1.0, "PLAIN-56": 1.0, "PLAIN-68": 1.0, "PLAIN-78": 0.3333333333333333, "PLAIN-91": 1.0, "PLAIN-102": 0.25, "PLAIN-112": 1.0, "PLAIN-123": 0.25, "PLAIN-133": 1.0, "PLAIN-143": 1.0, "PLAIN-153": 1.0, "PLAIN-165": 0.5, "PLAIN-175": 1.0, "PLAIN-186": 1.0, "PLAIN-196": 1.0, "PLAIN-207": 1.0, "PLAIN-217": 0.5, "PLAIN-227": 0.058823529411764705, "PLAIN-238": 0.0, "PLAIN-248": 0.3333333333333333, "PLAIN-259": 1.0, "PLAIN-270": 1.0, "PLAIN-280": 0.25, "PLAIN-291": 1.0, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 1.0, "PLAIN-344": 0.5, "PLAIN-358": 0.06666666666666667, "PLAIN-371": 1.0, "PLAIN-383": 0.3333333333333333, "PLAIN-395": 0.0, "PLAIN-407": 1.0, "PLAIN-418": 1.0, "PLAIN-430": 1.0, "PLAIN-441": 1.0, "PLAIN-457": 0.0, "PLAIN-468": 1.0, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 1.0, "PLAIN-510": 0.07692307692307693, "PLAIN-520": 0.0, "PLAIN-531": 1.0, "PLAIN-541": 0.125, "PLAIN-551": 1.0, "PLAIN-561": 1.0, "PLAIN-571": 0.25, "PLAIN-583": 1.0, "PLAIN-593": 1.0, "PLAIN-603": 0.0, "PLAIN-613": 0.3333333333333333, "PLAIN-623": 1.0, "PLAIN-634": 1.0, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 1.0, "PLAIN-711": 1.0, "PLAIN-721": 1.0, "PLAIN-731": 1.0, "PLAIN-741": 1.0, "PLAIN-751": 0.0, "PLAIN-761": 0.3333333333333333, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 1.0, "PLAIN-806": 1.0, "PLAIN-817": 0.25, "PLAIN-827": 1.0, "PLAIN-838": 1.0, "PLAIN-850": 1.0, "PLAIN-872": 0.0, "PLAIN-882": 1.0, "PLAIN-892": 1.0, "PLAIN-902": 1.0, "PLAIN-913": 1.0, "PLAIN-924": 1.0, "PLAIN-934": 0.5, "PLAIN-946": 1.0, "PLAIN-956": 1.0, "PLAIN-966": 0.0, "PLAIN-977": 1.0, "PLAIN-987": 0.08333333333333333, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.16666666666666666, "PLAIN-1066": 0.0, "PLAIN-1088": 1.0, "PLAIN-1098": 0.0, "PLAIN-1109": 1.0, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 1.0, "PLAIN-1151": 1.0, "PLAIN-1161": 0.0, "PLAIN-1172": 1.0, "PLAIN-1183": 0.25, "PLAIN-1193": 0.1, "PLAIN-1203": 1.0, "PLAIN-1214": 0.06666666666666667, "PLAIN-1225": 1.0, "PLAIN-1236": 1.0, "PLAIN-1249": 0.0, "PLAIN-1262": 1.0, "PLAIN-1275": 1.0, "PLAIN-1288": 0.2, "PLAIN-1299": 1.0, "PLAIN-1309": 0.0, "PLAIN-1320": 1.0, "PLAIN-1331": 0.0, "PLAIN-1342": 1.0, "PLAIN-1353": 0.0, "PLAIN-1363": 0.2, "PLAIN-1374": 0.5, "PLAIN-1387": 1.0, "PLAIN-1398": 1.0, "PLAIN-1409": 1.0, "PLAIN-1419": 1.0, "PLAIN-1429": 0.0, "PLAIN-1441": 0.5, "PLAIN-1453": 0.3333333333333333, "PLAIN-1463": 1.0, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 1.0, "PLAIN-1516": 0.0, "PLAIN-1527": 1.0, "PLAIN-1537": 1.0, "PLAIN-1547": 0.0, "PLAIN-1557": 0.5, "PLAIN-1568": 1.0, "PLAIN-1579": 1.0, "PLAIN-1590": 0.125, "PLAIN-1601": 1.0, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.5, "PLAIN-1645": 0.07692307692307693, "PLAIN-1656": 0.06666666666666667, "PLAIN-1667": 1.0, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 0.14285714285714285, "PLAIN-1731": 1.0, "PLAIN-1741": 1.0, "PLAIN-1752": 0.05, "PLAIN-1762": 1.0, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 1.0, "PLAIN-1837": 1.0, "PLAIN-1847": 0.125, "PLAIN-1857": 1.0, "PLAIN-1867": 0.16666666666666666, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 1.0, "PLAIN-1919": 1.0, "PLAIN-1929": 1.0, "PLAIN-1940": 1.0, "PLAIN-1950": 0.5, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 1.0, "PLAIN-1995": 0.07142857142857142, "PLAIN-2009": 0.3333333333333333, "PLAIN-2019": 0.5, "PLAIN-2030": 0.0, "PLAIN-2040": 1.0, "PLAIN-2051": 1.0, "PLAIN-2061": 1.0, "PLAIN-2071": 1.0, "PLAIN-2081": 1.0, "PLAIN-2092": 1.0, "PLAIN-2102": 1.0, "PLAIN-2113": 0.0, "PLAIN-2124": 1.0, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 1.0, "PLAIN-2167": 0.0, "PLAIN-2177": 1.0, "PLAIN-2187": 0.25, "PLAIN-2197": 1.0, "PLAIN-2209": 0.0, "PLAIN-2220": 1.0, "PLAIN-2230": 0.0, "PLAIN-2240": 1.0, "PLAIN-2250": 0.0, "PLAIN-2261": 1.0, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.125, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 1.0, "PLAIN-2343": 0.0, "PLAIN-2354": 1.0, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 1.0, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 1.0, "PLAIN-2440": 0.25, "PLAIN-2450": 1.0, "PLAIN-2460": 0.2, "PLAIN-2470": 1.0, "PLAIN-2480": 0.3333333333333333, "PLAIN-2490": 0.5, "PLAIN-2500": 0.5, "PLAIN-2510": 0.5, "PLAIN-2520": 0.5, "PLAIN-2530": 1.0, "PLAIN-2540": 0.25, "PLAIN-2550": 0.09090909090909091, "PLAIN-2560": 1.0, "PLAIN-2570": 1.0, "PLAIN-2580": 1.0, "PLAIN-2590": 0.5, "PLAIN-2600": 1.0, "PLAIN-2610": 1.0, "PLAIN-2620": 1.0, "PLAIN-2630": 1.0, "PLAIN-2640": 1.0, "PLAIN-2650": 1.0, "PLAIN-2660": 1.0, "PLAIN-2670": 1.0, "PLAIN-2680": 0.5, "PLAIN-2690": 1.0, "PLAIN-2700": 0.3333333333333333, "PLAIN-2710": 1.0, "PLAIN-2720": 1.0, "PLAIN-2730": 1.0, "PLAIN-2740": 1.0, "PLAIN-2750": 1.0, "PLAIN-2760": 0.2, "PLAIN-2770": 0.5, "PLAIN-2780": 1.0, "PLAIN-2790": 1.0, "PLAIN-2800": 0.2, "PLAIN-2810": 0.07692307692307693, "PLAIN-2820": 0.2, "PLAIN-2830": 1.0, "PLAIN-2840": 0.0, "PLAIN-2850": 1.0, "PLAIN-2860": 1.0, "PLAIN-2870": 0.2, "PLAIN-2880": 0.0, "PLAIN-2890": 0.3333333333333333, "PLAIN-2900": 0.5, "PLAIN-2910": 1.0, "PLAIN-2920": 0.0, "PLAIN-2930": 1.0, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.25, "PLAIN-2970": 1.0, "PLAIN-2981": 1.0, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 0.5, "PLAIN-3037": 0.0, "PLAIN-3053": 1.0, "PLAIN-3063": 0.08333333333333333, "PLAIN-3074": 1.0, "PLAIN-3085": 0.25, "PLAIN-3097": 1.0, "PLAIN-3116": 0.0, "PLAIN-3131": 1.0, "PLAIN-3141": 0.25, "PLAIN-3151": 1.0, "PLAIN-3161": 1.0, "PLAIN-3171": 0.25, "PLAIN-3181": 0.16666666666666666, "PLAIN-3191": 1.0, "PLAIN-3201": 1.0, "PLAIN-3211": 0.0, "PLAIN-3221": 0.1111111111111111, "PLAIN-3231": 0.08333333333333333, "PLAIN-3241": 1.0, "PLAIN-3251": 0.3333333333333333, "PLAIN-3261": 0.2, "PLAIN-3271": 0.0, "PLAIN-3281": 0.3333333333333333, "PLAIN-3292": 1.0, "PLAIN-3302": 0.09090909090909091, "PLAIN-3312": 0.5, "PLAIN-3322": 0.14285714285714285, "PLAIN-3332": 1.0, "PLAIN-3342": 1.0, "PLAIN-3352": 0.06666666666666667, "PLAIN-3362": 1.0, "PLAIN-3372": 1.0, "PLAIN-3382": 1.0, "PLAIN-3392": 0.1, "PLAIN-3402": 0.08333333333333333, "PLAIN-3412": 0.3333333333333333, "PLAIN-3422": 0.14285714285714285, "PLAIN-3432": 0.3333333333333333, "PLAIN-3442": 1.0, "PLAIN-3452": 1.0, "PLAIN-3462": 0.5, "PLAIN-3472": 0.125}}} diff --git a/evals/benchmark/results/nfcorpus/ir-w0.25.jsonl b/evals/benchmark/results/nfcorpus/ir-w0.25.jsonl new file mode 100644 index 000000000..f32db31e0 --- /dev/null +++ b/evals/benchmark/results/nfcorpus/ir-w0.25.jsonl @@ -0,0 +1 @@ +{"dataset": "nfcorpus", "run_tag": "w0.25", "aggregated": {"nDCG@10": 0.3767, "Recall@20": 0.2135, "MRR@10": 0.6016}, "per_query": {"nDCG@10": {"PLAIN-2": 0.7495470737433306, "PLAIN-12": 0.16950960215087577, "PLAIN-23": 0.46963035246027246, "PLAIN-33": 0.2640886107812416, "PLAIN-44": 0.2565477167950536, "PLAIN-56": 0.814153485068656, "PLAIN-68": 0.47459215156136747, "PLAIN-78": 0.06370866775185362, "PLAIN-91": 0.6947482816557203, "PLAIN-102": 0.37789901297017203, "PLAIN-112": 0.627217471333092, "PLAIN-123": 0.08039138809008574, "PLAIN-133": 0.13347537859706934, "PLAIN-143": 0.6917350424665718, "PLAIN-153": 1.0, "PLAIN-165": 0.45365777747025504, "PLAIN-175": 0.42027556825478546, "PLAIN-186": 0.22009176629808017, "PLAIN-196": 0.22009176629808017, "PLAIN-207": 0.5330752584449808, "PLAIN-217": 0.13305201013572898, "PLAIN-227": 0.11004588314904008, "PLAIN-238": 0.0, "PLAIN-248": 0.2710884709729799, "PLAIN-259": 0.3589542101716347, "PLAIN-270": 0.2934556883974402, "PLAIN-280": 0.36680070555131283, "PLAIN-291": 0.43231813227099475, "PLAIN-307": 0.9674679834891693, "PLAIN-320": 0.0, "PLAIN-332": 0.6131471927654584, "PLAIN-344": 0.4777009208974601, "PLAIN-358": 0.0, "PLAIN-371": 0.38009376671593426, "PLAIN-383": 0.21398626473452756, "PLAIN-395": 0.0, "PLAIN-407": 0.6387878864795979, "PLAIN-418": 0.8670870086853021, "PLAIN-430": 0.6174886499090687, "PLAIN-441": 0.777747077360283, "PLAIN-457": 0.0, "PLAIN-468": 0.4318845862516251, "PLAIN-478": 0.0, "PLAIN-488": 0.9363792118010483, "PLAIN-499": 0.5174612499126158, "PLAIN-510": 0.0, "PLAIN-520": 0.0, "PLAIN-531": 0.5079937796926363, "PLAIN-541": 0.11706259313796354, "PLAIN-551": 0.7903864795495061, "PLAIN-561": 0.43735247915031, "PLAIN-571": 0.22471947398559278, "PLAIN-583": 0.22009176629808017, "PLAIN-593": 0.46927872602275644, "PLAIN-603": 0.0, "PLAIN-613": 0.11004588314904008, "PLAIN-623": 0.33013764944712026, "PLAIN-634": 0.424926013816671, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.4772656487866461, "PLAIN-711": 0.7605591431973849, "PLAIN-721": 1.0, "PLAIN-731": 0.6758704024811183, "PLAIN-741": 0.8701249883466593, "PLAIN-751": 0.0, "PLAIN-761": 0.15130120674940672, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.644824486427155, "PLAIN-806": 0.857980942822373, "PLAIN-817": 0.43067655807339306, "PLAIN-827": 0.7191184989488184, "PLAIN-838": 0.7767471075223497, "PLAIN-850": 0.6936634693435663, "PLAIN-872": 0.0, "PLAIN-882": 0.4690000933206748, "PLAIN-892": 0.5326208815196265, "PLAIN-902": 0.6366824387328317, "PLAIN-913": 0.800693766409882, "PLAIN-924": 0.30523488393970116, "PLAIN-934": 0.6851198693323691, "PLAIN-946": 0.22009176629808017, "PLAIN-956": 0.7312625444731047, "PLAIN-966": 0.0, "PLAIN-977": 0.46927872602275644, "PLAIN-987": 0.0, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.4263100803427562, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.07839826897867533, "PLAIN-1066": 0.0, "PLAIN-1088": 0.5531464700081437, "PLAIN-1098": 0.0, "PLAIN-1109": 0.6627919420317119, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.22009176629808017, "PLAIN-1151": 0.7453016244178956, "PLAIN-1161": 0.0, "PLAIN-1172": 0.22009176629808017, "PLAIN-1183": 0.1731866333482261, "PLAIN-1193": 0.0, "PLAIN-1203": 0.5036067026046991, "PLAIN-1214": 0.0, "PLAIN-1225": 0.7411902110103873, "PLAIN-1236": 0.3391602052736161, "PLAIN-1249": 0.0, "PLAIN-1262": 0.4690000933206748, "PLAIN-1275": 0.6274092458891772, "PLAIN-1288": 0.4936790360674166, "PLAIN-1299": 0.3589542101716347, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5842273861585908, "PLAIN-1331": 0.0, "PLAIN-1342": 0.5957616317188126, "PLAIN-1353": 0.0, "PLAIN-1363": 0.22009176629808017, "PLAIN-1374": 0.07839826897867533, "PLAIN-1387": 0.7819835854675755, "PLAIN-1398": 1.0, "PLAIN-1409": 0.48068140169045714, "PLAIN-1419": 0.4690000933206748, "PLAIN-1429": 0.0, "PLAIN-1441": 0.6489315753318465, "PLAIN-1453": 0.46035860613699886, "PLAIN-1463": 0.4576752747037683, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.33013764944712026, "PLAIN-1516": 0.0, "PLAIN-1527": 0.841492960259019, "PLAIN-1537": 0.5141863642577489, "PLAIN-1547": 0.0, "PLAIN-1557": 0.2489083270225946, "PLAIN-1568": 1.0, "PLAIN-1579": 0.3391602052736161, "PLAIN-1590": 0.08514311764162098, "PLAIN-1601": 0.800693766409882, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.8643145546088337, "PLAIN-1645": 0.30260241349881345, "PLAIN-1656": 0.0, "PLAIN-1667": 0.7967610662472993, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 0.22009176629808017, "PLAIN-1731": 0.7653606369886217, "PLAIN-1741": 1.0, "PLAIN-1752": 0.0, "PLAIN-1762": 0.4856486183364634, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 0.568003577662161, "PLAIN-1837": 1.0, "PLAIN-1847": 0.22009176629808017, "PLAIN-1857": 0.627507133061656, "PLAIN-1867": 0.27487633291429087, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.7917267193679839, "PLAIN-1919": 0.866947989864271, "PLAIN-1929": 0.1736666713479918, "PLAIN-1940": 0.3589542101716347, "PLAIN-1950": 0.3222722491219547, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.9363792118010483, "PLAIN-1995": 0.0, "PLAIN-2009": 0.31442295467135667, "PLAIN-2019": 0.6364391809889587, "PLAIN-2030": 0.0, "PLAIN-2040": 0.5637884576902256, "PLAIN-2051": 0.7385567757549499, "PLAIN-2061": 0.933745776545611, "PLAIN-2071": 0.513528549750033, "PLAIN-2081": 1.0, "PLAIN-2092": 0.6489315753318465, "PLAIN-2102": 0.8165901947515999, "PLAIN-2113": 0.0, "PLAIN-2124": 0.6906478832608419, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.644824486427155, "PLAIN-2167": 0.0, "PLAIN-2177": 0.5481540639795447, "PLAIN-2187": 0.23504554941448536, "PLAIN-2197": 0.8643145546088337, "PLAIN-2209": 0.0, "PLAIN-2220": 0.5526193270751085, "PLAIN-2230": 0.0, "PLAIN-2240": 0.5637884576902256, "PLAIN-2250": 0.0, "PLAIN-2261": 0.715550648768261, "PLAIN-2271": 0.0, "PLAIN-2281": 0.6309297535714575, "PLAIN-2291": 0.3391602052736161, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.44988737637276544, "PLAIN-2343": 0.0, "PLAIN-2354": 0.5531464700081437, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.31204907722178066, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.4832964623025934, "PLAIN-2440": 0.07336392209936005, "PLAIN-2450": 0.28952298823485745, "PLAIN-2460": 0.2873703978647362, "PLAIN-2470": 0.6923056707325975, "PLAIN-2480": 0.19546838582022094, "PLAIN-2490": 0.4104014996484018, "PLAIN-2500": 0.06760523069231983, "PLAIN-2510": 0.6320787427864673, "PLAIN-2520": 0.2085303067457527, "PLAIN-2530": 0.9633180389503199, "PLAIN-2540": 0.5749687413009162, "PLAIN-2550": 0.0, "PLAIN-2560": 0.866947989864271, "PLAIN-2570": 0.4205032672422578, "PLAIN-2580": 0.3388812950446278, "PLAIN-2590": 0.1581173338315241, "PLAIN-2600": 0.6542721934360559, "PLAIN-2610": 0.3168370282761786, "PLAIN-2620": 0.5549841905774386, "PLAIN-2630": 0.7222954974312066, "PLAIN-2640": 0.5469224065306436, "PLAIN-2650": 0.5384313152574521, "PLAIN-2660": 0.31954962756492084, "PLAIN-2670": 0.37456050428247883, "PLAIN-2680": 0.4236154895010551, "PLAIN-2690": 0.5760823635418191, "PLAIN-2700": 0.2957940082374155, "PLAIN-2710": 0.6665181205254729, "PLAIN-2720": 0.2489083270225946, "PLAIN-2730": 0.6338426833522841, "PLAIN-2740": 0.6854074373766073, "PLAIN-2750": 0.5946665900227162, "PLAIN-2760": 0.2469526638494753, "PLAIN-2770": 0.47694248314715576, "PLAIN-2780": 0.4769783244417837, "PLAIN-2790": 0.8178102956254405, "PLAIN-2800": 0.16152991454925997, "PLAIN-2810": 0.054784966022752124, "PLAIN-2820": 0.17388902521579527, "PLAIN-2830": 0.6387878864795979, "PLAIN-2840": 0.0, "PLAIN-2850": 0.24014860122439108, "PLAIN-2860": 1.0, "PLAIN-2870": 0.08442111803551768, "PLAIN-2880": 0.0, "PLAIN-2890": 0.15537907391531905, "PLAIN-2900": 0.09382844813642036, "PLAIN-2910": 0.3239134396887189, "PLAIN-2920": 0.0, "PLAIN-2930": 0.3607790370815594, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.3273949503887395, "PLAIN-2970": 0.38062652754129855, "PLAIN-2981": 0.43122038713502525, "PLAIN-2991": 0.38685280723454163, "PLAIN-3001": 0.38685280723454163, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.7653606369886217, "PLAIN-3063": 0.0, "PLAIN-3074": 1.0, "PLAIN-3085": 0.5012658353418871, "PLAIN-3097": 0.4770382338730848, "PLAIN-3116": 0.0, "PLAIN-3131": 0.4599311598225878, "PLAIN-3141": 0.1890570933316362, "PLAIN-3151": 0.7605170401458373, "PLAIN-3161": 0.4906128388857768, "PLAIN-3171": 0.06978419164318803, "PLAIN-3181": 0.07839826897867533, "PLAIN-3191": 0.35185668051157637, "PLAIN-3201": 0.484865884708906, "PLAIN-3211": 0.0, "PLAIN-3221": 0.0, "PLAIN-3231": 0.0, "PLAIN-3241": 0.43056322872474745, "PLAIN-3251": 0.24051935805437294, "PLAIN-3261": 0.14982420171299154, "PLAIN-3271": 0.0, "PLAIN-3281": 0.20436662668128655, "PLAIN-3292": 0.5970279223737377, "PLAIN-3302": 0.12438588249677479, "PLAIN-3312": 0.30860285350590366, "PLAIN-3322": 0.06943122193677727, "PLAIN-3332": 0.8909491779135381, "PLAIN-3342": 0.7039180890341347, "PLAIN-3352": 0.0, "PLAIN-3362": 0.4634153650075699, "PLAIN-3372": 0.227626228959616, "PLAIN-3382": 0.5082201284571713, "PLAIN-3392": 0.0, "PLAIN-3402": 0.0, "PLAIN-3412": 0.16483672506293193, "PLAIN-3422": 0.0, "PLAIN-3432": 0.21222636597291458, "PLAIN-3442": 0.07037133610231527, "PLAIN-3452": 0.5216665747946869, "PLAIN-3462": 0.5999767516907479, "PLAIN-3472": 0.08036018528064806}, "Recall@20": {"PLAIN-2": 0.4166666666666667, "PLAIN-12": 0.13333333333333333, "PLAIN-23": 0.06666666666666667, "PLAIN-33": 0.125, "PLAIN-44": 0.05172413793103448, "PLAIN-56": 0.2, "PLAIN-68": 0.07272727272727272, "PLAIN-78": 0.016129032258064516, "PLAIN-91": 0.15625, "PLAIN-102": 0.08333333333333333, "PLAIN-112": 0.08928571428571429, "PLAIN-123": 0.0196078431372549, "PLAIN-133": 0.027777777777777776, "PLAIN-143": 0.17647058823529413, "PLAIN-153": 0.23404255319148937, "PLAIN-165": 0.2222222222222222, "PLAIN-175": 0.10810810810810811, "PLAIN-186": 0.041666666666666664, "PLAIN-196": 0.014492753623188406, "PLAIN-207": 0.14545454545454545, "PLAIN-217": 0.16666666666666666, "PLAIN-227": 0.025, "PLAIN-238": 0.0, "PLAIN-248": 0.16129032258064516, "PLAIN-259": 0.21428571428571427, "PLAIN-270": 0.057692307692307696, "PLAIN-280": 0.25, "PLAIN-291": 0.3076923076923077, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.5, "PLAIN-344": 0.11475409836065574, "PLAIN-358": 0.3333333333333333, "PLAIN-371": 0.5, "PLAIN-383": 0.4, "PLAIN-395": 0.0, "PLAIN-407": 0.3333333333333333, "PLAIN-418": 0.6666666666666666, "PLAIN-430": 0.375, "PLAIN-441": 0.8, "PLAIN-457": 0.0, "PLAIN-468": 0.2, "PLAIN-478": 0.0, "PLAIN-488": 0.7333333333333333, "PLAIN-499": 0.11764705882352941, "PLAIN-510": 0.16666666666666666, "PLAIN-520": 0.0, "PLAIN-531": 0.03896103896103896, "PLAIN-541": 0.16666666666666666, "PLAIN-551": 1.0, "PLAIN-561": 0.12, "PLAIN-571": 0.42857142857142855, "PLAIN-583": 0.022727272727272728, "PLAIN-593": 0.3333333333333333, "PLAIN-603": 0.0, "PLAIN-613": 0.1, "PLAIN-623": 0.12195121951219512, "PLAIN-634": 0.14285714285714285, "PLAIN-645": 0.0, "PLAIN-660": 0.03368421052631579, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.375, "PLAIN-711": 0.21052631578947367, "PLAIN-721": 0.68, "PLAIN-731": 0.2037037037037037, "PLAIN-741": 0.47368421052631576, "PLAIN-751": 0.0, "PLAIN-761": 0.16666666666666666, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.5, "PLAIN-806": 0.13541666666666666, "PLAIN-817": 1.0, "PLAIN-827": 0.040983606557377046, "PLAIN-838": 0.6666666666666666, "PLAIN-850": 0.19298245614035087, "PLAIN-872": 0.0, "PLAIN-882": 0.2, "PLAIN-892": 0.06521739130434782, "PLAIN-902": 0.5, "PLAIN-913": 0.23529411764705882, "PLAIN-924": 0.0625, "PLAIN-934": 0.09900990099009901, "PLAIN-946": 0.03225806451612903, "PLAIN-956": 0.05825242718446602, "PLAIN-966": 0.0, "PLAIN-977": 0.3333333333333333, "PLAIN-987": 0.3333333333333333, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.1, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.010638297872340425, "PLAIN-1066": 0.0, "PLAIN-1088": 0.4, "PLAIN-1098": 0.0, "PLAIN-1109": 0.11224489795918367, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.0625, "PLAIN-1151": 0.06535947712418301, "PLAIN-1161": 0.0, "PLAIN-1172": 0.05263157894736842, "PLAIN-1183": 0.08, "PLAIN-1193": 0.05555555555555555, "PLAIN-1203": 0.3333333333333333, "PLAIN-1214": 0.058823529411764705, "PLAIN-1225": 0.3333333333333333, "PLAIN-1236": 0.2, "PLAIN-1249": 0.0, "PLAIN-1262": 0.3, "PLAIN-1275": 0.16363636363636364, "PLAIN-1288": 0.02040816326530612, "PLAIN-1299": 0.03278688524590164, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5, "PLAIN-1331": 0.0, "PLAIN-1342": 0.3125, "PLAIN-1353": 0.0, "PLAIN-1363": 0.09090909090909091, "PLAIN-1374": 0.07692307692307693, "PLAIN-1387": 0.5833333333333334, "PLAIN-1398": 0.1553398058252427, "PLAIN-1409": 0.02766798418972332, "PLAIN-1419": 0.06153846153846154, "PLAIN-1429": 0.0, "PLAIN-1441": 0.04838709677419355, "PLAIN-1453": 0.044897959183673466, "PLAIN-1463": 0.20833333333333334, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.13333333333333333, "PLAIN-1516": 0.0, "PLAIN-1527": 0.06930693069306931, "PLAIN-1537": 0.13333333333333333, "PLAIN-1547": 0.0, "PLAIN-1557": 0.07692307692307693, "PLAIN-1568": 1.0, "PLAIN-1579": 0.2, "PLAIN-1590": 0.05555555555555555, "PLAIN-1601": 0.0945945945945946, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.030162412993039442, "PLAIN-1645": 0.16666666666666666, "PLAIN-1656": 0.03571428571428571, "PLAIN-1667": 0.06918238993710692, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 0.9375, "PLAIN-1721": 0.03225806451612903, "PLAIN-1731": 0.6666666666666666, "PLAIN-1741": 0.04523809523809524, "PLAIN-1752": 1.0, "PLAIN-1762": 0.5, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 0.13793103448275862, "PLAIN-1817": 0.2222222222222222, "PLAIN-1837": 0.08673469387755102, "PLAIN-1847": 0.18181818181818182, "PLAIN-1857": 0.24242424242424243, "PLAIN-1867": 0.14285714285714285, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.023758099352051837, "PLAIN-1919": 0.3333333333333333, "PLAIN-1929": 0.3, "PLAIN-1940": 0.16666666666666666, "PLAIN-1950": 0.23076923076923078, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.28125, "PLAIN-1995": 0.09090909090909091, "PLAIN-2009": 0.375, "PLAIN-2019": 0.6666666666666666, "PLAIN-2030": 0.0, "PLAIN-2040": 0.05303030303030303, "PLAIN-2051": 0.030985915492957747, "PLAIN-2061": 0.04390243902439024, "PLAIN-2071": 0.15555555555555556, "PLAIN-2081": 1.0, "PLAIN-2092": 0.5, "PLAIN-2102": 0.030878859857482184, "PLAIN-2113": 0.0, "PLAIN-2124": 0.35294117647058826, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.5, "PLAIN-2167": 0.0, "PLAIN-2177": 0.20833333333333334, "PLAIN-2187": 0.1111111111111111, "PLAIN-2197": 0.1724137931034483, "PLAIN-2209": 0.0, "PLAIN-2220": 0.4444444444444444, "PLAIN-2230": 0.0, "PLAIN-2240": 0.36363636363636365, "PLAIN-2250": 0.0, "PLAIN-2261": 0.11666666666666667, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.4, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.09278350515463918, "PLAIN-2343": 0.0, "PLAIN-2354": 0.4, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.3684210526315789, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.13043478260869565, "PLAIN-2440": 0.047619047619047616, "PLAIN-2450": 0.07894736842105263, "PLAIN-2460": 0.0967741935483871, "PLAIN-2470": 0.1643835616438356, "PLAIN-2480": 0.16666666666666666, "PLAIN-2490": 0.13043478260869565, "PLAIN-2500": 0.42857142857142855, "PLAIN-2510": 0.21739130434782608, "PLAIN-2520": 0.0821917808219178, "PLAIN-2530": 0.20833333333333334, "PLAIN-2540": 0.38095238095238093, "PLAIN-2550": 0.04, "PLAIN-2560": 0.47619047619047616, "PLAIN-2570": 0.075, "PLAIN-2580": 0.08695652173913043, "PLAIN-2590": 0.07936507936507936, "PLAIN-2600": 0.4, "PLAIN-2610": 0.11475409836065574, "PLAIN-2620": 0.18181818181818182, "PLAIN-2630": 0.1875, "PLAIN-2640": 0.22916666666666666, "PLAIN-2650": 0.2413793103448276, "PLAIN-2660": 0.14814814814814814, "PLAIN-2670": 0.05714285714285714, "PLAIN-2680": 0.35714285714285715, "PLAIN-2690": 0.20930232558139536, "PLAIN-2700": 0.2, "PLAIN-2710": 0.23809523809523808, "PLAIN-2720": 0.05555555555555555, "PLAIN-2730": 0.20689655172413793, "PLAIN-2740": 0.12, "PLAIN-2750": 0.15254237288135594, "PLAIN-2760": 0.25, "PLAIN-2770": 0.21951219512195122, "PLAIN-2780": 0.3181818181818182, "PLAIN-2790": 0.15254237288135594, "PLAIN-2800": 0.08333333333333333, "PLAIN-2810": 0.125, "PLAIN-2820": 0.125, "PLAIN-2830": 0.3333333333333333, "PLAIN-2840": 0.0, "PLAIN-2850": 0.1111111111111111, "PLAIN-2860": 1.0, "PLAIN-2870": 0.2, "PLAIN-2880": 0.0, "PLAIN-2890": 0.034482758620689655, "PLAIN-2900": 0.08333333333333333, "PLAIN-2910": 0.06666666666666667, "PLAIN-2920": 0.0, "PLAIN-2930": 0.14285714285714285, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.5, "PLAIN-2970": 0.4444444444444444, "PLAIN-2981": 0.14285714285714285, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.6666666666666666, "PLAIN-3063": 0.25, "PLAIN-3074": 1.0, "PLAIN-3085": 1.0, "PLAIN-3097": 0.5, "PLAIN-3116": 0.0, "PLAIN-3131": 0.3333333333333333, "PLAIN-3141": 0.029411764705882353, "PLAIN-3151": 0.5, "PLAIN-3161": 0.13953488372093023, "PLAIN-3171": 0.029411764705882353, "PLAIN-3181": 0.037037037037037035, "PLAIN-3191": 0.1875, "PLAIN-3201": 0.09375, "PLAIN-3211": 0.0, "PLAIN-3221": 0.125, "PLAIN-3231": 0.25, "PLAIN-3241": 0.06896551724137931, "PLAIN-3251": 0.09523809523809523, "PLAIN-3261": 0.07692307692307693, "PLAIN-3271": 0.0, "PLAIN-3281": 0.038461538461538464, "PLAIN-3292": 0.7142857142857143, "PLAIN-3302": 0.08108108108108109, "PLAIN-3312": 0.5384615384615384, "PLAIN-3322": 0.02531645569620253, "PLAIN-3332": 0.6428571428571429, "PLAIN-3342": 0.6666666666666666, "PLAIN-3352": 0.02564102564102564, "PLAIN-3362": 0.18181818181818182, "PLAIN-3372": 0.07142857142857142, "PLAIN-3382": 0.5714285714285714, "PLAIN-3392": 0.13043478260869565, "PLAIN-3402": 0.07142857142857142, "PLAIN-3412": 0.061224489795918366, "PLAIN-3422": 0.0967741935483871, "PLAIN-3432": 0.09523809523809523, "PLAIN-3442": 0.034482758620689655, "PLAIN-3452": 0.29411764705882354, "PLAIN-3462": 0.1875, "PLAIN-3472": 0.058823529411764705}, "MRR@10": {"PLAIN-2": 1.0, "PLAIN-12": 0.3333333333333333, "PLAIN-23": 0.5, "PLAIN-33": 0.5, "PLAIN-44": 0.3333333333333333, "PLAIN-56": 1.0, "PLAIN-68": 1.0, "PLAIN-78": 0.3333333333333333, "PLAIN-91": 1.0, "PLAIN-102": 1.0, "PLAIN-112": 1.0, "PLAIN-123": 0.5, "PLAIN-133": 0.3333333333333333, "PLAIN-143": 1.0, "PLAIN-153": 1.0, "PLAIN-165": 0.5, "PLAIN-175": 1.0, "PLAIN-186": 1.0, "PLAIN-196": 1.0, "PLAIN-207": 1.0, "PLAIN-217": 0.125, "PLAIN-227": 0.3333333333333333, "PLAIN-238": 0.0, "PLAIN-248": 0.3333333333333333, "PLAIN-259": 1.0, "PLAIN-270": 1.0, "PLAIN-280": 0.5, "PLAIN-291": 1.0, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 1.0, "PLAIN-344": 0.5, "PLAIN-358": 0.06666666666666667, "PLAIN-371": 1.0, "PLAIN-383": 0.5, "PLAIN-395": 0.0, "PLAIN-407": 1.0, "PLAIN-418": 1.0, "PLAIN-430": 1.0, "PLAIN-441": 1.0, "PLAIN-457": 0.0, "PLAIN-468": 1.0, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 1.0, "PLAIN-510": 0.07692307692307693, "PLAIN-520": 0.0, "PLAIN-531": 0.3333333333333333, "PLAIN-541": 0.2, "PLAIN-551": 1.0, "PLAIN-561": 1.0, "PLAIN-571": 0.25, "PLAIN-583": 1.0, "PLAIN-593": 1.0, "PLAIN-603": 0.0, "PLAIN-613": 0.3333333333333333, "PLAIN-623": 1.0, "PLAIN-634": 1.0, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 1.0, "PLAIN-711": 1.0, "PLAIN-721": 1.0, "PLAIN-731": 1.0, "PLAIN-741": 1.0, "PLAIN-751": 0.0, "PLAIN-761": 0.3333333333333333, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 1.0, "PLAIN-806": 1.0, "PLAIN-817": 0.25, "PLAIN-827": 1.0, "PLAIN-838": 1.0, "PLAIN-850": 1.0, "PLAIN-872": 0.0, "PLAIN-882": 1.0, "PLAIN-892": 1.0, "PLAIN-902": 1.0, "PLAIN-913": 1.0, "PLAIN-924": 1.0, "PLAIN-934": 0.5, "PLAIN-946": 1.0, "PLAIN-956": 1.0, "PLAIN-966": 0.0, "PLAIN-977": 1.0, "PLAIN-987": 0.07692307692307693, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.16666666666666666, "PLAIN-1066": 0.0, "PLAIN-1088": 1.0, "PLAIN-1098": 0.0, "PLAIN-1109": 1.0, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 1.0, "PLAIN-1151": 1.0, "PLAIN-1161": 0.0, "PLAIN-1172": 1.0, "PLAIN-1183": 0.25, "PLAIN-1193": 0.09090909090909091, "PLAIN-1203": 1.0, "PLAIN-1214": 0.06666666666666667, "PLAIN-1225": 1.0, "PLAIN-1236": 1.0, "PLAIN-1249": 0.0, "PLAIN-1262": 1.0, "PLAIN-1275": 1.0, "PLAIN-1288": 1.0, "PLAIN-1299": 1.0, "PLAIN-1309": 0.0, "PLAIN-1320": 1.0, "PLAIN-1331": 0.0, "PLAIN-1342": 1.0, "PLAIN-1353": 0.0, "PLAIN-1363": 1.0, "PLAIN-1374": 0.16666666666666666, "PLAIN-1387": 1.0, "PLAIN-1398": 1.0, "PLAIN-1409": 0.5, "PLAIN-1419": 1.0, "PLAIN-1429": 0.0, "PLAIN-1441": 1.0, "PLAIN-1453": 0.5, "PLAIN-1463": 1.0, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 1.0, "PLAIN-1516": 0.0, "PLAIN-1527": 1.0, "PLAIN-1537": 1.0, "PLAIN-1547": 0.0, "PLAIN-1557": 0.5, "PLAIN-1568": 1.0, "PLAIN-1579": 1.0, "PLAIN-1590": 0.2, "PLAIN-1601": 1.0, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 1.0, "PLAIN-1645": 1.0, "PLAIN-1656": 0.06666666666666667, "PLAIN-1667": 1.0, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 1.0, "PLAIN-1731": 1.0, "PLAIN-1741": 1.0, "PLAIN-1752": 0.05, "PLAIN-1762": 1.0, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 1.0, "PLAIN-1837": 1.0, "PLAIN-1847": 1.0, "PLAIN-1857": 1.0, "PLAIN-1867": 1.0, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 1.0, "PLAIN-1919": 1.0, "PLAIN-1929": 0.3333333333333333, "PLAIN-1940": 1.0, "PLAIN-1950": 0.5, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 1.0, "PLAIN-1995": 0.06666666666666667, "PLAIN-2009": 0.3333333333333333, "PLAIN-2019": 1.0, "PLAIN-2030": 0.0, "PLAIN-2040": 1.0, "PLAIN-2051": 1.0, "PLAIN-2061": 1.0, "PLAIN-2071": 1.0, "PLAIN-2081": 1.0, "PLAIN-2092": 1.0, "PLAIN-2102": 1.0, "PLAIN-2113": 0.0, "PLAIN-2124": 1.0, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 1.0, "PLAIN-2167": 0.0, "PLAIN-2177": 1.0, "PLAIN-2187": 1.0, "PLAIN-2197": 1.0, "PLAIN-2209": 0.0, "PLAIN-2220": 1.0, "PLAIN-2230": 0.0, "PLAIN-2240": 1.0, "PLAIN-2250": 0.0, "PLAIN-2261": 1.0, "PLAIN-2271": 0.0, "PLAIN-2281": 0.5, "PLAIN-2291": 1.0, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 1.0, "PLAIN-2343": 0.0, "PLAIN-2354": 1.0, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.5, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 1.0, "PLAIN-2440": 0.14285714285714285, "PLAIN-2450": 1.0, "PLAIN-2460": 0.5, "PLAIN-2470": 1.0, "PLAIN-2480": 0.3333333333333333, "PLAIN-2490": 1.0, "PLAIN-2500": 0.16666666666666666, "PLAIN-2510": 0.5, "PLAIN-2520": 0.5, "PLAIN-2530": 1.0, "PLAIN-2540": 1.0, "PLAIN-2550": 0.08333333333333333, "PLAIN-2560": 1.0, "PLAIN-2570": 1.0, "PLAIN-2580": 1.0, "PLAIN-2590": 0.3333333333333333, "PLAIN-2600": 1.0, "PLAIN-2610": 0.3333333333333333, "PLAIN-2620": 1.0, "PLAIN-2630": 1.0, "PLAIN-2640": 1.0, "PLAIN-2650": 1.0, "PLAIN-2660": 0.25, "PLAIN-2670": 1.0, "PLAIN-2680": 0.5, "PLAIN-2690": 1.0, "PLAIN-2700": 0.5, "PLAIN-2710": 1.0, "PLAIN-2720": 1.0, "PLAIN-2730": 1.0, "PLAIN-2740": 1.0, "PLAIN-2750": 1.0, "PLAIN-2760": 0.16666666666666666, "PLAIN-2770": 1.0, "PLAIN-2780": 1.0, "PLAIN-2790": 1.0, "PLAIN-2800": 0.3333333333333333, "PLAIN-2810": 0.14285714285714285, "PLAIN-2820": 0.25, "PLAIN-2830": 1.0, "PLAIN-2840": 0.0, "PLAIN-2850": 0.5, "PLAIN-2860": 1.0, "PLAIN-2870": 0.14285714285714285, "PLAIN-2880": 0.0, "PLAIN-2890": 0.25, "PLAIN-2900": 0.14285714285714285, "PLAIN-2910": 1.0, "PLAIN-2920": 0.0, "PLAIN-2930": 1.0, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.25, "PLAIN-2970": 1.0, "PLAIN-2981": 1.0, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 1.0, "PLAIN-3063": 0.07692307692307693, "PLAIN-3074": 1.0, "PLAIN-3085": 0.25, "PLAIN-3097": 1.0, "PLAIN-3116": 0.0, "PLAIN-3131": 1.0, "PLAIN-3141": 0.5, "PLAIN-3151": 1.0, "PLAIN-3161": 1.0, "PLAIN-3171": 0.2, "PLAIN-3181": 0.16666666666666666, "PLAIN-3191": 1.0, "PLAIN-3201": 1.0, "PLAIN-3211": 0.0, "PLAIN-3221": 0.08333333333333333, "PLAIN-3231": 0.07142857142857142, "PLAIN-3241": 1.0, "PLAIN-3251": 0.3333333333333333, "PLAIN-3261": 0.3333333333333333, "PLAIN-3271": 0.0, "PLAIN-3281": 0.5, "PLAIN-3292": 1.0, "PLAIN-3302": 0.16666666666666666, "PLAIN-3312": 0.5, "PLAIN-3322": 0.125, "PLAIN-3332": 1.0, "PLAIN-3342": 1.0, "PLAIN-3352": 0.06666666666666667, "PLAIN-3362": 1.0, "PLAIN-3372": 0.5, "PLAIN-3382": 1.0, "PLAIN-3392": 0.07692307692307693, "PLAIN-3402": 0.07692307692307693, "PLAIN-3412": 0.5, "PLAIN-3422": 0.08333333333333333, "PLAIN-3432": 0.5, "PLAIN-3442": 0.3333333333333333, "PLAIN-3452": 1.0, "PLAIN-3462": 0.5, "PLAIN-3472": 0.1111111111111111}}} diff --git a/evals/benchmark/results/nfcorpus/ir-w0.5.jsonl b/evals/benchmark/results/nfcorpus/ir-w0.5.jsonl new file mode 100644 index 000000000..184e0fbdb --- /dev/null +++ b/evals/benchmark/results/nfcorpus/ir-w0.5.jsonl @@ -0,0 +1 @@ +{"dataset": "nfcorpus", "run_tag": "w0.5", "aggregated": {"nDCG@10": 0.3756, "Recall@20": 0.2135, "MRR@10": 0.5911}, "per_query": {"nDCG@10": {"PLAIN-2": 0.7495470737433306, "PLAIN-12": 0.16950960215087577, "PLAIN-23": 0.46459600558095715, "PLAIN-33": 0.2640886107812416, "PLAIN-44": 0.23888170027858516, "PLAIN-56": 0.814153485068656, "PLAIN-68": 0.47459215156136747, "PLAIN-78": 0.06370866775185362, "PLAIN-91": 0.6947482816557203, "PLAIN-102": 0.37789901297017203, "PLAIN-112": 0.627217471333092, "PLAIN-123": 0.08039138809008574, "PLAIN-133": 0.13347537859706934, "PLAIN-143": 0.6917350424665718, "PLAIN-153": 1.0, "PLAIN-165": 0.45365777747025504, "PLAIN-175": 0.42027556825478546, "PLAIN-186": 0.22009176629808017, "PLAIN-196": 0.22009176629808017, "PLAIN-207": 0.4230293752959408, "PLAIN-217": 0.13305201013572898, "PLAIN-227": 0.11004588314904008, "PLAIN-238": 0.0, "PLAIN-248": 0.38113435412202, "PLAIN-259": 0.3589542101716347, "PLAIN-270": 0.21222636597291458, "PLAIN-280": 0.39673781793804724, "PLAIN-291": 0.43231813227099475, "PLAIN-307": 0.9674679834891693, "PLAIN-320": 0.0, "PLAIN-332": 0.6131471927654584, "PLAIN-344": 0.4777009208974601, "PLAIN-358": 0.0, "PLAIN-371": 0.23981246656813146, "PLAIN-383": 0.16958010263680806, "PLAIN-395": 0.0, "PLAIN-407": 0.6387878864795979, "PLAIN-418": 0.8670870086853021, "PLAIN-430": 0.6174886499090687, "PLAIN-441": 0.777747077360283, "PLAIN-457": 0.0, "PLAIN-468": 0.4318845862516251, "PLAIN-478": 0.0, "PLAIN-488": 0.9363792118010483, "PLAIN-499": 0.5174612499126158, "PLAIN-510": 0.0, "PLAIN-520": 0.0, "PLAIN-531": 0.5079937796926363, "PLAIN-541": 0.11706259313796354, "PLAIN-551": 0.7903864795495061, "PLAIN-561": 0.43735247915031, "PLAIN-571": 0.22471947398559278, "PLAIN-583": 0.22009176629808017, "PLAIN-593": 0.46927872602275644, "PLAIN-603": 0.0, "PLAIN-613": 0.11004588314904008, "PLAIN-623": 0.33013764944712026, "PLAIN-634": 0.424926013816671, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.510383374527714, "PLAIN-711": 0.7605591431973849, "PLAIN-721": 1.0, "PLAIN-731": 0.642186726668901, "PLAIN-741": 0.866947989864271, "PLAIN-751": 0.0, "PLAIN-761": 0.15130120674940672, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.644824486427155, "PLAIN-806": 0.857980942822373, "PLAIN-817": 0.43067655807339306, "PLAIN-827": 0.7411902110103873, "PLAIN-838": 0.7767471075223497, "PLAIN-850": 0.6936634693435663, "PLAIN-872": 0.0, "PLAIN-882": 0.4690000933206748, "PLAIN-892": 0.5326208815196265, "PLAIN-902": 0.6366824387328317, "PLAIN-913": 0.8643145546088337, "PLAIN-924": 0.30523488393970116, "PLAIN-934": 0.6311443278613471, "PLAIN-946": 0.22009176629808017, "PLAIN-956": 0.6500332220485792, "PLAIN-966": 0.0, "PLAIN-977": 0.46927872602275644, "PLAIN-987": 0.0, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.48993086854170786, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.07839826897867533, "PLAIN-1066": 0.0, "PLAIN-1088": 0.5531464700081437, "PLAIN-1098": 0.0, "PLAIN-1109": 0.6627919420317119, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.22009176629808017, "PLAIN-1151": 0.7385567757549499, "PLAIN-1161": 0.0, "PLAIN-1172": 0.22009176629808017, "PLAIN-1183": 0.1731866333482261, "PLAIN-1193": 0.0, "PLAIN-1203": 0.5036067026046991, "PLAIN-1214": 0.0, "PLAIN-1225": 0.6874708847784536, "PLAIN-1236": 0.3391602052736161, "PLAIN-1249": 0.0, "PLAIN-1262": 0.4537425745411855, "PLAIN-1275": 0.6274092458891772, "PLAIN-1288": 0.49828993591603105, "PLAIN-1299": 0.3589542101716347, "PLAIN-1309": 0.0, "PLAIN-1320": 0.6238470455881188, "PLAIN-1331": 0.0, "PLAIN-1342": 0.5861163849908828, "PLAIN-1353": 0.0, "PLAIN-1363": 0.22009176629808017, "PLAIN-1374": 0.07839826897867533, "PLAIN-1387": 0.7819835854675755, "PLAIN-1398": 1.0, "PLAIN-1409": 0.48068140169045714, "PLAIN-1419": 0.4690000933206748, "PLAIN-1429": 0.0, "PLAIN-1441": 0.642186726668901, "PLAIN-1453": 0.47674870152787435, "PLAIN-1463": 0.4576752747037683, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.33013764944712026, "PLAIN-1516": 0.0, "PLAIN-1527": 0.8482378089219645, "PLAIN-1537": 0.43295704183322337, "PLAIN-1547": 0.0, "PLAIN-1557": 0.2489083270225946, "PLAIN-1568": 1.0, "PLAIN-1579": 0.3391602052736161, "PLAIN-1590": 0.08514311764162098, "PLAIN-1601": 0.800693766409882, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.8643145546088337, "PLAIN-1645": 0.30260241349881345, "PLAIN-1656": 0.0, "PLAIN-1667": 0.8603818544462509, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 0.22009176629808017, "PLAIN-1731": 0.7653606369886217, "PLAIN-1741": 1.0, "PLAIN-1752": 0.0, "PLAIN-1762": 0.4856486183364634, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 0.568003577662161, "PLAIN-1837": 1.0, "PLAIN-1847": 0.22009176629808017, "PLAIN-1857": 0.627507133061656, "PLAIN-1867": 0.27487633291429087, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.7917267193679839, "PLAIN-1919": 0.866947989864271, "PLAIN-1929": 0.1736666713479918, "PLAIN-1940": 0.3589542101716347, "PLAIN-1950": 0.3222722491219547, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.9363792118010483, "PLAIN-1995": 0.0, "PLAIN-2009": 0.31442295467135667, "PLAIN-2019": 0.6364391809889587, "PLAIN-2030": 0.0, "PLAIN-2040": 0.5637884576902256, "PLAIN-2051": 0.6901935063354876, "PLAIN-2061": 0.933745776545611, "PLAIN-2071": 0.5771493379489847, "PLAIN-2081": 1.0, "PLAIN-2092": 0.6489315753318465, "PLAIN-2102": 0.8165901947515999, "PLAIN-2113": 0.0, "PLAIN-2124": 0.6906478832608419, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.644824486427155, "PLAIN-2167": 0.0, "PLAIN-2177": 0.46692474155501895, "PLAIN-2187": 0.23504554941448536, "PLAIN-2197": 0.8603818544462509, "PLAIN-2209": 0.0, "PLAIN-2220": 0.5526193270751085, "PLAIN-2230": 0.0, "PLAIN-2240": 0.5637884576902256, "PLAIN-2250": 0.0, "PLAIN-2261": 0.715550648768261, "PLAIN-2271": 0.0, "PLAIN-2281": 0.6309297535714575, "PLAIN-2291": 0.3391602052736161, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.44988737637276544, "PLAIN-2343": 0.0, "PLAIN-2354": 0.5531464700081437, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.31204907722178066, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.4832964623025934, "PLAIN-2440": 0.07336392209936005, "PLAIN-2450": 0.28952298823485745, "PLAIN-2460": 0.2873703978647362, "PLAIN-2470": 0.6923056707325975, "PLAIN-2480": 0.22025065699347168, "PLAIN-2490": 0.4104014996484018, "PLAIN-2500": 0.06760523069231983, "PLAIN-2510": 0.6320787427864673, "PLAIN-2520": 0.2085303067457527, "PLAIN-2530": 0.9633180389503199, "PLAIN-2540": 0.5907275541269412, "PLAIN-2550": 0.0, "PLAIN-2560": 0.866947989864271, "PLAIN-2570": 0.4205032672422578, "PLAIN-2580": 0.3941769500918012, "PLAIN-2590": 0.1581173338315241, "PLAIN-2600": 0.6542721934360559, "PLAIN-2610": 0.3168370282761786, "PLAIN-2620": 0.5549841905774386, "PLAIN-2630": 0.7222954974312066, "PLAIN-2640": 0.5469224065306436, "PLAIN-2650": 0.5352543167750639, "PLAIN-2660": 0.31954962756492084, "PLAIN-2670": 0.37456050428247883, "PLAIN-2680": 0.43478330062838466, "PLAIN-2690": 0.5760823635418191, "PLAIN-2700": 0.2957940082374155, "PLAIN-2710": 0.6665181205254729, "PLAIN-2720": 0.2489083270225946, "PLAIN-2730": 0.6338426833522841, "PLAIN-2740": 0.6854074373766073, "PLAIN-2750": 0.5946665900227162, "PLAIN-2760": 0.2469526638494753, "PLAIN-2770": 0.40880314014762503, "PLAIN-2780": 0.4769783244417837, "PLAIN-2790": 0.8217429957880231, "PLAIN-2800": 0.15472892909388752, "PLAIN-2810": 0.054784966022752124, "PLAIN-2820": 0.15619484341784115, "PLAIN-2830": 0.6387878864795979, "PLAIN-2840": 0.0, "PLAIN-2850": 0.24014860122439108, "PLAIN-2860": 1.0, "PLAIN-2870": 0.08442111803551768, "PLAIN-2880": 0.0, "PLAIN-2890": 0.15537907391531905, "PLAIN-2900": 0.09382844813642036, "PLAIN-2910": 0.3239134396887189, "PLAIN-2920": 0.0, "PLAIN-2930": 0.3607790370815594, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.3273949503887395, "PLAIN-2970": 0.38062652754129855, "PLAIN-2981": 0.43122038713502525, "PLAIN-2991": 0.38685280723454163, "PLAIN-3001": 0.38685280723454163, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.7653606369886217, "PLAIN-3063": 0.0, "PLAIN-3074": 1.0, "PLAIN-3085": 0.5012658353418871, "PLAIN-3097": 0.4770382338730848, "PLAIN-3116": 0.0, "PLAIN-3131": 0.4599311598225878, "PLAIN-3141": 0.2996484034259831, "PLAIN-3151": 0.7605170401458373, "PLAIN-3161": 0.4906128388857768, "PLAIN-3171": 0.06978419164318803, "PLAIN-3181": 0.07839826897867533, "PLAIN-3191": 0.35185668051157637, "PLAIN-3201": 0.37212364421416444, "PLAIN-3211": 0.0, "PLAIN-3221": 0.0, "PLAIN-3231": 0.0, "PLAIN-3241": 0.43056322872474745, "PLAIN-3251": 0.2155089134289123, "PLAIN-3261": 0.14982420171299154, "PLAIN-3271": 0.0, "PLAIN-3281": 0.20436662668128655, "PLAIN-3292": 0.501703228798527, "PLAIN-3302": 0.14747372860084756, "PLAIN-3312": 0.3170759870637039, "PLAIN-3322": 0.06943122193677727, "PLAIN-3332": 0.8909491779135381, "PLAIN-3342": 0.7653606369886217, "PLAIN-3352": 0.0, "PLAIN-3362": 0.4634153650075699, "PLAIN-3372": 0.1803895185407797, "PLAIN-3382": 0.4484467219534551, "PLAIN-3392": 0.0, "PLAIN-3402": 0.0, "PLAIN-3412": 0.22461013156664808, "PLAIN-3422": 0.0, "PLAIN-3432": 0.2934556883974402, "PLAIN-3442": 0.07037133610231527, "PLAIN-3452": 0.5216665747946869, "PLAIN-3462": 0.6067216003536936, "PLAIN-3472": 0.08036018528064806}, "Recall@20": {"PLAIN-2": 0.4166666666666667, "PLAIN-12": 0.13333333333333333, "PLAIN-23": 0.06666666666666667, "PLAIN-33": 0.125, "PLAIN-44": 0.05172413793103448, "PLAIN-56": 0.2, "PLAIN-68": 0.07272727272727272, "PLAIN-78": 0.016129032258064516, "PLAIN-91": 0.15625, "PLAIN-102": 0.08333333333333333, "PLAIN-112": 0.08928571428571429, "PLAIN-123": 0.0196078431372549, "PLAIN-133": 0.027777777777777776, "PLAIN-143": 0.17647058823529413, "PLAIN-153": 0.23404255319148937, "PLAIN-165": 0.2222222222222222, "PLAIN-175": 0.10810810810810811, "PLAIN-186": 0.041666666666666664, "PLAIN-196": 0.014492753623188406, "PLAIN-207": 0.14545454545454545, "PLAIN-217": 0.16666666666666666, "PLAIN-227": 0.025, "PLAIN-238": 0.0, "PLAIN-248": 0.16129032258064516, "PLAIN-259": 0.21428571428571427, "PLAIN-270": 0.057692307692307696, "PLAIN-280": 0.25, "PLAIN-291": 0.3076923076923077, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.5, "PLAIN-344": 0.11475409836065574, "PLAIN-358": 0.3333333333333333, "PLAIN-371": 0.5, "PLAIN-383": 0.4, "PLAIN-395": 0.0, "PLAIN-407": 0.3333333333333333, "PLAIN-418": 0.6666666666666666, "PLAIN-430": 0.375, "PLAIN-441": 0.8, "PLAIN-457": 0.0, "PLAIN-468": 0.2, "PLAIN-478": 0.0, "PLAIN-488": 0.7333333333333333, "PLAIN-499": 0.11764705882352941, "PLAIN-510": 0.16666666666666666, "PLAIN-520": 0.0, "PLAIN-531": 0.03896103896103896, "PLAIN-541": 0.16666666666666666, "PLAIN-551": 1.0, "PLAIN-561": 0.12, "PLAIN-571": 0.42857142857142855, "PLAIN-583": 0.022727272727272728, "PLAIN-593": 0.3333333333333333, "PLAIN-603": 0.0, "PLAIN-613": 0.1, "PLAIN-623": 0.12195121951219512, "PLAIN-634": 0.14285714285714285, "PLAIN-645": 0.0, "PLAIN-660": 0.03368421052631579, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.375, "PLAIN-711": 0.21052631578947367, "PLAIN-721": 0.68, "PLAIN-731": 0.2037037037037037, "PLAIN-741": 0.47368421052631576, "PLAIN-751": 0.0, "PLAIN-761": 0.16666666666666666, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.5, "PLAIN-806": 0.13541666666666666, "PLAIN-817": 1.0, "PLAIN-827": 0.040983606557377046, "PLAIN-838": 0.6666666666666666, "PLAIN-850": 0.19298245614035087, "PLAIN-872": 0.0, "PLAIN-882": 0.2, "PLAIN-892": 0.06521739130434782, "PLAIN-902": 0.5, "PLAIN-913": 0.23529411764705882, "PLAIN-924": 0.0625, "PLAIN-934": 0.09900990099009901, "PLAIN-946": 0.03225806451612903, "PLAIN-956": 0.05825242718446602, "PLAIN-966": 0.0, "PLAIN-977": 0.3333333333333333, "PLAIN-987": 0.3333333333333333, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.1, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.010638297872340425, "PLAIN-1066": 0.0, "PLAIN-1088": 0.4, "PLAIN-1098": 0.0, "PLAIN-1109": 0.11224489795918367, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.0625, "PLAIN-1151": 0.06535947712418301, "PLAIN-1161": 0.0, "PLAIN-1172": 0.05263157894736842, "PLAIN-1183": 0.08, "PLAIN-1193": 0.05555555555555555, "PLAIN-1203": 0.3333333333333333, "PLAIN-1214": 0.058823529411764705, "PLAIN-1225": 0.3333333333333333, "PLAIN-1236": 0.2, "PLAIN-1249": 0.0, "PLAIN-1262": 0.3, "PLAIN-1275": 0.16363636363636364, "PLAIN-1288": 0.02040816326530612, "PLAIN-1299": 0.03278688524590164, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5, "PLAIN-1331": 0.0, "PLAIN-1342": 0.3125, "PLAIN-1353": 0.0, "PLAIN-1363": 0.09090909090909091, "PLAIN-1374": 0.07692307692307693, "PLAIN-1387": 0.5833333333333334, "PLAIN-1398": 0.1553398058252427, "PLAIN-1409": 0.02766798418972332, "PLAIN-1419": 0.06153846153846154, "PLAIN-1429": 0.0, "PLAIN-1441": 0.04838709677419355, "PLAIN-1453": 0.044897959183673466, "PLAIN-1463": 0.20833333333333334, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.13333333333333333, "PLAIN-1516": 0.0, "PLAIN-1527": 0.06930693069306931, "PLAIN-1537": 0.13333333333333333, "PLAIN-1547": 0.0, "PLAIN-1557": 0.07692307692307693, "PLAIN-1568": 1.0, "PLAIN-1579": 0.2, "PLAIN-1590": 0.05555555555555555, "PLAIN-1601": 0.0945945945945946, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.030162412993039442, "PLAIN-1645": 0.16666666666666666, "PLAIN-1656": 0.03571428571428571, "PLAIN-1667": 0.06918238993710692, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 0.9375, "PLAIN-1721": 0.03225806451612903, "PLAIN-1731": 0.6666666666666666, "PLAIN-1741": 0.04523809523809524, "PLAIN-1752": 1.0, "PLAIN-1762": 0.5, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 0.13793103448275862, "PLAIN-1817": 0.2222222222222222, "PLAIN-1837": 0.08673469387755102, "PLAIN-1847": 0.18181818181818182, "PLAIN-1857": 0.24242424242424243, "PLAIN-1867": 0.14285714285714285, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.023758099352051837, "PLAIN-1919": 0.3333333333333333, "PLAIN-1929": 0.3, "PLAIN-1940": 0.16666666666666666, "PLAIN-1950": 0.23076923076923078, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.28125, "PLAIN-1995": 0.09090909090909091, "PLAIN-2009": 0.375, "PLAIN-2019": 0.6666666666666666, "PLAIN-2030": 0.0, "PLAIN-2040": 0.05303030303030303, "PLAIN-2051": 0.030985915492957747, "PLAIN-2061": 0.04390243902439024, "PLAIN-2071": 0.15555555555555556, "PLAIN-2081": 1.0, "PLAIN-2092": 0.5, "PLAIN-2102": 0.030878859857482184, "PLAIN-2113": 0.0, "PLAIN-2124": 0.35294117647058826, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.5, "PLAIN-2167": 0.0, "PLAIN-2177": 0.20833333333333334, "PLAIN-2187": 0.1111111111111111, "PLAIN-2197": 0.1724137931034483, "PLAIN-2209": 0.0, "PLAIN-2220": 0.4444444444444444, "PLAIN-2230": 0.0, "PLAIN-2240": 0.36363636363636365, "PLAIN-2250": 0.0, "PLAIN-2261": 0.11666666666666667, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.4, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.09278350515463918, "PLAIN-2343": 0.0, "PLAIN-2354": 0.4, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.3684210526315789, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.13043478260869565, "PLAIN-2440": 0.047619047619047616, "PLAIN-2450": 0.07894736842105263, "PLAIN-2460": 0.0967741935483871, "PLAIN-2470": 0.1643835616438356, "PLAIN-2480": 0.16666666666666666, "PLAIN-2490": 0.13043478260869565, "PLAIN-2500": 0.42857142857142855, "PLAIN-2510": 0.21739130434782608, "PLAIN-2520": 0.0821917808219178, "PLAIN-2530": 0.20833333333333334, "PLAIN-2540": 0.38095238095238093, "PLAIN-2550": 0.04, "PLAIN-2560": 0.47619047619047616, "PLAIN-2570": 0.075, "PLAIN-2580": 0.08695652173913043, "PLAIN-2590": 0.07936507936507936, "PLAIN-2600": 0.4, "PLAIN-2610": 0.11475409836065574, "PLAIN-2620": 0.18181818181818182, "PLAIN-2630": 0.1875, "PLAIN-2640": 0.22916666666666666, "PLAIN-2650": 0.2413793103448276, "PLAIN-2660": 0.14814814814814814, "PLAIN-2670": 0.05714285714285714, "PLAIN-2680": 0.35714285714285715, "PLAIN-2690": 0.20930232558139536, "PLAIN-2700": 0.2, "PLAIN-2710": 0.23809523809523808, "PLAIN-2720": 0.05555555555555555, "PLAIN-2730": 0.20689655172413793, "PLAIN-2740": 0.12, "PLAIN-2750": 0.15254237288135594, "PLAIN-2760": 0.25, "PLAIN-2770": 0.21951219512195122, "PLAIN-2780": 0.3181818181818182, "PLAIN-2790": 0.15254237288135594, "PLAIN-2800": 0.08333333333333333, "PLAIN-2810": 0.125, "PLAIN-2820": 0.125, "PLAIN-2830": 0.3333333333333333, "PLAIN-2840": 0.0, "PLAIN-2850": 0.1111111111111111, "PLAIN-2860": 1.0, "PLAIN-2870": 0.2, "PLAIN-2880": 0.0, "PLAIN-2890": 0.034482758620689655, "PLAIN-2900": 0.08333333333333333, "PLAIN-2910": 0.06666666666666667, "PLAIN-2920": 0.0, "PLAIN-2930": 0.14285714285714285, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.5, "PLAIN-2970": 0.4444444444444444, "PLAIN-2981": 0.14285714285714285, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.6666666666666666, "PLAIN-3063": 0.25, "PLAIN-3074": 1.0, "PLAIN-3085": 1.0, "PLAIN-3097": 0.5, "PLAIN-3116": 0.0, "PLAIN-3131": 0.3333333333333333, "PLAIN-3141": 0.029411764705882353, "PLAIN-3151": 0.5, "PLAIN-3161": 0.13953488372093023, "PLAIN-3171": 0.029411764705882353, "PLAIN-3181": 0.037037037037037035, "PLAIN-3191": 0.1875, "PLAIN-3201": 0.09375, "PLAIN-3211": 0.0, "PLAIN-3221": 0.125, "PLAIN-3231": 0.25, "PLAIN-3241": 0.06896551724137931, "PLAIN-3251": 0.09523809523809523, "PLAIN-3261": 0.07692307692307693, "PLAIN-3271": 0.0, "PLAIN-3281": 0.038461538461538464, "PLAIN-3292": 0.7142857142857143, "PLAIN-3302": 0.08108108108108109, "PLAIN-3312": 0.5384615384615384, "PLAIN-3322": 0.02531645569620253, "PLAIN-3332": 0.6428571428571429, "PLAIN-3342": 0.6666666666666666, "PLAIN-3352": 0.02564102564102564, "PLAIN-3362": 0.18181818181818182, "PLAIN-3372": 0.07142857142857142, "PLAIN-3382": 0.5714285714285714, "PLAIN-3392": 0.13043478260869565, "PLAIN-3402": 0.07142857142857142, "PLAIN-3412": 0.061224489795918366, "PLAIN-3422": 0.0967741935483871, "PLAIN-3432": 0.09523809523809523, "PLAIN-3442": 0.034482758620689655, "PLAIN-3452": 0.29411764705882354, "PLAIN-3462": 0.1875, "PLAIN-3472": 0.058823529411764705}, "MRR@10": {"PLAIN-2": 1.0, "PLAIN-12": 0.3333333333333333, "PLAIN-23": 0.5, "PLAIN-33": 0.5, "PLAIN-44": 0.25, "PLAIN-56": 1.0, "PLAIN-68": 1.0, "PLAIN-78": 0.3333333333333333, "PLAIN-91": 1.0, "PLAIN-102": 1.0, "PLAIN-112": 1.0, "PLAIN-123": 0.5, "PLAIN-133": 0.3333333333333333, "PLAIN-143": 1.0, "PLAIN-153": 1.0, "PLAIN-165": 0.5, "PLAIN-175": 1.0, "PLAIN-186": 1.0, "PLAIN-196": 1.0, "PLAIN-207": 0.3333333333333333, "PLAIN-217": 0.125, "PLAIN-227": 0.3333333333333333, "PLAIN-238": 0.0, "PLAIN-248": 1.0, "PLAIN-259": 1.0, "PLAIN-270": 0.5, "PLAIN-280": 0.5, "PLAIN-291": 1.0, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 1.0, "PLAIN-344": 0.5, "PLAIN-358": 0.06666666666666667, "PLAIN-371": 0.5, "PLAIN-383": 0.3333333333333333, "PLAIN-395": 0.0, "PLAIN-407": 1.0, "PLAIN-418": 1.0, "PLAIN-430": 1.0, "PLAIN-441": 1.0, "PLAIN-457": 0.0, "PLAIN-468": 1.0, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 1.0, "PLAIN-510": 0.07692307692307693, "PLAIN-520": 0.0, "PLAIN-531": 0.3333333333333333, "PLAIN-541": 0.2, "PLAIN-551": 1.0, "PLAIN-561": 1.0, "PLAIN-571": 0.25, "PLAIN-583": 1.0, "PLAIN-593": 1.0, "PLAIN-603": 0.0, "PLAIN-613": 0.3333333333333333, "PLAIN-623": 1.0, "PLAIN-634": 1.0, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 1.0, "PLAIN-711": 1.0, "PLAIN-721": 1.0, "PLAIN-731": 1.0, "PLAIN-741": 1.0, "PLAIN-751": 0.0, "PLAIN-761": 0.3333333333333333, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 1.0, "PLAIN-806": 1.0, "PLAIN-817": 0.25, "PLAIN-827": 1.0, "PLAIN-838": 1.0, "PLAIN-850": 1.0, "PLAIN-872": 0.0, "PLAIN-882": 1.0, "PLAIN-892": 1.0, "PLAIN-902": 1.0, "PLAIN-913": 1.0, "PLAIN-924": 1.0, "PLAIN-934": 0.5, "PLAIN-946": 1.0, "PLAIN-956": 0.5, "PLAIN-966": 0.0, "PLAIN-977": 1.0, "PLAIN-987": 0.07692307692307693, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.16666666666666666, "PLAIN-1066": 0.0, "PLAIN-1088": 1.0, "PLAIN-1098": 0.0, "PLAIN-1109": 1.0, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 1.0, "PLAIN-1151": 1.0, "PLAIN-1161": 0.0, "PLAIN-1172": 1.0, "PLAIN-1183": 0.25, "PLAIN-1193": 0.09090909090909091, "PLAIN-1203": 1.0, "PLAIN-1214": 0.06666666666666667, "PLAIN-1225": 1.0, "PLAIN-1236": 1.0, "PLAIN-1249": 0.0, "PLAIN-1262": 1.0, "PLAIN-1275": 1.0, "PLAIN-1288": 1.0, "PLAIN-1299": 1.0, "PLAIN-1309": 0.0, "PLAIN-1320": 1.0, "PLAIN-1331": 0.0, "PLAIN-1342": 1.0, "PLAIN-1353": 0.0, "PLAIN-1363": 1.0, "PLAIN-1374": 0.16666666666666666, "PLAIN-1387": 1.0, "PLAIN-1398": 1.0, "PLAIN-1409": 0.5, "PLAIN-1419": 1.0, "PLAIN-1429": 0.0, "PLAIN-1441": 1.0, "PLAIN-1453": 0.5, "PLAIN-1463": 1.0, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 1.0, "PLAIN-1516": 0.0, "PLAIN-1527": 1.0, "PLAIN-1537": 0.5, "PLAIN-1547": 0.0, "PLAIN-1557": 0.5, "PLAIN-1568": 1.0, "PLAIN-1579": 1.0, "PLAIN-1590": 0.2, "PLAIN-1601": 1.0, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 1.0, "PLAIN-1645": 1.0, "PLAIN-1656": 0.06666666666666667, "PLAIN-1667": 1.0, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 1.0, "PLAIN-1731": 1.0, "PLAIN-1741": 1.0, "PLAIN-1752": 0.05, "PLAIN-1762": 1.0, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 1.0, "PLAIN-1837": 1.0, "PLAIN-1847": 1.0, "PLAIN-1857": 1.0, "PLAIN-1867": 1.0, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 1.0, "PLAIN-1919": 1.0, "PLAIN-1929": 0.3333333333333333, "PLAIN-1940": 1.0, "PLAIN-1950": 0.5, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 1.0, "PLAIN-1995": 0.06666666666666667, "PLAIN-2009": 0.3333333333333333, "PLAIN-2019": 1.0, "PLAIN-2030": 0.0, "PLAIN-2040": 1.0, "PLAIN-2051": 1.0, "PLAIN-2061": 1.0, "PLAIN-2071": 1.0, "PLAIN-2081": 1.0, "PLAIN-2092": 1.0, "PLAIN-2102": 1.0, "PLAIN-2113": 0.0, "PLAIN-2124": 1.0, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 1.0, "PLAIN-2167": 0.0, "PLAIN-2177": 0.5, "PLAIN-2187": 1.0, "PLAIN-2197": 1.0, "PLAIN-2209": 0.0, "PLAIN-2220": 1.0, "PLAIN-2230": 0.0, "PLAIN-2240": 1.0, "PLAIN-2250": 0.0, "PLAIN-2261": 1.0, "PLAIN-2271": 0.0, "PLAIN-2281": 0.5, "PLAIN-2291": 1.0, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 1.0, "PLAIN-2343": 0.0, "PLAIN-2354": 1.0, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.5, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 1.0, "PLAIN-2440": 0.14285714285714285, "PLAIN-2450": 1.0, "PLAIN-2460": 0.5, "PLAIN-2470": 1.0, "PLAIN-2480": 0.5, "PLAIN-2490": 1.0, "PLAIN-2500": 0.16666666666666666, "PLAIN-2510": 0.5, "PLAIN-2520": 0.5, "PLAIN-2530": 1.0, "PLAIN-2540": 1.0, "PLAIN-2550": 0.08333333333333333, "PLAIN-2560": 1.0, "PLAIN-2570": 1.0, "PLAIN-2580": 1.0, "PLAIN-2590": 0.3333333333333333, "PLAIN-2600": 1.0, "PLAIN-2610": 0.3333333333333333, "PLAIN-2620": 1.0, "PLAIN-2630": 1.0, "PLAIN-2640": 1.0, "PLAIN-2650": 1.0, "PLAIN-2660": 0.25, "PLAIN-2670": 1.0, "PLAIN-2680": 0.5, "PLAIN-2690": 1.0, "PLAIN-2700": 0.5, "PLAIN-2710": 1.0, "PLAIN-2720": 1.0, "PLAIN-2730": 1.0, "PLAIN-2740": 1.0, "PLAIN-2750": 1.0, "PLAIN-2760": 0.16666666666666666, "PLAIN-2770": 0.5, "PLAIN-2780": 1.0, "PLAIN-2790": 1.0, "PLAIN-2800": 0.25, "PLAIN-2810": 0.14285714285714285, "PLAIN-2820": 0.2, "PLAIN-2830": 1.0, "PLAIN-2840": 0.0, "PLAIN-2850": 0.5, "PLAIN-2860": 1.0, "PLAIN-2870": 0.14285714285714285, "PLAIN-2880": 0.0, "PLAIN-2890": 0.25, "PLAIN-2900": 0.14285714285714285, "PLAIN-2910": 1.0, "PLAIN-2920": 0.0, "PLAIN-2930": 1.0, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.25, "PLAIN-2970": 1.0, "PLAIN-2981": 1.0, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 1.0, "PLAIN-3063": 0.07692307692307693, "PLAIN-3074": 1.0, "PLAIN-3085": 0.25, "PLAIN-3097": 1.0, "PLAIN-3116": 0.0, "PLAIN-3131": 1.0, "PLAIN-3141": 1.0, "PLAIN-3151": 1.0, "PLAIN-3161": 1.0, "PLAIN-3171": 0.2, "PLAIN-3181": 0.16666666666666666, "PLAIN-3191": 1.0, "PLAIN-3201": 0.5, "PLAIN-3211": 0.0, "PLAIN-3221": 0.08333333333333333, "PLAIN-3231": 0.07142857142857142, "PLAIN-3241": 1.0, "PLAIN-3251": 0.25, "PLAIN-3261": 0.3333333333333333, "PLAIN-3271": 0.0, "PLAIN-3281": 0.5, "PLAIN-3292": 0.5, "PLAIN-3302": 0.25, "PLAIN-3312": 0.5, "PLAIN-3322": 0.125, "PLAIN-3332": 1.0, "PLAIN-3342": 1.0, "PLAIN-3352": 0.06666666666666667, "PLAIN-3362": 1.0, "PLAIN-3372": 0.3333333333333333, "PLAIN-3382": 0.5, "PLAIN-3392": 0.07692307692307693, "PLAIN-3402": 0.07692307692307693, "PLAIN-3412": 1.0, "PLAIN-3422": 0.08333333333333333, "PLAIN-3432": 1.0, "PLAIN-3442": 0.3333333333333333, "PLAIN-3452": 1.0, "PLAIN-3462": 0.5, "PLAIN-3472": 0.1111111111111111}}} diff --git a/evals/benchmark/results/nfcorpus/ir-w0.75.jsonl b/evals/benchmark/results/nfcorpus/ir-w0.75.jsonl new file mode 100644 index 000000000..a7c40a561 --- /dev/null +++ b/evals/benchmark/results/nfcorpus/ir-w0.75.jsonl @@ -0,0 +1 @@ +{"dataset": "nfcorpus", "run_tag": "w0.75", "aggregated": {"nDCG@10": 0.3743, "Recall@20": 0.2135, "MRR@10": 0.5887}, "per_query": {"nDCG@10": {"PLAIN-2": 0.7495470737433306, "PLAIN-12": 0.16950960215087577, "PLAIN-23": 0.46459600558095715, "PLAIN-33": 0.2640886107812416, "PLAIN-44": 0.23888170027858516, "PLAIN-56": 0.814153485068656, "PLAIN-68": 0.47459215156136747, "PLAIN-78": 0.06370866775185362, "PLAIN-91": 0.6947482816557203, "PLAIN-102": 0.21594229312581256, "PLAIN-112": 0.5903626645556106, "PLAIN-123": 0.08039138809008574, "PLAIN-133": 0.13347537859706934, "PLAIN-143": 0.6917350424665718, "PLAIN-153": 1.0, "PLAIN-165": 0.45365777747025504, "PLAIN-175": 0.42027556825478546, "PLAIN-186": 0.22009176629808017, "PLAIN-196": 0.22009176629808017, "PLAIN-207": 0.4230293752959408, "PLAIN-217": 0.13305201013572898, "PLAIN-227": 0.11004588314904008, "PLAIN-238": 0.0, "PLAIN-248": 0.38113435412202, "PLAIN-259": 0.3589542101716347, "PLAIN-270": 0.21222636597291458, "PLAIN-280": 0.4131279133289227, "PLAIN-291": 0.43231813227099475, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.6131471927654584, "PLAIN-344": 0.4602250224698888, "PLAIN-358": 0.0, "PLAIN-371": 0.23981246656813146, "PLAIN-383": 0.16958010263680806, "PLAIN-395": 0.0, "PLAIN-407": 0.6387878864795979, "PLAIN-418": 0.8670870086853021, "PLAIN-430": 0.6174886499090687, "PLAIN-441": 0.777747077360283, "PLAIN-457": 0.0, "PLAIN-468": 0.4318845862516251, "PLAIN-478": 0.0, "PLAIN-488": 0.9363792118010483, "PLAIN-499": 0.5107164012496701, "PLAIN-510": 0.0, "PLAIN-520": 0.0, "PLAIN-531": 0.5368103404171507, "PLAIN-541": 0.11706259313796354, "PLAIN-551": 0.7903864795495061, "PLAIN-561": 0.43735247915031, "PLAIN-571": 0.22471947398559278, "PLAIN-583": 0.22009176629808017, "PLAIN-593": 0.46927872602275644, "PLAIN-603": 0.0, "PLAIN-613": 0.08514311764162098, "PLAIN-623": 0.33013764944712026, "PLAIN-634": 0.41528076708874123, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.5214682725103094, "PLAIN-711": 0.7605591431973849, "PLAIN-721": 1.0, "PLAIN-731": 0.7058075148678526, "PLAIN-741": 0.866947989864271, "PLAIN-751": 0.0, "PLAIN-761": 0.15130120674940672, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.644824486427155, "PLAIN-806": 0.857980942822373, "PLAIN-817": 0.43067655807339306, "PLAIN-827": 0.7564477297898766, "PLAIN-838": 0.7767471075223497, "PLAIN-850": 0.6936634693435663, "PLAIN-872": 0.0, "PLAIN-882": 0.4690000933206748, "PLAIN-892": 0.5326208815196265, "PLAIN-902": 0.6366824387328317, "PLAIN-913": 0.8643145546088337, "PLAIN-924": 0.30523488393970116, "PLAIN-934": 0.7289115290270202, "PLAIN-946": 0.22009176629808017, "PLAIN-956": 0.6468562235661909, "PLAIN-966": 0.0, "PLAIN-977": 0.46927872602275644, "PLAIN-987": 0.0, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5063209639325833, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.07839826897867533, "PLAIN-1066": 0.0, "PLAIN-1088": 0.5531464700081437, "PLAIN-1098": 0.0, "PLAIN-1109": 0.6555476069276601, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.22009176629808017, "PLAIN-1151": 0.7289115290270202, "PLAIN-1161": 0.0, "PLAIN-1172": 0.22009176629808017, "PLAIN-1183": 0.1731866333482261, "PLAIN-1193": 0.0, "PLAIN-1203": 0.5036067026046991, "PLAIN-1214": 0.0, "PLAIN-1225": 0.6874708847784536, "PLAIN-1236": 0.3391602052736161, "PLAIN-1249": 0.0, "PLAIN-1262": 0.4537425745411855, "PLAIN-1275": 0.6274092458891772, "PLAIN-1288": 0.49828993591603105, "PLAIN-1299": 0.3589542101716347, "PLAIN-1309": 0.0, "PLAIN-1320": 0.6238470455881188, "PLAIN-1331": 0.0, "PLAIN-1342": 0.5861163849908828, "PLAIN-1353": 0.0, "PLAIN-1363": 0.22009176629808017, "PLAIN-1374": 0.07839826897867533, "PLAIN-1387": 0.7819835854675755, "PLAIN-1398": 1.0, "PLAIN-1409": 0.48068140169045714, "PLAIN-1419": 0.4690000933206748, "PLAIN-1429": 0.0, "PLAIN-1441": 0.627507133061656, "PLAIN-1453": 0.47674870152787435, "PLAIN-1463": 0.46270962158308354, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.33013764944712026, "PLAIN-1516": 0.0, "PLAIN-1527": 0.8482378089219645, "PLAIN-1537": 0.42331179510529354, "PLAIN-1547": 0.0, "PLAIN-1557": 0.2489083270225946, "PLAIN-1568": 1.0, "PLAIN-1579": 0.3391602052736161, "PLAIN-1590": 0.09478836436955078, "PLAIN-1601": 0.800693766409882, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.8643145546088337, "PLAIN-1645": 0.30260241349881345, "PLAIN-1656": 0.0, "PLAIN-1667": 0.8572048559638628, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 0.22009176629808017, "PLAIN-1731": 0.7653606369886217, "PLAIN-1741": 1.0, "PLAIN-1752": 0.0, "PLAIN-1762": 0.4856486183364634, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 0.5730379245414764, "PLAIN-1837": 1.0, "PLAIN-1847": 0.22009176629808017, "PLAIN-1857": 0.627507133061656, "PLAIN-1867": 0.27487633291429087, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.7917267193679839, "PLAIN-1919": 0.866947989864271, "PLAIN-1929": 0.1736666713479918, "PLAIN-1940": 0.3589542101716347, "PLAIN-1950": 0.3222722491219547, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.9363792118010483, "PLAIN-1995": 0.0, "PLAIN-2009": 0.31442295467135667, "PLAIN-2019": 0.6364391809889587, "PLAIN-2030": 0.0, "PLAIN-2040": 0.5637884576902256, "PLAIN-2051": 0.6870165078530993, "PLAIN-2061": 0.933745776545611, "PLAIN-2071": 0.5797827732044221, "PLAIN-2081": 1.0, "PLAIN-2092": 0.6489315753318465, "PLAIN-2102": 0.8318477135310893, "PLAIN-2113": 0.0, "PLAIN-2124": 0.6906478832608419, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.644824486427155, "PLAIN-2167": 0.0, "PLAIN-2177": 0.45166722277552973, "PLAIN-2187": 0.23504554941448536, "PLAIN-2197": 0.7967610662472993, "PLAIN-2209": 0.0, "PLAIN-2220": 0.5526193270751085, "PLAIN-2230": 0.0, "PLAIN-2240": 0.5637884576902256, "PLAIN-2250": 0.0, "PLAIN-2261": 0.715550648768261, "PLAIN-2271": 0.0, "PLAIN-2281": 0.6309297535714575, "PLAIN-2291": 0.3391602052736161, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.44988737637276544, "PLAIN-2343": 0.0, "PLAIN-2354": 0.5531464700081437, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.31204907722178066, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.4832964623025934, "PLAIN-2440": 0.07336392209936005, "PLAIN-2450": 0.28952298823485745, "PLAIN-2460": 0.2873703978647362, "PLAIN-2470": 0.6923056707325975, "PLAIN-2480": 0.214167553296389, "PLAIN-2490": 0.4104014996484018, "PLAIN-2500": 0.06760523069231983, "PLAIN-2510": 0.7093753650484099, "PLAIN-2520": 0.2085303067457527, "PLAIN-2530": 0.9633180389503199, "PLAIN-2540": 0.5907275541269412, "PLAIN-2550": 0.0, "PLAIN-2560": 0.800693766409882, "PLAIN-2570": 0.4205032672422578, "PLAIN-2580": 0.3941769500918012, "PLAIN-2590": 0.1581173338315241, "PLAIN-2600": 0.6604400652351458, "PLAIN-2610": 0.3052695562492681, "PLAIN-2620": 0.5549841905774386, "PLAIN-2630": 0.7183627972686238, "PLAIN-2640": 0.5567466860572858, "PLAIN-2650": 0.5384313152574521, "PLAIN-2660": 0.31954962756492084, "PLAIN-2670": 0.37456050428247883, "PLAIN-2680": 0.43478330062838466, "PLAIN-2690": 0.5760823635418191, "PLAIN-2700": 0.2957940082374155, "PLAIN-2710": 0.6665181205254729, "PLAIN-2720": 0.2489083270225946, "PLAIN-2730": 0.5469722089928789, "PLAIN-2740": 0.6854074373766073, "PLAIN-2750": 0.5946665900227162, "PLAIN-2760": 0.2469526638494753, "PLAIN-2770": 0.4257317410032713, "PLAIN-2780": 0.4769783244417837, "PLAIN-2790": 0.8251154201194959, "PLAIN-2800": 0.15472892909388752, "PLAIN-2810": 0.05184819766823351, "PLAIN-2820": 0.15619484341784115, "PLAIN-2830": 0.4030302838010049, "PLAIN-2840": 0.0, "PLAIN-2850": 0.24014860122439108, "PLAIN-2860": 1.0, "PLAIN-2870": 0.08442111803551768, "PLAIN-2880": 0.0, "PLAIN-2890": 0.15537907391531905, "PLAIN-2900": 0.09382844813642036, "PLAIN-2910": 0.3239134396887189, "PLAIN-2920": 0.0, "PLAIN-2930": 0.3607790370815594, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.3273949503887395, "PLAIN-2970": 0.38062652754129855, "PLAIN-2981": 0.43122038713502525, "PLAIN-2991": 0.38685280723454163, "PLAIN-3001": 0.38685280723454163, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.7653606369886217, "PLAIN-3063": 0.0, "PLAIN-3074": 1.0, "PLAIN-3085": 0.5012658353418871, "PLAIN-3097": 0.4770382338730848, "PLAIN-3116": 0.0, "PLAIN-3131": 0.4599311598225878, "PLAIN-3141": 0.2996484034259831, "PLAIN-3151": 0.707059297977083, "PLAIN-3161": 0.4906128388857768, "PLAIN-3171": 0.06978419164318803, "PLAIN-3181": 0.07839826897867533, "PLAIN-3191": 0.35185668051157637, "PLAIN-3201": 0.37212364421416444, "PLAIN-3211": 0.0, "PLAIN-3221": 0.0, "PLAIN-3231": 0.0, "PLAIN-3241": 0.43056322872474745, "PLAIN-3251": 0.2155089134289123, "PLAIN-3261": 0.14982420171299154, "PLAIN-3271": 0.0, "PLAIN-3281": 0.20436662668128655, "PLAIN-3292": 0.4689564279260148, "PLAIN-3302": 0.1599789509135779, "PLAIN-3312": 0.3170759870637039, "PLAIN-3322": 0.06943122193677727, "PLAIN-3332": 0.8909491779135381, "PLAIN-3342": 0.7653606369886217, "PLAIN-3352": 0.0, "PLAIN-3362": 0.4634153650075699, "PLAIN-3372": 0.15537907391531905, "PLAIN-3382": 0.4596741192162093, "PLAIN-3392": 0.0, "PLAIN-3402": 0.0, "PLAIN-3412": 0.23170768250378496, "PLAIN-3422": 0.0, "PLAIN-3432": 0.2934556883974402, "PLAIN-3442": 0.07037133610231527, "PLAIN-3452": 0.5325407313108637, "PLAIN-3462": 0.6067216003536936, "PLAIN-3472": 0.08036018528064806}, "Recall@20": {"PLAIN-2": 0.4166666666666667, "PLAIN-12": 0.13333333333333333, "PLAIN-23": 0.06666666666666667, "PLAIN-33": 0.125, "PLAIN-44": 0.05172413793103448, "PLAIN-56": 0.2, "PLAIN-68": 0.07272727272727272, "PLAIN-78": 0.016129032258064516, "PLAIN-91": 0.15625, "PLAIN-102": 0.08333333333333333, "PLAIN-112": 0.08928571428571429, "PLAIN-123": 0.0196078431372549, "PLAIN-133": 0.027777777777777776, "PLAIN-143": 0.17647058823529413, "PLAIN-153": 0.23404255319148937, "PLAIN-165": 0.2222222222222222, "PLAIN-175": 0.10810810810810811, "PLAIN-186": 0.041666666666666664, "PLAIN-196": 0.014492753623188406, "PLAIN-207": 0.14545454545454545, "PLAIN-217": 0.16666666666666666, "PLAIN-227": 0.025, "PLAIN-238": 0.0, "PLAIN-248": 0.16129032258064516, "PLAIN-259": 0.21428571428571427, "PLAIN-270": 0.057692307692307696, "PLAIN-280": 0.25, "PLAIN-291": 0.3076923076923077, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.5, "PLAIN-344": 0.11475409836065574, "PLAIN-358": 0.3333333333333333, "PLAIN-371": 0.5, "PLAIN-383": 0.4, "PLAIN-395": 0.0, "PLAIN-407": 0.3333333333333333, "PLAIN-418": 0.6666666666666666, "PLAIN-430": 0.375, "PLAIN-441": 0.8, "PLAIN-457": 0.0, "PLAIN-468": 0.2, "PLAIN-478": 0.0, "PLAIN-488": 0.7333333333333333, "PLAIN-499": 0.11764705882352941, "PLAIN-510": 0.16666666666666666, "PLAIN-520": 0.0, "PLAIN-531": 0.03896103896103896, "PLAIN-541": 0.16666666666666666, "PLAIN-551": 1.0, "PLAIN-561": 0.12, "PLAIN-571": 0.42857142857142855, "PLAIN-583": 0.022727272727272728, "PLAIN-593": 0.3333333333333333, "PLAIN-603": 0.0, "PLAIN-613": 0.1, "PLAIN-623": 0.12195121951219512, "PLAIN-634": 0.14285714285714285, "PLAIN-645": 0.0, "PLAIN-660": 0.03368421052631579, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.375, "PLAIN-711": 0.21052631578947367, "PLAIN-721": 0.68, "PLAIN-731": 0.2037037037037037, "PLAIN-741": 0.47368421052631576, "PLAIN-751": 0.0, "PLAIN-761": 0.16666666666666666, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.5, "PLAIN-806": 0.13541666666666666, "PLAIN-817": 1.0, "PLAIN-827": 0.040983606557377046, "PLAIN-838": 0.6666666666666666, "PLAIN-850": 0.19298245614035087, "PLAIN-872": 0.0, "PLAIN-882": 0.2, "PLAIN-892": 0.06521739130434782, "PLAIN-902": 0.5, "PLAIN-913": 0.23529411764705882, "PLAIN-924": 0.0625, "PLAIN-934": 0.09900990099009901, "PLAIN-946": 0.03225806451612903, "PLAIN-956": 0.05825242718446602, "PLAIN-966": 0.0, "PLAIN-977": 0.3333333333333333, "PLAIN-987": 0.3333333333333333, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.1, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.010638297872340425, "PLAIN-1066": 0.0, "PLAIN-1088": 0.4, "PLAIN-1098": 0.0, "PLAIN-1109": 0.11224489795918367, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.0625, "PLAIN-1151": 0.06535947712418301, "PLAIN-1161": 0.0, "PLAIN-1172": 0.05263157894736842, "PLAIN-1183": 0.08, "PLAIN-1193": 0.05555555555555555, "PLAIN-1203": 0.3333333333333333, "PLAIN-1214": 0.058823529411764705, "PLAIN-1225": 0.3333333333333333, "PLAIN-1236": 0.2, "PLAIN-1249": 0.0, "PLAIN-1262": 0.3, "PLAIN-1275": 0.16363636363636364, "PLAIN-1288": 0.02040816326530612, "PLAIN-1299": 0.03278688524590164, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5, "PLAIN-1331": 0.0, "PLAIN-1342": 0.3125, "PLAIN-1353": 0.0, "PLAIN-1363": 0.09090909090909091, "PLAIN-1374": 0.07692307692307693, "PLAIN-1387": 0.5833333333333334, "PLAIN-1398": 0.1553398058252427, "PLAIN-1409": 0.02766798418972332, "PLAIN-1419": 0.06153846153846154, "PLAIN-1429": 0.0, "PLAIN-1441": 0.04838709677419355, "PLAIN-1453": 0.044897959183673466, "PLAIN-1463": 0.20833333333333334, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.13333333333333333, "PLAIN-1516": 0.0, "PLAIN-1527": 0.06930693069306931, "PLAIN-1537": 0.13333333333333333, "PLAIN-1547": 0.0, "PLAIN-1557": 0.07692307692307693, "PLAIN-1568": 1.0, "PLAIN-1579": 0.2, "PLAIN-1590": 0.05555555555555555, "PLAIN-1601": 0.0945945945945946, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.030162412993039442, "PLAIN-1645": 0.16666666666666666, "PLAIN-1656": 0.03571428571428571, "PLAIN-1667": 0.06918238993710692, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 0.9375, "PLAIN-1721": 0.03225806451612903, "PLAIN-1731": 0.6666666666666666, "PLAIN-1741": 0.04523809523809524, "PLAIN-1752": 1.0, "PLAIN-1762": 0.5, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 0.13793103448275862, "PLAIN-1817": 0.2222222222222222, "PLAIN-1837": 0.08673469387755102, "PLAIN-1847": 0.18181818181818182, "PLAIN-1857": 0.24242424242424243, "PLAIN-1867": 0.14285714285714285, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.023758099352051837, "PLAIN-1919": 0.3333333333333333, "PLAIN-1929": 0.3, "PLAIN-1940": 0.16666666666666666, "PLAIN-1950": 0.23076923076923078, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.28125, "PLAIN-1995": 0.09090909090909091, "PLAIN-2009": 0.375, "PLAIN-2019": 0.6666666666666666, "PLAIN-2030": 0.0, "PLAIN-2040": 0.05303030303030303, "PLAIN-2051": 0.030985915492957747, "PLAIN-2061": 0.04390243902439024, "PLAIN-2071": 0.15555555555555556, "PLAIN-2081": 1.0, "PLAIN-2092": 0.5, "PLAIN-2102": 0.030878859857482184, "PLAIN-2113": 0.0, "PLAIN-2124": 0.35294117647058826, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 0.5, "PLAIN-2167": 0.0, "PLAIN-2177": 0.20833333333333334, "PLAIN-2187": 0.1111111111111111, "PLAIN-2197": 0.1724137931034483, "PLAIN-2209": 0.0, "PLAIN-2220": 0.4444444444444444, "PLAIN-2230": 0.0, "PLAIN-2240": 0.36363636363636365, "PLAIN-2250": 0.0, "PLAIN-2261": 0.11666666666666667, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.4, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.09278350515463918, "PLAIN-2343": 0.0, "PLAIN-2354": 0.4, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.3684210526315789, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.13043478260869565, "PLAIN-2440": 0.047619047619047616, "PLAIN-2450": 0.07894736842105263, "PLAIN-2460": 0.0967741935483871, "PLAIN-2470": 0.1643835616438356, "PLAIN-2480": 0.16666666666666666, "PLAIN-2490": 0.13043478260869565, "PLAIN-2500": 0.42857142857142855, "PLAIN-2510": 0.21739130434782608, "PLAIN-2520": 0.0821917808219178, "PLAIN-2530": 0.20833333333333334, "PLAIN-2540": 0.38095238095238093, "PLAIN-2550": 0.04, "PLAIN-2560": 0.47619047619047616, "PLAIN-2570": 0.075, "PLAIN-2580": 0.08695652173913043, "PLAIN-2590": 0.07936507936507936, "PLAIN-2600": 0.4, "PLAIN-2610": 0.11475409836065574, "PLAIN-2620": 0.18181818181818182, "PLAIN-2630": 0.1875, "PLAIN-2640": 0.22916666666666666, "PLAIN-2650": 0.2413793103448276, "PLAIN-2660": 0.14814814814814814, "PLAIN-2670": 0.05714285714285714, "PLAIN-2680": 0.35714285714285715, "PLAIN-2690": 0.20930232558139536, "PLAIN-2700": 0.2, "PLAIN-2710": 0.23809523809523808, "PLAIN-2720": 0.05555555555555555, "PLAIN-2730": 0.20689655172413793, "PLAIN-2740": 0.12, "PLAIN-2750": 0.15254237288135594, "PLAIN-2760": 0.25, "PLAIN-2770": 0.21951219512195122, "PLAIN-2780": 0.3181818181818182, "PLAIN-2790": 0.15254237288135594, "PLAIN-2800": 0.08333333333333333, "PLAIN-2810": 0.125, "PLAIN-2820": 0.125, "PLAIN-2830": 0.3333333333333333, "PLAIN-2840": 0.0, "PLAIN-2850": 0.1111111111111111, "PLAIN-2860": 1.0, "PLAIN-2870": 0.2, "PLAIN-2880": 0.0, "PLAIN-2890": 0.034482758620689655, "PLAIN-2900": 0.08333333333333333, "PLAIN-2910": 0.06666666666666667, "PLAIN-2920": 0.0, "PLAIN-2930": 0.14285714285714285, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.5, "PLAIN-2970": 0.4444444444444444, "PLAIN-2981": 0.14285714285714285, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.6666666666666666, "PLAIN-3063": 0.25, "PLAIN-3074": 1.0, "PLAIN-3085": 1.0, "PLAIN-3097": 0.5, "PLAIN-3116": 0.0, "PLAIN-3131": 0.3333333333333333, "PLAIN-3141": 0.029411764705882353, "PLAIN-3151": 0.5, "PLAIN-3161": 0.13953488372093023, "PLAIN-3171": 0.029411764705882353, "PLAIN-3181": 0.037037037037037035, "PLAIN-3191": 0.1875, "PLAIN-3201": 0.09375, "PLAIN-3211": 0.0, "PLAIN-3221": 0.125, "PLAIN-3231": 0.25, "PLAIN-3241": 0.06896551724137931, "PLAIN-3251": 0.09523809523809523, "PLAIN-3261": 0.07692307692307693, "PLAIN-3271": 0.0, "PLAIN-3281": 0.038461538461538464, "PLAIN-3292": 0.7142857142857143, "PLAIN-3302": 0.08108108108108109, "PLAIN-3312": 0.5384615384615384, "PLAIN-3322": 0.02531645569620253, "PLAIN-3332": 0.6428571428571429, "PLAIN-3342": 0.6666666666666666, "PLAIN-3352": 0.02564102564102564, "PLAIN-3362": 0.18181818181818182, "PLAIN-3372": 0.07142857142857142, "PLAIN-3382": 0.5714285714285714, "PLAIN-3392": 0.13043478260869565, "PLAIN-3402": 0.07142857142857142, "PLAIN-3412": 0.061224489795918366, "PLAIN-3422": 0.0967741935483871, "PLAIN-3432": 0.09523809523809523, "PLAIN-3442": 0.034482758620689655, "PLAIN-3452": 0.29411764705882354, "PLAIN-3462": 0.1875, "PLAIN-3472": 0.058823529411764705}, "MRR@10": {"PLAIN-2": 1.0, "PLAIN-12": 0.3333333333333333, "PLAIN-23": 0.5, "PLAIN-33": 0.5, "PLAIN-44": 0.25, "PLAIN-56": 1.0, "PLAIN-68": 1.0, "PLAIN-78": 0.3333333333333333, "PLAIN-91": 1.0, "PLAIN-102": 0.3333333333333333, "PLAIN-112": 1.0, "PLAIN-123": 0.5, "PLAIN-133": 0.3333333333333333, "PLAIN-143": 1.0, "PLAIN-153": 1.0, "PLAIN-165": 0.5, "PLAIN-175": 1.0, "PLAIN-186": 1.0, "PLAIN-196": 1.0, "PLAIN-207": 0.3333333333333333, "PLAIN-217": 0.125, "PLAIN-227": 0.3333333333333333, "PLAIN-238": 0.0, "PLAIN-248": 1.0, "PLAIN-259": 1.0, "PLAIN-270": 0.5, "PLAIN-280": 0.5, "PLAIN-291": 1.0, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 1.0, "PLAIN-344": 0.5, "PLAIN-358": 0.06666666666666667, "PLAIN-371": 0.5, "PLAIN-383": 0.3333333333333333, "PLAIN-395": 0.0, "PLAIN-407": 1.0, "PLAIN-418": 1.0, "PLAIN-430": 1.0, "PLAIN-441": 1.0, "PLAIN-457": 0.0, "PLAIN-468": 1.0, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 1.0, "PLAIN-510": 0.07692307692307693, "PLAIN-520": 0.0, "PLAIN-531": 0.5, "PLAIN-541": 0.2, "PLAIN-551": 1.0, "PLAIN-561": 1.0, "PLAIN-571": 0.25, "PLAIN-583": 1.0, "PLAIN-593": 1.0, "PLAIN-603": 0.0, "PLAIN-613": 0.2, "PLAIN-623": 1.0, "PLAIN-634": 1.0, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 1.0, "PLAIN-711": 1.0, "PLAIN-721": 1.0, "PLAIN-731": 1.0, "PLAIN-741": 1.0, "PLAIN-751": 0.0, "PLAIN-761": 0.3333333333333333, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 1.0, "PLAIN-806": 1.0, "PLAIN-817": 0.25, "PLAIN-827": 1.0, "PLAIN-838": 1.0, "PLAIN-850": 1.0, "PLAIN-872": 0.0, "PLAIN-882": 1.0, "PLAIN-892": 1.0, "PLAIN-902": 1.0, "PLAIN-913": 1.0, "PLAIN-924": 1.0, "PLAIN-934": 1.0, "PLAIN-946": 1.0, "PLAIN-956": 0.5, "PLAIN-966": 0.0, "PLAIN-977": 1.0, "PLAIN-987": 0.07692307692307693, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.16666666666666666, "PLAIN-1066": 0.0, "PLAIN-1088": 1.0, "PLAIN-1098": 0.0, "PLAIN-1109": 1.0, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 1.0, "PLAIN-1151": 1.0, "PLAIN-1161": 0.0, "PLAIN-1172": 1.0, "PLAIN-1183": 0.25, "PLAIN-1193": 0.09090909090909091, "PLAIN-1203": 1.0, "PLAIN-1214": 0.06666666666666667, "PLAIN-1225": 1.0, "PLAIN-1236": 1.0, "PLAIN-1249": 0.0, "PLAIN-1262": 1.0, "PLAIN-1275": 1.0, "PLAIN-1288": 1.0, "PLAIN-1299": 1.0, "PLAIN-1309": 0.0, "PLAIN-1320": 1.0, "PLAIN-1331": 0.0, "PLAIN-1342": 1.0, "PLAIN-1353": 0.0, "PLAIN-1363": 1.0, "PLAIN-1374": 0.16666666666666666, "PLAIN-1387": 1.0, "PLAIN-1398": 1.0, "PLAIN-1409": 0.5, "PLAIN-1419": 1.0, "PLAIN-1429": 0.0, "PLAIN-1441": 1.0, "PLAIN-1453": 0.5, "PLAIN-1463": 1.0, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 1.0, "PLAIN-1516": 0.0, "PLAIN-1527": 1.0, "PLAIN-1537": 0.5, "PLAIN-1547": 0.0, "PLAIN-1557": 0.5, "PLAIN-1568": 1.0, "PLAIN-1579": 1.0, "PLAIN-1590": 0.25, "PLAIN-1601": 1.0, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 1.0, "PLAIN-1645": 1.0, "PLAIN-1656": 0.06666666666666667, "PLAIN-1667": 1.0, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 1.0, "PLAIN-1731": 1.0, "PLAIN-1741": 1.0, "PLAIN-1752": 0.05, "PLAIN-1762": 1.0, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 1.0, "PLAIN-1837": 1.0, "PLAIN-1847": 1.0, "PLAIN-1857": 1.0, "PLAIN-1867": 1.0, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 1.0, "PLAIN-1919": 1.0, "PLAIN-1929": 0.3333333333333333, "PLAIN-1940": 1.0, "PLAIN-1950": 0.5, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 1.0, "PLAIN-1995": 0.06666666666666667, "PLAIN-2009": 0.3333333333333333, "PLAIN-2019": 1.0, "PLAIN-2030": 0.0, "PLAIN-2040": 1.0, "PLAIN-2051": 1.0, "PLAIN-2061": 1.0, "PLAIN-2071": 1.0, "PLAIN-2081": 1.0, "PLAIN-2092": 1.0, "PLAIN-2102": 1.0, "PLAIN-2113": 0.0, "PLAIN-2124": 1.0, "PLAIN-2134": 0.0, "PLAIN-2145": 0.0, "PLAIN-2156": 1.0, "PLAIN-2167": 0.0, "PLAIN-2177": 0.5, "PLAIN-2187": 1.0, "PLAIN-2197": 1.0, "PLAIN-2209": 0.0, "PLAIN-2220": 1.0, "PLAIN-2230": 0.0, "PLAIN-2240": 1.0, "PLAIN-2250": 0.0, "PLAIN-2261": 1.0, "PLAIN-2271": 0.0, "PLAIN-2281": 0.5, "PLAIN-2291": 1.0, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 1.0, "PLAIN-2343": 0.0, "PLAIN-2354": 1.0, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.5, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 1.0, "PLAIN-2440": 0.14285714285714285, "PLAIN-2450": 1.0, "PLAIN-2460": 0.5, "PLAIN-2470": 1.0, "PLAIN-2480": 0.5, "PLAIN-2490": 1.0, "PLAIN-2500": 0.16666666666666666, "PLAIN-2510": 1.0, "PLAIN-2520": 0.5, "PLAIN-2530": 1.0, "PLAIN-2540": 1.0, "PLAIN-2550": 0.08333333333333333, "PLAIN-2560": 1.0, "PLAIN-2570": 1.0, "PLAIN-2580": 1.0, "PLAIN-2590": 0.3333333333333333, "PLAIN-2600": 1.0, "PLAIN-2610": 0.3333333333333333, "PLAIN-2620": 1.0, "PLAIN-2630": 1.0, "PLAIN-2640": 1.0, "PLAIN-2650": 1.0, "PLAIN-2660": 0.25, "PLAIN-2670": 1.0, "PLAIN-2680": 0.5, "PLAIN-2690": 1.0, "PLAIN-2700": 0.5, "PLAIN-2710": 1.0, "PLAIN-2720": 1.0, "PLAIN-2730": 0.5, "PLAIN-2740": 1.0, "PLAIN-2750": 1.0, "PLAIN-2760": 0.16666666666666666, "PLAIN-2770": 0.5, "PLAIN-2780": 1.0, "PLAIN-2790": 1.0, "PLAIN-2800": 0.25, "PLAIN-2810": 0.125, "PLAIN-2820": 0.2, "PLAIN-2830": 0.5, "PLAIN-2840": 0.0, "PLAIN-2850": 0.5, "PLAIN-2860": 1.0, "PLAIN-2870": 0.14285714285714285, "PLAIN-2880": 0.0, "PLAIN-2890": 0.25, "PLAIN-2900": 0.14285714285714285, "PLAIN-2910": 1.0, "PLAIN-2920": 0.0, "PLAIN-2930": 1.0, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.25, "PLAIN-2970": 1.0, "PLAIN-2981": 1.0, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 1.0, "PLAIN-3063": 0.07692307692307693, "PLAIN-3074": 1.0, "PLAIN-3085": 0.25, "PLAIN-3097": 1.0, "PLAIN-3116": 0.0, "PLAIN-3131": 1.0, "PLAIN-3141": 1.0, "PLAIN-3151": 1.0, "PLAIN-3161": 1.0, "PLAIN-3171": 0.2, "PLAIN-3181": 0.16666666666666666, "PLAIN-3191": 1.0, "PLAIN-3201": 0.5, "PLAIN-3211": 0.0, "PLAIN-3221": 0.08333333333333333, "PLAIN-3231": 0.07142857142857142, "PLAIN-3241": 1.0, "PLAIN-3251": 0.25, "PLAIN-3261": 0.3333333333333333, "PLAIN-3271": 0.0, "PLAIN-3281": 0.5, "PLAIN-3292": 0.3333333333333333, "PLAIN-3302": 0.3333333333333333, "PLAIN-3312": 0.5, "PLAIN-3322": 0.125, "PLAIN-3332": 1.0, "PLAIN-3342": 1.0, "PLAIN-3352": 0.06666666666666667, "PLAIN-3362": 1.0, "PLAIN-3372": 0.25, "PLAIN-3382": 0.5, "PLAIN-3392": 0.07692307692307693, "PLAIN-3402": 0.07692307692307693, "PLAIN-3412": 1.0, "PLAIN-3422": 0.08333333333333333, "PLAIN-3432": 1.0, "PLAIN-3442": 0.3333333333333333, "PLAIN-3452": 1.0, "PLAIN-3462": 0.5, "PLAIN-3472": 0.1111111111111111}}} diff --git a/evals/benchmark/results/nfcorpus/ir-w1.0.jsonl b/evals/benchmark/results/nfcorpus/ir-w1.0.jsonl new file mode 100644 index 000000000..160123903 --- /dev/null +++ b/evals/benchmark/results/nfcorpus/ir-w1.0.jsonl @@ -0,0 +1 @@ +{"dataset": "nfcorpus", "run_tag": "w1.0", "aggregated": {"nDCG@10": 0.3734, "Recall@20": 0.2159, "MRR@10": 0.5885}, "per_query": {"nDCG@10": {"PLAIN-2": 0.7826943468387285, "PLAIN-12": 0.16078413422405044, "PLAIN-23": 0.5295335114076275, "PLAIN-33": 0.18391188799974265, "PLAIN-44": 0.27140908923143764, "PLAIN-56": 0.7734697289723091, "PLAIN-68": 0.47459215156136747, "PLAIN-78": 0.1296831420200403, "PLAIN-91": 0.6593205835781908, "PLAIN-102": 0.21071055051685958, "PLAIN-112": 0.5089951523629167, "PLAIN-123": 0.08039138809008574, "PLAIN-133": 0.13347537859706934, "PLAIN-143": 0.5953395623481915, "PLAIN-153": 0.943093442760096, "PLAIN-165": 0.45365777747025504, "PLAIN-175": 0.42027556825478546, "PLAIN-186": 0.22009176629808017, "PLAIN-196": 0.5009732673492616, "PLAIN-207": 0.4269620754585236, "PLAIN-217": 0.06625422345438903, "PLAIN-227": 0.11004588314904008, "PLAIN-238": 0.0, "PLAIN-248": 0.42836504693009625, "PLAIN-259": 0.3589542101716347, "PLAIN-270": 0.2051166673279436, "PLAIN-280": 0.4131279133289227, "PLAIN-291": 0.42520843362602373, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.6131471927654584, "PLAIN-344": 0.4130185848872811, "PLAIN-358": 0.0, "PLAIN-371": 0.23981246656813146, "PLAIN-383": 0.16958010263680806, "PLAIN-395": 0.0, "PLAIN-407": 0.6387878864795979, "PLAIN-418": 0.8670870086853021, "PLAIN-430": 0.6174886499090687, "PLAIN-441": 0.9092187667269354, "PLAIN-457": 0.0, "PLAIN-468": 0.43929373490672136, "PLAIN-478": 0.0, "PLAIN-488": 0.933745776545611, "PLAIN-499": 0.5107164012496701, "PLAIN-510": 0.0, "PLAIN-520": 0.0, "PLAIN-531": 0.5181001594748443, "PLAIN-541": 0.11706259313796354, "PLAIN-551": 0.7903864795495061, "PLAIN-561": 0.43735247915031, "PLAIN-571": 0.22471947398559278, "PLAIN-583": 0.22009176629808017, "PLAIN-593": 0.46927872602275644, "PLAIN-603": 0.0, "PLAIN-613": 0.08514311764162098, "PLAIN-623": 0.39375843764607193, "PLAIN-634": 0.3340514446642156, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.0, "PLAIN-701": 0.5214682725103094, "PLAIN-711": 0.5473983622993502, "PLAIN-721": 1.0, "PLAIN-731": 0.7151857987862356, "PLAIN-741": 0.8643145546088337, "PLAIN-751": 0.0, "PLAIN-761": 0.15130120674940672, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.644824486427155, "PLAIN-806": 0.9216017310213247, "PLAIN-817": 0.3333333333333333, "PLAIN-827": 0.6870165078530993, "PLAIN-838": 0.8508957228074242, "PLAIN-850": 0.715550648768261, "PLAIN-872": 0.0, "PLAIN-882": 0.4690000933206748, "PLAIN-892": 0.6235744328990731, "PLAIN-902": 0.6366824387328317, "PLAIN-913": 0.866947989864271, "PLAIN-924": 0.30523488393970116, "PLAIN-934": 0.7218018303820491, "PLAIN-946": 0.30701473034246535, "PLAIN-956": 0.6468562235661909, "PLAIN-966": 0.0, "PLAIN-977": 0.46927872602275644, "PLAIN-987": 0.18154179253735267, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5113553108118986, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.0, "PLAIN-1066": 0.0, "PLAIN-1088": 0.5531464700081437, "PLAIN-1098": 0.0, "PLAIN-1109": 0.6651928536555899, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.22009176629808017, "PLAIN-1151": 0.6156886473955918, "PLAIN-1161": 0.0, "PLAIN-1172": 0.22009176629808017, "PLAIN-1183": 0.17993148201117176, "PLAIN-1193": 0.0, "PLAIN-1203": 0.42838543210841196, "PLAIN-1214": 0.0, "PLAIN-1225": 0.6874708847784536, "PLAIN-1236": 0.3391602052736161, "PLAIN-1249": 0.0, "PLAIN-1262": 0.4537425745411855, "PLAIN-1275": 0.6274092458891772, "PLAIN-1288": 0.5271064966405455, "PLAIN-1299": 0.3589542101716347, "PLAIN-1309": 0.0, "PLAIN-1320": 0.6238470455881188, "PLAIN-1331": 0.0, "PLAIN-1342": 0.5769706247040591, "PLAIN-1353": 0.0, "PLAIN-1363": 0.22009176629808017, "PLAIN-1374": 0.07839826897867533, "PLAIN-1387": 0.7183627972686238, "PLAIN-1398": 1.0, "PLAIN-1409": 0.39767223286316733, "PLAIN-1419": 0.5326208815196265, "PLAIN-1429": 0.0, "PLAIN-1441": 0.6207622843987103, "PLAIN-1453": 0.4288398090337664, "PLAIN-1463": 0.46945447024602915, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.2489083270225946, "PLAIN-1516": 0.0, "PLAIN-1527": 0.8521705090845474, "PLAIN-1537": 0.3529461582433962, "PLAIN-1547": 0.0, "PLAIN-1557": 0.31516255047698366, "PLAIN-1568": 1.0, "PLAIN-1579": 0.3391602052736161, "PLAIN-1590": 0.09478836436955078, "PLAIN-1601": 0.800693766409882, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.800693766409882, "PLAIN-1645": 0.30260241349881345, "PLAIN-1656": 0.08514311764162098, "PLAIN-1667": 0.8572048559638628, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 0.38113435412202, "PLAIN-1731": 0.7653606369886217, "PLAIN-1741": 1.0, "PLAIN-1752": 0.0, "PLAIN-1762": 0.41253177989255346, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 0.5797827732044221, "PLAIN-1837": 1.0, "PLAIN-1847": 0.2984900352767555, "PLAIN-1857": 0.6203974344166848, "PLAIN-1867": 0.27487633291429087, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.9216017310213247, "PLAIN-1919": 0.866947989864271, "PLAIN-1929": 0.11004588314904008, "PLAIN-1940": 0.3589542101716347, "PLAIN-1950": 0.31516255047698366, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 1.0, "PLAIN-1995": 0.0, "PLAIN-2009": 0.3086371802748747, "PLAIN-2019": 0.6364391809889587, "PLAIN-2030": 0.15130120674940672, "PLAIN-2040": 0.5637884576902256, "PLAIN-2051": 0.6870165078530993, "PLAIN-2061": 1.0, "PLAIN-2071": 0.5797827732044221, "PLAIN-2081": 1.0, "PLAIN-2092": 0.6489315753318465, "PLAIN-2102": 0.835780413693672, "PLAIN-2113": 0.0, "PLAIN-2124": 0.6906478832608419, "PLAIN-2134": 0.0, "PLAIN-2145": 0.10122860821230018, "PLAIN-2156": 0.644824486427155, "PLAIN-2167": 0.0, "PLAIN-2177": 0.44903378752009243, "PLAIN-2187": 0.33627415762678553, "PLAIN-2197": 0.7967610662472993, "PLAIN-2209": 0.0, "PLAIN-2220": 0.5526193270751085, "PLAIN-2230": 0.07945707943276742, "PLAIN-2240": 0.6332196796270029, "PLAIN-2250": 0.0, "PLAIN-2261": 0.7849818707050383, "PLAIN-2271": 0.0, "PLAIN-2281": 0.6309297535714575, "PLAIN-2291": 0.3391602052736161, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.3785988060390612, "PLAIN-2343": 0.0, "PLAIN-2354": 0.5087403079104241, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.2866919347890071, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.5087500511683787, "PLAIN-2440": 0.06943122193677727, "PLAIN-2450": 0.28952298823485745, "PLAIN-2460": 0.16842717545221192, "PLAIN-2470": 0.5581745575043326, "PLAIN-2480": 0.2652101672883918, "PLAIN-2490": 0.44491792865689783, "PLAIN-2500": 0.06326395904697463, "PLAIN-2510": 0.7093753650484099, "PLAIN-2520": 0.2085303067457527, "PLAIN-2530": 0.9301909272231254, "PLAIN-2540": 0.493671009737819, "PLAIN-2550": 0.0, "PLAIN-2560": 0.800693766409882, "PLAIN-2570": 0.33901920430175153, "PLAIN-2580": 0.3941769500918012, "PLAIN-2590": 0.0489513437202143, "PLAIN-2600": 0.6604400652351458, "PLAIN-2610": 0.41531543939830806, "PLAIN-2620": 0.5547124089639631, "PLAIN-2630": 0.7183627972686238, "PLAIN-2640": 0.5567466860572858, "PLAIN-2650": 0.5231737964779628, "PLAIN-2660": 0.32542886242602714, "PLAIN-2670": 0.37456050428247883, "PLAIN-2680": 0.4920033254983605, "PLAIN-2690": 0.5066511416050418, "PLAIN-2700": 0.2835856399974543, "PLAIN-2710": 0.6163982004016604, "PLAIN-2720": 0.2489083270225946, "PLAIN-2730": 0.47271918059164886, "PLAIN-2740": 0.6015234953647823, "PLAIN-2750": 0.5960747500504401, "PLAIN-2760": 0.21650681306156466, "PLAIN-2770": 0.3928761991697183, "PLAIN-2780": 0.40753108831444834, "PLAIN-2790": 0.8251154201194959, "PLAIN-2800": 0.15114358520260918, "PLAIN-2810": 0.05184819766823351, "PLAIN-2820": 0.15619484341784115, "PLAIN-2830": 0.4030302838010049, "PLAIN-2840": 0.0, "PLAIN-2850": 0.19031326377064928, "PLAIN-2860": 1.0, "PLAIN-2870": 0.07623986638854101, "PLAIN-2880": 0.0, "PLAIN-2890": 0.15537907391531905, "PLAIN-2900": 0.08879873949105598, "PLAIN-2910": 0.20436662668128655, "PLAIN-2920": 0.0, "PLAIN-2930": 0.3607790370815594, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.3273949503887395, "PLAIN-2970": 0.38062652754129855, "PLAIN-2981": 0.43122038713502525, "PLAIN-2991": 0.38685280723454163, "PLAIN-3001": 0.38685280723454163, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.7653606369886217, "PLAIN-3063": 0.0, "PLAIN-3074": 1.0, "PLAIN-3085": 0.48247555939075504, "PLAIN-3097": 0.4770382338730848, "PLAIN-3116": 0.0, "PLAIN-3131": 0.5760048611532679, "PLAIN-3141": 0.2996484034259831, "PLAIN-3151": 0.707059297977083, "PLAIN-3161": 0.4384685740392189, "PLAIN-3171": 0.194089844787399, "PLAIN-3181": 0.0, "PLAIN-3191": 0.3702840839003171, "PLAIN-3201": 0.37212364421416444, "PLAIN-3211": 0.0, "PLAIN-3221": 0.0, "PLAIN-3231": 0.0, "PLAIN-3241": 0.415081693065718, "PLAIN-3251": 0.2155089134289123, "PLAIN-3261": 0.14982420171299154, "PLAIN-3271": 0.0, "PLAIN-3281": 0.20436662668128655, "PLAIN-3292": 0.4689564279260148, "PLAIN-3302": 0.27058427781116956, "PLAIN-3312": 0.313127676263947, "PLAIN-3322": 0.0, "PLAIN-3332": 0.8334232396556904, "PLAIN-3342": 0.7653606369886217, "PLAIN-3352": 0.0, "PLAIN-3362": 0.5822656396514586, "PLAIN-3372": 0.15537907391531905, "PLAIN-3382": 0.35225906326661305, "PLAIN-3392": 0.0, "PLAIN-3402": 0.0, "PLAIN-3412": 0.23170768250378496, "PLAIN-3422": 0.03542769807752961, "PLAIN-3432": 0.2934556883974402, "PLAIN-3442": 0.060614569639142094, "PLAIN-3452": 0.5427357380331039, "PLAIN-3462": 0.5964984284535195, "PLAIN-3472": 0.0}, "Recall@20": {"PLAIN-2": 0.4166666666666667, "PLAIN-12": 0.13333333333333333, "PLAIN-23": 0.1, "PLAIN-33": 0.125, "PLAIN-44": 0.06896551724137931, "PLAIN-56": 0.2, "PLAIN-68": 0.05454545454545454, "PLAIN-78": 0.03225806451612903, "PLAIN-91": 0.15625, "PLAIN-102": 0.08333333333333333, "PLAIN-112": 0.08928571428571429, "PLAIN-123": 0.0196078431372549, "PLAIN-133": 0.027777777777777776, "PLAIN-143": 0.11764705882352941, "PLAIN-153": 0.2127659574468085, "PLAIN-165": 0.19444444444444445, "PLAIN-175": 0.10810810810810811, "PLAIN-186": 0.041666666666666664, "PLAIN-196": 0.07246376811594203, "PLAIN-207": 0.14545454545454545, "PLAIN-217": 0.08333333333333333, "PLAIN-227": 0.025, "PLAIN-238": 0.0, "PLAIN-248": 0.12903225806451613, "PLAIN-259": 0.14285714285714285, "PLAIN-270": 0.038461538461538464, "PLAIN-280": 0.2, "PLAIN-291": 0.3076923076923077, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 0.5, "PLAIN-344": 0.11475409836065574, "PLAIN-358": 0.0, "PLAIN-371": 0.5, "PLAIN-383": 0.4, "PLAIN-395": 0.0, "PLAIN-407": 0.3333333333333333, "PLAIN-418": 0.6666666666666666, "PLAIN-430": 0.375, "PLAIN-441": 1.0, "PLAIN-457": 0.0, "PLAIN-468": 0.2, "PLAIN-478": 0.0, "PLAIN-488": 0.7333333333333333, "PLAIN-499": 0.11764705882352941, "PLAIN-510": 0.16666666666666666, "PLAIN-520": 0.0, "PLAIN-531": 0.02922077922077922, "PLAIN-541": 0.3333333333333333, "PLAIN-551": 1.0, "PLAIN-561": 0.12, "PLAIN-571": 0.42857142857142855, "PLAIN-583": 0.022727272727272728, "PLAIN-593": 0.3333333333333333, "PLAIN-603": 0.0, "PLAIN-613": 0.1, "PLAIN-623": 0.07317073170731707, "PLAIN-634": 0.14285714285714285, "PLAIN-645": 0.0, "PLAIN-660": 0.035789473684210524, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.047619047619047616, "PLAIN-701": 0.5, "PLAIN-711": 0.14035087719298245, "PLAIN-721": 0.68, "PLAIN-731": 0.24074074074074073, "PLAIN-741": 0.47368421052631576, "PLAIN-751": 0.0, "PLAIN-761": 0.16666666666666666, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 0.5, "PLAIN-806": 0.125, "PLAIN-817": 1.0, "PLAIN-827": 0.04918032786885246, "PLAIN-838": 0.7777777777777778, "PLAIN-850": 0.17543859649122806, "PLAIN-872": 0.0, "PLAIN-882": 0.2, "PLAIN-892": 0.09782608695652174, "PLAIN-902": 0.5, "PLAIN-913": 0.2647058823529412, "PLAIN-924": 0.0625, "PLAIN-934": 0.1188118811881188, "PLAIN-946": 0.0967741935483871, "PLAIN-956": 0.05825242718446602, "PLAIN-966": 0.3333333333333333, "PLAIN-977": 0.3333333333333333, "PLAIN-987": 0.6666666666666666, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.11666666666666667, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.010638297872340425, "PLAIN-1066": 0.0, "PLAIN-1088": 0.4, "PLAIN-1098": 0.0, "PLAIN-1109": 0.10204081632653061, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 0.0625, "PLAIN-1151": 0.0718954248366013, "PLAIN-1161": 0.0, "PLAIN-1172": 0.05263157894736842, "PLAIN-1183": 0.08, "PLAIN-1193": 0.1111111111111111, "PLAIN-1203": 0.3333333333333333, "PLAIN-1214": 0.058823529411764705, "PLAIN-1225": 0.37037037037037035, "PLAIN-1236": 0.2, "PLAIN-1249": 0.0, "PLAIN-1262": 0.3, "PLAIN-1275": 0.14545454545454545, "PLAIN-1288": 0.04591836734693878, "PLAIN-1299": 0.03278688524590164, "PLAIN-1309": 0.0, "PLAIN-1320": 0.5, "PLAIN-1331": 0.0, "PLAIN-1342": 0.3125, "PLAIN-1353": 0.0, "PLAIN-1363": 0.09090909090909091, "PLAIN-1374": 0.07692307692307693, "PLAIN-1387": 0.6666666666666666, "PLAIN-1398": 0.17475728155339806, "PLAIN-1409": 0.019762845849802372, "PLAIN-1419": 0.06153846153846154, "PLAIN-1429": 0.0, "PLAIN-1441": 0.03763440860215054, "PLAIN-1453": 0.0326530612244898, "PLAIN-1463": 0.25, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.13333333333333333, "PLAIN-1516": 0.0, "PLAIN-1527": 0.07920792079207921, "PLAIN-1537": 0.13333333333333333, "PLAIN-1547": 0.0, "PLAIN-1557": 0.11538461538461539, "PLAIN-1568": 1.0, "PLAIN-1579": 0.2, "PLAIN-1590": 0.05555555555555555, "PLAIN-1601": 0.10810810810810811, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 0.03248259860788863, "PLAIN-1645": 0.16666666666666666, "PLAIN-1656": 0.03571428571428571, "PLAIN-1667": 0.06289308176100629, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 0.9375, "PLAIN-1721": 0.1935483870967742, "PLAIN-1731": 0.6666666666666666, "PLAIN-1741": 0.047619047619047616, "PLAIN-1752": 1.0, "PLAIN-1762": 0.375, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 0.12931034482758622, "PLAIN-1817": 0.18518518518518517, "PLAIN-1837": 0.08673469387755102, "PLAIN-1847": 0.18181818181818182, "PLAIN-1857": 0.21212121212121213, "PLAIN-1867": 0.14285714285714285, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 0.028077753779697623, "PLAIN-1919": 0.2962962962962963, "PLAIN-1929": 0.2, "PLAIN-1940": 0.16666666666666666, "PLAIN-1950": 0.23076923076923078, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 0.375, "PLAIN-1995": 0.0, "PLAIN-2009": 0.375, "PLAIN-2019": 0.6666666666666666, "PLAIN-2030": 0.16666666666666666, "PLAIN-2040": 0.06818181818181818, "PLAIN-2051": 0.030985915492957747, "PLAIN-2061": 0.041463414634146344, "PLAIN-2071": 0.2, "PLAIN-2081": 1.0, "PLAIN-2092": 0.5, "PLAIN-2102": 0.030878859857482184, "PLAIN-2113": 0.0, "PLAIN-2124": 0.35294117647058826, "PLAIN-2134": 0.0, "PLAIN-2145": 0.1111111111111111, "PLAIN-2156": 0.5, "PLAIN-2167": 0.0, "PLAIN-2177": 0.25, "PLAIN-2187": 0.2222222222222222, "PLAIN-2197": 0.1896551724137931, "PLAIN-2209": 0.0, "PLAIN-2220": 0.4444444444444444, "PLAIN-2230": 0.14285714285714285, "PLAIN-2240": 0.45454545454545453, "PLAIN-2250": 0.0, "PLAIN-2261": 0.11666666666666667, "PLAIN-2271": 0.0, "PLAIN-2281": 1.0, "PLAIN-2291": 0.2, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 0.08247422680412371, "PLAIN-2343": 0.0, "PLAIN-2354": 0.4, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.3157894736842105, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 0.15217391304347827, "PLAIN-2440": 0.047619047619047616, "PLAIN-2450": 0.05263157894736842, "PLAIN-2460": 0.12903225806451613, "PLAIN-2470": 0.0958904109589041, "PLAIN-2480": 0.2222222222222222, "PLAIN-2490": 0.21739130434782608, "PLAIN-2500": 0.14285714285714285, "PLAIN-2510": 0.2391304347826087, "PLAIN-2520": 0.0958904109589041, "PLAIN-2530": 0.20833333333333334, "PLAIN-2540": 0.3333333333333333, "PLAIN-2550": 0.0, "PLAIN-2560": 0.5238095238095238, "PLAIN-2570": 0.075, "PLAIN-2580": 0.08695652173913043, "PLAIN-2590": 0.06349206349206349, "PLAIN-2600": 0.4, "PLAIN-2610": 0.08196721311475409, "PLAIN-2620": 0.23636363636363636, "PLAIN-2630": 0.16666666666666666, "PLAIN-2640": 0.25, "PLAIN-2650": 0.2413793103448276, "PLAIN-2660": 0.18518518518518517, "PLAIN-2670": 0.05714285714285714, "PLAIN-2680": 0.35714285714285715, "PLAIN-2690": 0.13953488372093023, "PLAIN-2700": 0.2, "PLAIN-2710": 0.23809523809523808, "PLAIN-2720": 0.05555555555555555, "PLAIN-2730": 0.2413793103448276, "PLAIN-2740": 0.12, "PLAIN-2750": 0.15254237288135594, "PLAIN-2760": 0.25, "PLAIN-2770": 0.1951219512195122, "PLAIN-2780": 0.3181818181818182, "PLAIN-2790": 0.13559322033898305, "PLAIN-2800": 0.08333333333333333, "PLAIN-2810": 0.25, "PLAIN-2820": 0.125, "PLAIN-2830": 0.3333333333333333, "PLAIN-2840": 0.0, "PLAIN-2850": 0.1111111111111111, "PLAIN-2860": 1.0, "PLAIN-2870": 0.2, "PLAIN-2880": 0.0, "PLAIN-2890": 0.034482758620689655, "PLAIN-2900": 0.08333333333333333, "PLAIN-2910": 0.06666666666666667, "PLAIN-2920": 0.0, "PLAIN-2930": 0.14285714285714285, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.5, "PLAIN-2970": 0.2222222222222222, "PLAIN-2981": 0.2857142857142857, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 0.6666666666666666, "PLAIN-3063": 0.25, "PLAIN-3074": 1.0, "PLAIN-3085": 1.0, "PLAIN-3097": 0.5, "PLAIN-3116": 0.0, "PLAIN-3131": 0.5, "PLAIN-3141": 0.029411764705882353, "PLAIN-3151": 0.4444444444444444, "PLAIN-3161": 0.09302325581395349, "PLAIN-3171": 0.08823529411764706, "PLAIN-3181": 0.037037037037037035, "PLAIN-3191": 0.125, "PLAIN-3201": 0.09375, "PLAIN-3211": 0.0, "PLAIN-3221": 0.041666666666666664, "PLAIN-3231": 0.125, "PLAIN-3241": 0.06896551724137931, "PLAIN-3251": 0.09523809523809523, "PLAIN-3261": 0.07692307692307693, "PLAIN-3271": 0.0, "PLAIN-3281": 0.038461538461538464, "PLAIN-3292": 0.7142857142857143, "PLAIN-3302": 0.05405405405405406, "PLAIN-3312": 0.38461538461538464, "PLAIN-3322": 0.02531645569620253, "PLAIN-3332": 0.7142857142857143, "PLAIN-3342": 0.6666666666666666, "PLAIN-3352": 0.0, "PLAIN-3362": 0.2727272727272727, "PLAIN-3372": 0.07142857142857142, "PLAIN-3382": 0.5, "PLAIN-3392": 0.043478260869565216, "PLAIN-3402": 0.07142857142857142, "PLAIN-3412": 0.061224489795918366, "PLAIN-3422": 0.12903225806451613, "PLAIN-3432": 0.11904761904761904, "PLAIN-3442": 0.034482758620689655, "PLAIN-3452": 0.23529411764705882, "PLAIN-3462": 0.140625, "PLAIN-3472": 0.058823529411764705}, "MRR@10": {"PLAIN-2": 1.0, "PLAIN-12": 0.3333333333333333, "PLAIN-23": 0.5, "PLAIN-33": 0.5, "PLAIN-44": 0.25, "PLAIN-56": 1.0, "PLAIN-68": 1.0, "PLAIN-78": 0.5, "PLAIN-91": 1.0, "PLAIN-102": 0.3333333333333333, "PLAIN-112": 1.0, "PLAIN-123": 0.5, "PLAIN-133": 0.3333333333333333, "PLAIN-143": 1.0, "PLAIN-153": 1.0, "PLAIN-165": 0.5, "PLAIN-175": 1.0, "PLAIN-186": 1.0, "PLAIN-196": 1.0, "PLAIN-207": 0.3333333333333333, "PLAIN-217": 0.1111111111111111, "PLAIN-227": 0.3333333333333333, "PLAIN-238": 0.0, "PLAIN-248": 1.0, "PLAIN-259": 1.0, "PLAIN-270": 0.5, "PLAIN-280": 0.5, "PLAIN-291": 1.0, "PLAIN-307": 1.0, "PLAIN-320": 0.0, "PLAIN-332": 1.0, "PLAIN-344": 0.5, "PLAIN-358": 0.0, "PLAIN-371": 0.5, "PLAIN-383": 0.3333333333333333, "PLAIN-395": 0.0, "PLAIN-407": 1.0, "PLAIN-418": 1.0, "PLAIN-430": 1.0, "PLAIN-441": 1.0, "PLAIN-457": 0.0, "PLAIN-468": 1.0, "PLAIN-478": 0.0, "PLAIN-488": 1.0, "PLAIN-499": 1.0, "PLAIN-510": 0.07142857142857142, "PLAIN-520": 0.0, "PLAIN-531": 0.5, "PLAIN-541": 0.2, "PLAIN-551": 1.0, "PLAIN-561": 1.0, "PLAIN-571": 0.25, "PLAIN-583": 1.0, "PLAIN-593": 1.0, "PLAIN-603": 0.0, "PLAIN-613": 0.2, "PLAIN-623": 1.0, "PLAIN-634": 0.5, "PLAIN-645": 0.0, "PLAIN-660": 1.0, "PLAIN-671": 1.0, "PLAIN-681": 1.0, "PLAIN-691": 0.05263157894736842, "PLAIN-701": 1.0, "PLAIN-711": 1.0, "PLAIN-721": 1.0, "PLAIN-731": 1.0, "PLAIN-741": 1.0, "PLAIN-751": 0.0, "PLAIN-761": 0.3333333333333333, "PLAIN-771": 1.0, "PLAIN-782": 0.0, "PLAIN-792": 1.0, "PLAIN-806": 1.0, "PLAIN-817": 0.14285714285714285, "PLAIN-827": 1.0, "PLAIN-838": 1.0, "PLAIN-850": 1.0, "PLAIN-872": 0.0, "PLAIN-882": 1.0, "PLAIN-892": 1.0, "PLAIN-902": 1.0, "PLAIN-913": 1.0, "PLAIN-924": 1.0, "PLAIN-934": 1.0, "PLAIN-946": 0.5, "PLAIN-956": 0.5, "PLAIN-966": 0.0625, "PLAIN-977": 1.0, "PLAIN-987": 0.2, "PLAIN-997": 0.0, "PLAIN-1008": 0.0, "PLAIN-1018": 0.5, "PLAIN-1028": 0.0, "PLAIN-1039": 1.0, "PLAIN-1050": 0.09090909090909091, "PLAIN-1066": 0.0, "PLAIN-1088": 1.0, "PLAIN-1098": 0.0, "PLAIN-1109": 1.0, "PLAIN-1119": 0.0, "PLAIN-1130": 1.0, "PLAIN-1141": 1.0, "PLAIN-1151": 0.5, "PLAIN-1161": 0.0, "PLAIN-1172": 1.0, "PLAIN-1183": 0.25, "PLAIN-1193": 0.0625, "PLAIN-1203": 1.0, "PLAIN-1214": 0.06666666666666667, "PLAIN-1225": 1.0, "PLAIN-1236": 1.0, "PLAIN-1249": 0.0, "PLAIN-1262": 1.0, "PLAIN-1275": 1.0, "PLAIN-1288": 1.0, "PLAIN-1299": 1.0, "PLAIN-1309": 0.0, "PLAIN-1320": 1.0, "PLAIN-1331": 0.0, "PLAIN-1342": 1.0, "PLAIN-1353": 0.0, "PLAIN-1363": 1.0, "PLAIN-1374": 0.16666666666666666, "PLAIN-1387": 1.0, "PLAIN-1398": 1.0, "PLAIN-1409": 0.5, "PLAIN-1419": 1.0, "PLAIN-1429": 0.0, "PLAIN-1441": 1.0, "PLAIN-1453": 0.5, "PLAIN-1463": 1.0, "PLAIN-1473": 1.0, "PLAIN-1485": 0.0, "PLAIN-1496": 0.0, "PLAIN-1506": 0.5, "PLAIN-1516": 0.0, "PLAIN-1527": 1.0, "PLAIN-1537": 0.5, "PLAIN-1547": 0.0, "PLAIN-1557": 0.5, "PLAIN-1568": 1.0, "PLAIN-1579": 1.0, "PLAIN-1590": 0.25, "PLAIN-1601": 1.0, "PLAIN-1611": 0.0, "PLAIN-1621": 0.0, "PLAIN-1635": 1.0, "PLAIN-1645": 1.0, "PLAIN-1656": 0.2, "PLAIN-1667": 1.0, "PLAIN-1679": 0.0, "PLAIN-1690": 0.0, "PLAIN-1700": 0.0, "PLAIN-1710": 1.0, "PLAIN-1721": 1.0, "PLAIN-1731": 1.0, "PLAIN-1741": 1.0, "PLAIN-1752": 0.05, "PLAIN-1762": 1.0, "PLAIN-1772": 1.0, "PLAIN-1784": 0.0, "PLAIN-1794": 1.0, "PLAIN-1805": 1.0, "PLAIN-1817": 1.0, "PLAIN-1837": 1.0, "PLAIN-1847": 1.0, "PLAIN-1857": 1.0, "PLAIN-1867": 1.0, "PLAIN-1877": 0.0, "PLAIN-1887": 0.0, "PLAIN-1897": 0.0, "PLAIN-1909": 1.0, "PLAIN-1919": 1.0, "PLAIN-1929": 0.3333333333333333, "PLAIN-1940": 1.0, "PLAIN-1950": 0.5, "PLAIN-1962": 0.0, "PLAIN-1972": 0.0, "PLAIN-1983": 1.0, "PLAIN-1995": 0.0, "PLAIN-2009": 0.3333333333333333, "PLAIN-2019": 1.0, "PLAIN-2030": 0.3333333333333333, "PLAIN-2040": 1.0, "PLAIN-2051": 1.0, "PLAIN-2061": 1.0, "PLAIN-2071": 1.0, "PLAIN-2081": 1.0, "PLAIN-2092": 1.0, "PLAIN-2102": 1.0, "PLAIN-2113": 0.0, "PLAIN-2124": 1.0, "PLAIN-2134": 0.0, "PLAIN-2145": 0.25, "PLAIN-2156": 1.0, "PLAIN-2167": 0.0, "PLAIN-2177": 0.5, "PLAIN-2187": 1.0, "PLAIN-2197": 1.0, "PLAIN-2209": 0.0, "PLAIN-2220": 1.0, "PLAIN-2230": 0.1, "PLAIN-2240": 1.0, "PLAIN-2250": 0.0, "PLAIN-2261": 1.0, "PLAIN-2271": 0.0, "PLAIN-2281": 0.5, "PLAIN-2291": 1.0, "PLAIN-2301": 0.0, "PLAIN-2311": 0.0, "PLAIN-2321": 0.0, "PLAIN-2332": 1.0, "PLAIN-2343": 0.0, "PLAIN-2354": 1.0, "PLAIN-2364": 0.0, "PLAIN-2375": 0.0, "PLAIN-2386": 0.5, "PLAIN-2396": 0.0, "PLAIN-2408": 0.0, "PLAIN-2430": 1.0, "PLAIN-2440": 0.125, "PLAIN-2450": 1.0, "PLAIN-2460": 0.5, "PLAIN-2470": 1.0, "PLAIN-2480": 1.0, "PLAIN-2490": 1.0, "PLAIN-2500": 0.14285714285714285, "PLAIN-2510": 1.0, "PLAIN-2520": 0.5, "PLAIN-2530": 1.0, "PLAIN-2540": 1.0, "PLAIN-2550": 0.0, "PLAIN-2560": 1.0, "PLAIN-2570": 1.0, "PLAIN-2580": 1.0, "PLAIN-2590": 0.25, "PLAIN-2600": 1.0, "PLAIN-2610": 1.0, "PLAIN-2620": 1.0, "PLAIN-2630": 1.0, "PLAIN-2640": 1.0, "PLAIN-2650": 1.0, "PLAIN-2660": 0.3333333333333333, "PLAIN-2670": 1.0, "PLAIN-2680": 1.0, "PLAIN-2690": 1.0, "PLAIN-2700": 0.5, "PLAIN-2710": 1.0, "PLAIN-2720": 1.0, "PLAIN-2730": 0.5, "PLAIN-2740": 1.0, "PLAIN-2750": 1.0, "PLAIN-2760": 0.125, "PLAIN-2770": 0.5, "PLAIN-2780": 1.0, "PLAIN-2790": 1.0, "PLAIN-2800": 0.25, "PLAIN-2810": 0.125, "PLAIN-2820": 0.2, "PLAIN-2830": 0.5, "PLAIN-2840": 0.0, "PLAIN-2850": 0.3333333333333333, "PLAIN-2860": 1.0, "PLAIN-2870": 0.1111111111111111, "PLAIN-2880": 0.0, "PLAIN-2890": 0.25, "PLAIN-2900": 0.125, "PLAIN-2910": 0.5, "PLAIN-2920": 0.0, "PLAIN-2930": 1.0, "PLAIN-2940": 0.0, "PLAIN-2950": 0.0, "PLAIN-2960": 0.25, "PLAIN-2970": 1.0, "PLAIN-2981": 1.0, "PLAIN-2991": 0.5, "PLAIN-3001": 0.5, "PLAIN-3014": 0.0, "PLAIN-3026": 1.0, "PLAIN-3037": 0.0, "PLAIN-3053": 1.0, "PLAIN-3063": 0.05555555555555555, "PLAIN-3074": 1.0, "PLAIN-3085": 0.25, "PLAIN-3097": 1.0, "PLAIN-3116": 0.0, "PLAIN-3131": 1.0, "PLAIN-3141": 1.0, "PLAIN-3151": 1.0, "PLAIN-3161": 1.0, "PLAIN-3171": 0.25, "PLAIN-3181": 0.07142857142857142, "PLAIN-3191": 1.0, "PLAIN-3201": 0.5, "PLAIN-3211": 0.0, "PLAIN-3221": 0.058823529411764705, "PLAIN-3231": 0.058823529411764705, "PLAIN-3241": 1.0, "PLAIN-3251": 0.25, "PLAIN-3261": 0.3333333333333333, "PLAIN-3271": 0.0, "PLAIN-3281": 0.5, "PLAIN-3292": 0.3333333333333333, "PLAIN-3302": 1.0, "PLAIN-3312": 0.5, "PLAIN-3322": 0.07142857142857142, "PLAIN-3332": 1.0, "PLAIN-3342": 1.0, "PLAIN-3352": 0.0, "PLAIN-3362": 1.0, "PLAIN-3372": 0.25, "PLAIN-3382": 0.5, "PLAIN-3392": 0.058823529411764705, "PLAIN-3402": 0.058823529411764705, "PLAIN-3412": 1.0, "PLAIN-3422": 0.1111111111111111, "PLAIN-3432": 1.0, "PLAIN-3442": 0.25, "PLAIN-3452": 1.0, "PLAIN-3462": 0.5, "PLAIN-3472": 0.0625}}} diff --git a/evals/benchmark/results/nfcorpus/qrels.json b/evals/benchmark/results/nfcorpus/qrels.json new file mode 100644 index 000000000..f0b246cdb --- /dev/null +++ b/evals/benchmark/results/nfcorpus/qrels.json @@ -0,0 +1 @@ +{"PLAIN-2": {"MED-2427": 2, "MED-10": 2, "MED-2429": 2, "MED-2430": 2, "MED-2431": 2, "MED-14": 2, "MED-2432": 2, "MED-2428": 1, "MED-2440": 1, "MED-2434": 1, "MED-2435": 1, "MED-2436": 1, "MED-2437": 1, "MED-2438": 1, "MED-2439": 1, "MED-3597": 1, "MED-3598": 1, "MED-3599": 1, "MED-4556": 1, "MED-4559": 1, "MED-4560": 1, "MED-4828": 1, "MED-4829": 1, "MED-4830": 1}, "PLAIN-12": {"MED-2513": 2, "MED-5237": 2, "MED-2517": 2, "MED-2518": 2, "MED-2519": 2, "MED-2520": 2, "MED-2521": 2, "MED-2514": 1, "MED-2943": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-23": {"MED-2644": 2, "MED-2646": 2, "MED-2651": 2, "MED-118": 2, "MED-2652": 2, "MED-2653": 2, "MED-2655": 2, "MED-2659": 2, "MED-2661": 2, "MED-2662": 2, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2649": 1, "MED-2650": 1, "MED-2654": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-3021": 1, "MED-3023": 1, "MED-2904": 1, "MED-2905": 1, "MED-2906": 1, "MED-2907": 1, "MED-3024": 1, "MED-3025": 1, "MED-2910": 1, "MED-3027": 1, "MED-3028": 1, "MED-2913": 1, "MED-3012": 1, "MED-3013": 1, "MED-3030": 1, "MED-2917": 1, "MED-3033": 1, "MED-3034": 1, "MED-3035": 1, "MED-2921": 1, "MED-3585": 1, "MED-3586": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3590": 1, "MED-3591": 1, "MED-3592": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-3954": 1, "MED-3955": 1, "MED-3956": 1, "MED-4680": 1, "MED-3958": 1, "MED-3959": 1, "MED-4169": 1, "MED-4170": 1, "MED-4171": 1, "MED-4180": 1, "MED-4950": 1, "MED-4951": 1, "MED-5102": 1, "MED-5104": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-33": {"MED-2717": 2, "MED-2715": 2, "MED-2716": 2, "MED-2718": 2, "MED-2721": 2, "MED-2720": 2, "MED-2719": 2, "MED-2723": 2, "MED-2724": 2, "MED-2722": 1, "MED-4686": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-44": {"MED-2780": 2, "MED-2810": 2, "MED-2787": 2, "MED-2788": 2, "MED-2777": 2, "MED-2816": 2, "MED-2800": 1, "MED-2790": 1, "MED-2791": 1, "MED-2792": 1, "MED-2793": 1, "MED-2794": 1, "MED-4390": 1, "MED-2825": 1, "MED-3226": 1, "MED-3228": 1, "MED-3231": 1, "MED-4722": 1, "MED-3215": 1, "MED-3216": 1, "MED-3217": 1, "MED-3237": 1, "MED-3227": 1, "MED-3220": 1, "MED-3221": 1, "MED-3229": 1, "MED-3230": 1, "MED-3233": 1, "MED-3236": 1, "MED-4456": 1, "MED-4457": 1, "MED-4837": 1, "MED-4838": 1, "MED-4998": 1, "MED-4999": 1, "MED-5000": 1, "MED-5001": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-56": {"MED-2896": 2, "MED-2899": 2, "MED-2900": 2, "MED-2901": 2, "MED-2882": 1, "MED-2898": 1, "MED-2884": 1, "MED-2885": 1, "MED-2886": 1, "MED-2888": 1, "MED-2889": 1, "MED-2890": 1, "MED-2891": 1, "MED-2892": 1, "MED-2893": 1, "MED-2894": 1, "MED-2895": 1, "MED-4384": 1, "MED-4523": 1, "MED-2966": 1, "MED-2967": 1, "MED-2968": 1, "MED-2969": 1, "MED-2970": 1, "MED-2971": 1, "MED-2972": 1, "MED-4095": 1, "MED-4383": 1, "MED-4385": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-68": {"MED-2991": 2, "MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3089": 1, "MED-3090": 1, "MED-3091": 1, "MED-3092": 1, "MED-3093": 1, "MED-3094": 1, "MED-3095": 1, "MED-3096": 1, "MED-3880": 1, "MED-4132": 1, "MED-3882": 1, "MED-4136": 1, "MED-3884": 1, "MED-3885": 1, "MED-3886": 1, "MED-3887": 1, "MED-3888": 1, "MED-3889": 1, "MED-3890": 1, "MED-3891": 1, "MED-3892": 1, "MED-4068": 1, "MED-4069": 1, "MED-4070": 1, "MED-4071": 1, "MED-4072": 1, "MED-4073": 1, "MED-5197": 1, "MED-4075": 1, "MED-4128": 1, "MED-4129": 1, "MED-4130": 1, "MED-4131": 1, "MED-4133": 1, "MED-4134": 1, "MED-4135": 1, "MED-4137": 1, "MED-4138": 1, "MED-4139": 1, "MED-4140": 1, "MED-4141": 1, "MED-4142": 1, "MED-4143": 1, "MED-4746": 1, "MED-4804": 1, "MED-4976": 1, "MED-4972": 1, "MED-5110": 1, "MED-5169": 1, "MED-5170": 1}, "PLAIN-78": {"MED-332": 2, "MED-3090": 2, "MED-334": 2, "MED-335": 2, "MED-3092": 2, "MED-3094": 2, "MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3091": 1, "MED-3095": 1, "MED-3089": 1, "MED-3093": 1, "MED-3096": 1, "MED-3378": 1, "MED-3379": 1, "MED-3380": 1, "MED-3381": 1, "MED-3382": 1, "MED-3960": 1, "MED-3961": 1, "MED-3962": 1, "MED-3963": 1, "MED-3964": 1, "MED-3965": 1, "MED-3966": 1, "MED-3967": 1, "MED-3968": 1, "MED-3969": 1, "MED-3970": 1, "MED-4316": 1, "MED-4317": 1, "MED-4318": 1, "MED-5340": 1, "MED-4327": 1, "MED-4553": 1, "MED-4554": 1, "MED-4912": 1, "MED-4911": 1, "MED-4936": 1, "MED-4932": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-91": {"MED-3178": 2, "MED-3169": 2, "MED-3172": 2, "MED-3181": 2, "MED-3174": 2, "MED-3175": 2, "MED-3176": 2, "MED-3182": 2, "MED-3170": 1, "MED-3171": 1, "MED-3173": 1, "MED-3177": 1, "MED-3179": 1, "MED-3180": 1, "MED-3305": 1, "MED-3306": 1, "MED-3307": 1, "MED-3308": 1, "MED-3288": 1, "MED-3310": 1, "MED-3311": 1, "MED-3312": 1, "MED-3292": 1, "MED-3313": 1, "MED-3294": 1, "MED-3295": 1, "MED-3314": 1, "MED-3315": 1, "MED-3316": 1, "MED-4433": 1, "MED-3317": 1, "MED-3318": 1, "MED-3302": 1, "MED-3320": 1, "MED-3321": 1, "MED-4356": 1, "MED-4403": 1, "MED-4672": 1, "MED-4742": 1, "MED-4743": 1, "MED-4955": 1, "MED-4956": 1, "MED-4957": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-102": {"MED-3254": 2, "MED-3253": 2, "MED-3255": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-112": {"MED-3378": 2, "MED-3380": 2, "MED-5062": 2, "MED-3382": 2, "MED-2938": 1, "MED-2939": 1, "MED-2940": 1, "MED-2941": 1, "MED-2942": 1, "MED-2943": 1, "MED-2945": 1, "MED-2946": 1, "MED-2947": 1, "MED-3369": 1, "MED-3370": 1, "MED-3371": 1, "MED-3372": 1, "MED-3373": 1, "MED-3374": 1, "MED-3375": 1, "MED-3376": 1, "MED-3377": 1, "MED-3379": 1, "MED-3381": 1, "MED-3385": 1, "MED-3386": 1, "MED-3387": 1, "MED-4488": 1, "MED-4489": 1, "MED-4490": 1, "MED-4491": 1, "MED-4492": 1, "MED-4936": 1, "MED-4932": 1, "MED-5063": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-123": {"MED-3474": 2, "MED-3479": 2, "MED-3482": 2, "MED-3484": 2, "MED-3485": 2, "MED-3477": 2, "MED-3467": 1, "MED-3473": 1, "MED-3469": 1, "MED-3481": 1, "MED-3471": 1, "MED-3478": 1, "MED-3475": 1, "MED-3476": 1, "MED-3480": 1, "MED-3483": 1, "MED-4022": 1, "MED-4023": 1, "MED-4024": 1, "MED-4025": 1, "MED-4029": 1, "MED-4013": 1, "MED-4030": 1, "MED-4031": 1, "MED-4032": 1, "MED-4033": 1, "MED-4034": 1, "MED-4019": 1, "MED-4035": 1, "MED-4036": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-133": {"MED-3549": 2, "MED-3550": 2, "MED-3551": 2, "MED-3554": 2, "MED-3555": 2, "MED-3548": 1, "MED-3552": 1, "MED-3553": 1, "MED-4089": 1, "MED-4090": 1, "MED-4652": 1, "MED-5072": 1, "MED-5069": 1, "MED-5070": 1, "MED-5071": 1, "MED-5341": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5342": 1}, "PLAIN-143": {"MED-3618": 2, "MED-3621": 2, "MED-3619": 1, "MED-3601": 1, "MED-3602": 1, "MED-3603": 1, "MED-3604": 1, "MED-3615": 1, "MED-3606": 1, "MED-3607": 1, "MED-3608": 1, "MED-3609": 1, "MED-3610": 1, "MED-3617": 1, "MED-3620": 1, "MED-3622": 1, "MED-3623": 1, "MED-3624": 1, "MED-3625": 1, "MED-3626": 1, "MED-3627": 1, "MED-3628": 1, "MED-3629": 1, "MED-3630": 1, "MED-3631": 1, "MED-3632": 1, "MED-3633": 1, "MED-3634": 1, "MED-3635": 1, "MED-3636": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-153": {"MED-3679": 2, "MED-3671": 1, "MED-3672": 1, "MED-3673": 1, "MED-3674": 1, "MED-3675": 1, "MED-3676": 1, "MED-3677": 1, "MED-3678": 1, "MED-3680": 1, "MED-3681": 1, "MED-3682": 1, "MED-3683": 1, "MED-3693": 1, "MED-3685": 1, "MED-3686": 1, "MED-3687": 1, "MED-3688": 1, "MED-3689": 1, "MED-3690": 1, "MED-3691": 1, "MED-3692": 1, "MED-3694": 1, "MED-3695": 1, "MED-4566": 1, "MED-4574": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-165": {"MED-3762": 2, "MED-3763": 2, "MED-3764": 2, "MED-3696": 1, "MED-3697": 1, "MED-3698": 1, "MED-3699": 1, "MED-3700": 1, "MED-3701": 1, "MED-3765": 1, "MED-3766": 1, "MED-3767": 1, "MED-3768": 1, "MED-4507": 1, "MED-4994": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-175": {"MED-3815": 2, "MED-3816": 2, "MED-3820": 2, "MED-3821": 2, "MED-3817": 2, "MED-3818": 1, "MED-3819": 1, "MED-3822": 1, "MED-4256": 1, "MED-4257": 1, "MED-4546": 1, "MED-4547": 1, "MED-4958": 1, "MED-4959": 1, "MED-4960": 1, "MED-4961": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-186": {"MED-3904": 1, "MED-5175": 1, "MED-4810": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-196": {"MED-3960": 1, "MED-3961": 1, "MED-3962": 1, "MED-3963": 1, "MED-3964": 1, "MED-3965": 1, "MED-3966": 1, "MED-3967": 1, "MED-3968": 1, "MED-3969": 1, "MED-3970": 1, "MED-4068": 1, "MED-4069": 1, "MED-4070": 1, "MED-4071": 1, "MED-4072": 1, "MED-4073": 1, "MED-5197": 1, "MED-4075": 1, "MED-4853": 1, "MED-4128": 1, "MED-4129": 1, "MED-4130": 1, "MED-4131": 1, "MED-4132": 1, "MED-4133": 1, "MED-4134": 1, "MED-4135": 1, "MED-4136": 1, "MED-4137": 1, "MED-4138": 1, "MED-4139": 1, "MED-4140": 1, "MED-4141": 1, "MED-4142": 1, "MED-4143": 1, "MED-4167": 1, "MED-4168": 1, "MED-4333": 1, "MED-4337": 1, "MED-4342": 1, "MED-4349": 1, "MED-5326": 1, "MED-4488": 1, "MED-4489": 1, "MED-4490": 1, "MED-4491": 1, "MED-4492": 1, "MED-4627": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-207": {"MED-4053": 1, "MED-4058": 1, "MED-4055": 1, "MED-4060": 1, "MED-4057": 1, "MED-4059": 1, "MED-4075": 1, "MED-4068": 1, "MED-4069": 1, "MED-4070": 1, "MED-4072": 1, "MED-5197": 1, "MED-4071": 1, "MED-4073": 1, "MED-4180": 1, "MED-4365": 1, "MED-4474": 1, "MED-4475": 1, "MED-4476": 1, "MED-4477": 1, "MED-4488": 1, "MED-4489": 1, "MED-4490": 1, "MED-4491": 1, "MED-4492": 1, "MED-4726": 1, "MED-4976": 1, "MED-4972": 1, "MED-4973": 1, "MED-4974": 1, "MED-4975": 1, "MED-4978": 1, "MED-5104": 1, "MED-5105": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-217": {"MED-4103": 1, "MED-4104": 1, "MED-4853": 1, "MED-4843": 1, "MED-4844": 1, "MED-4845": 1, "MED-4846": 1, "MED-4847": 1, "MED-4848": 1, "MED-4849": 1, "MED-4850": 1, "MED-4851": 1, "MED-4852": 1, "MED-4854": 1, "MED-5090": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-227": {"MED-4544": 1, "MED-4150": 1, "MED-5341": 1, "MED-5337": 1, "MED-4464": 1, "MED-4465": 1, "MED-4468": 1, "MED-4469": 1, "MED-4501": 1, "MED-4502": 1, "MED-4543": 1, "MED-4538": 1, "MED-4539": 1, "MED-4540": 1, "MED-4541": 1, "MED-4671": 1, "MED-5047": 1, "MED-5048": 1, "MED-5049": 1, "MED-5050": 1, "MED-5140": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5342": 1}, "PLAIN-238": {"MED-4474": 1, "MED-4475": 1, "MED-4476": 1, "MED-4477": 1, "MED-4479": 1, "MED-4480": 1, "MED-4481": 1, "MED-4482": 1, "MED-4483": 1, "MED-4484": 1, "MED-4485": 1, "MED-4486": 1, "MED-4487": 1, "MED-4488": 1, "MED-4489": 1, "MED-4490": 1, "MED-4491": 1, "MED-4492": 1, "MED-4739": 1, "MED-4740": 1, "MED-4741": 1, "MED-4747": 1, "MED-4748": 1, "MED-4749": 1, "MED-4750": 1, "MED-4751": 1, "MED-4804": 1, "MED-5072": 1, "MED-5069": 1, "MED-5070": 1, "MED-5071": 1, "MED-5167": 1, "MED-5168": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-248": {"MED-5176": 1, "MED-4230": 1, "MED-4231": 1, "MED-4232": 1, "MED-4233": 1, "MED-4234": 1, "MED-4235": 1, "MED-4236": 1, "MED-4237": 1, "MED-4239": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-259": {"MED-4295": 1, "MED-4846": 1, "MED-4296": 1, "MED-4298": 1, "MED-4300": 1, "MED-4301": 1, "MED-4637": 1, "MED-4638": 1, "MED-4639": 1, "MED-4640": 1, "MED-4641": 1, "MED-4642": 1, "MED-4643": 1, "MED-4644": 1}, "PLAIN-270": {"MED-4328": 1, "MED-4329": 1, "MED-4330": 1, "MED-4331": 1, "MED-4332": 1, "MED-4415": 1, "MED-4416": 1, "MED-4417": 1, "MED-4418": 1, "MED-4419": 1, "MED-5165": 1, "MED-4421": 1, "MED-4422": 1, "MED-4438": 1, "MED-4439": 1, "MED-4471": 1, "MED-4472": 1, "MED-4976": 1, "MED-4543": 1, "MED-4538": 1, "MED-4539": 1, "MED-4540": 1, "MED-4541": 1, "MED-4602": 1, "MED-4671": 1, "MED-4775": 1, "MED-4867": 1, "MED-4868": 1, "MED-4869": 1, "MED-4877": 1, "MED-4878": 1, "MED-4879": 1, "MED-4886": 1, "MED-4887": 1, "MED-4888": 1, "MED-5337": 1, "MED-4890": 1, "MED-4891": 1, "MED-5046": 1, "MED-5047": 1, "MED-5048": 1, "MED-5049": 1, "MED-5050": 1, "MED-5052": 1, "MED-5058": 1, "MED-5064": 1, "MED-5065": 1, "MED-5176": 1, "MED-5177": 1, "MED-5178": 1, "MED-5179": 1, "MED-5184": 1}, "PLAIN-280": {"MED-4375": 1, "MED-4376": 1, "MED-4378": 1, "MED-4379": 1, "MED-4380": 1, "MED-4381": 1, "MED-4531": 1, "MED-4546": 1, "MED-4547": 1, "MED-4622": 1, "MED-4736": 1, "MED-4739": 1, "MED-4738": 1, "MED-4946": 1, "MED-4951": 1, "MED-5097": 1, "MED-5098": 1, "MED-5099": 1, "MED-5100": 1, "MED-5101": 1}, "PLAIN-291": {"MED-4516": 1, "MED-4517": 1, "MED-4518": 1, "MED-4522": 1, "MED-4635": 1, "MED-4640": 1, "MED-4637": 1, "MED-4638": 1, "MED-4639": 1, "MED-4641": 1, "MED-4642": 1, "MED-4643": 1, "MED-4644": 1}, "PLAIN-307": {"MED-714": 2, "MED-919": 2, "MED-716": 2}, "PLAIN-320": {"MED-4650": 1, "MED-4747": 1, "MED-4748": 1, "MED-4749": 1, "MED-4750": 1, "MED-4751": 1, "MED-4771": 1, "MED-4783": 1, "MED-5116": 1, "MED-4785": 1, "MED-4786": 1, "MED-4896": 1, "MED-4897": 1, "MED-4898": 1, "MED-4899": 1, "MED-4994": 1, "MED-5032": 1, "MED-5033": 1, "MED-5034": 1, "MED-5035": 1, "MED-5110": 1, "MED-5112": 1, "MED-5113": 1, "MED-5114": 1, "MED-5115": 1, "MED-5117": 1, "MED-5118": 1, "MED-5176": 1, "MED-5177": 1, "MED-5178": 1, "MED-5179": 1, "MED-5184": 1, "MED-5188": 1, "MED-5189": 1, "MED-5190": 1, "MED-5191": 1, "MED-5192": 1, "MED-5193": 1, "MED-5194": 1, "MED-5195": 1, "MED-5196": 1, "MED-5197": 1}, "PLAIN-332": {"MED-759": 2, "MED-760": 2}, "PLAIN-344": {"MED-2394": 2, "MED-2395": 2, "MED-2397": 2, "MED-2404": 2, "MED-2400": 2, "MED-2385": 1, "MED-2386": 1, "MED-2388": 1, "MED-2389": 1, "MED-2396": 1, "MED-2391": 1, "MED-2398": 1, "MED-3028": 1, "MED-2402": 1, "MED-2406": 1, "MED-2409": 1, "MED-2412": 1, "MED-2413": 1, "MED-4551": 1, "MED-2643": 1, "MED-2644": 1, "MED-2645": 1, "MED-2646": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2649": 1, "MED-2650": 1, "MED-2651": 1, "MED-2652": 1, "MED-2653": 1, "MED-2654": 1, "MED-2655": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2659": 1, "MED-2660": 1, "MED-2661": 1, "MED-2662": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-358": {"MED-4324": 1, "MED-4556": 1, "MED-4559": 1}, "PLAIN-371": {"MED-861": 2, "MED-4618": 1}, "PLAIN-383": {"MED-4203": 1, "MED-4299": 1, "MED-4205": 1, "MED-4206": 1, "MED-4676": 1}, "PLAIN-395": {"MED-892": 2, "MED-4390": 2, "MED-4993": 1}, "PLAIN-407": {"MED-906": 2, "MED-4936": 1, "MED-4932": 1}, "PLAIN-418": {"MED-3714": 2, "MED-917": 2, "MED-4545": 1}, "PLAIN-430": {"MED-927": 2, "MED-928": 2, "MED-4936": 2, "MED-930": 2, "MED-931": 2, "MED-5091": 1, "MED-5092": 1, "MED-5093": 1}, "PLAIN-441": {"MED-941": 2, "MED-942": 2, "MED-4701": 1, "MED-4702": 1, "MED-4703": 1}, "PLAIN-457": {"MED-952": 2, "MED-5159": 1, "MED-5160": 1, "MED-5161": 1, "MED-5162": 1}, "PLAIN-468": {"MED-961": 2, "MED-962": 2, "MED-5128": 1, "MED-5129": 1, "MED-5130": 1, "MED-5131": 1, "MED-5132": 1, "MED-5133": 1, "MED-5134": 1, "MED-5135": 1}, "PLAIN-478": {"MED-4222": 1}, "PLAIN-488": {"MED-1796": 1, "MED-1797": 1, "MED-1798": 1, "MED-1799": 1, "MED-1800": 1, "MED-1801": 1, "MED-1802": 1, "MED-1803": 1, "MED-1804": 1, "MED-4767": 1, "MED-1806": 1, "MED-1807": 1, "MED-1808": 1, "MED-4769": 1, "MED-1810": 1}, "PLAIN-499": {"MED-2896": 1, "MED-2899": 1, "MED-2900": 1, "MED-2901": 1, "MED-1103": 1, "MED-1104": 1, "MED-1105": 1, "MED-1106": 1, "MED-1107": 1, "MED-1108": 1, "MED-1109": 1, "MED-1110": 1, "MED-1111": 1, "MED-1112": 1, "MED-1113": 1, "MED-1114": 1, "MED-1115": 1, "MED-2211": 1, "MED-2212": 1, "MED-2213": 1, "MED-2214": 1, "MED-2215": 1, "MED-2216": 1, "MED-2217": 1, "MED-2218": 1, "MED-4384": 1, "MED-2898": 1, "MED-3874": 1, "MED-3875": 1, "MED-3876": 1, "MED-3877": 1, "MED-3878": 1, "MED-4233": 1, "MED-4095": 1}, "PLAIN-510": {"MED-3617": 1, "MED-3618": 1, "MED-3620": 1, "MED-3621": 1, "MED-3615": 1, "MED-3622": 1}, "PLAIN-520": {"MED-5155": 1}, "PLAIN-531": {"MED-2527": 1, "MED-2706": 1, "MED-2708": 1, "MED-2707": 1, "MED-2711": 1, "MED-2712": 1, "MED-2713": 1, "MED-2709": 1, "MED-2710": 1, "MED-2780": 1, "MED-2810": 1, "MED-2787": 1, "MED-2788": 1, "MED-2777": 1, "MED-2816": 1, "MED-2790": 1, "MED-2792": 1, "MED-2793": 1, "MED-2794": 1, "MED-4390": 1, "MED-2797": 1, "MED-2809": 1, "MED-2799": 1, "MED-2800": 1, "MED-2801": 1, "MED-2802": 1, "MED-2803": 1, "MED-2804": 1, "MED-2805": 1, "MED-2807": 1, "MED-2813": 1, "MED-2822": 1, "MED-2830": 1, "MED-3146": 1, "MED-3145": 1, "MED-3383": 1, "MED-3384": 1, "MED-3534": 1, "MED-3464": 1, "MED-3460": 1, "MED-3465": 1, "MED-3461": 1, "MED-3466": 1, "MED-5293": 1, "MED-2082": 1, "MED-1655": 1, "MED-1097": 1, "MED-1240": 1, "MED-1241": 1, "MED-1242": 1, "MED-1243": 1, "MED-1244": 1, "MED-1245": 1, "MED-1246": 1, "MED-1247": 1, "MED-1248": 1, "MED-1311": 1, "MED-1312": 1, "MED-1313": 1, "MED-1314": 1, "MED-1315": 1, "MED-1316": 1, "MED-1317": 1, "MED-1441": 1, "MED-1442": 1, "MED-1443": 1, "MED-1444": 1, "MED-1937": 1, "MED-2783": 1, "MED-1939": 1, "MED-2786": 1, "MED-1941": 1, "MED-2529": 1, "MED-2524": 1, "MED-2525": 1, "MED-2530": 1, "MED-2528": 1, "MED-2274": 1, "MED-3535": 1, "MED-1998": 1, "MED-1999": 1, "MED-2000": 1, "MED-2001": 1, "MED-2002": 1, "MED-2003": 1, "MED-2004": 1, "MED-2005": 1, "MED-2006": 1, "MED-2007": 1, "MED-2042": 1, "MED-2043": 1, "MED-2044": 1, "MED-2045": 1, "MED-2046": 1, "MED-2047": 1, "MED-2048": 1, "MED-2049": 1, "MED-2050": 1, "MED-2051": 1, "MED-2052": 1, "MED-2140": 1, "MED-2141": 1, "MED-4319": 1, "MED-2143": 1, "MED-2144": 1, "MED-2145": 1, "MED-2146": 1, "MED-2374": 1, "MED-2237": 1, "MED-2238": 1, "MED-2239": 1, "MED-2240": 1, "MED-2245": 1, "MED-2814": 1, "MED-2243": 1, "MED-2265": 1, "MED-2266": 1, "MED-2267": 1, "MED-3462": 1, "MED-2269": 1, "MED-2270": 1, "MED-2271": 1, "MED-2279": 1, "MED-2280": 1, "MED-2281": 1, "MED-2282": 1, "MED-2284": 1, "MED-2285": 1, "MED-2287": 1, "MED-2297": 1, "MED-2298": 1, "MED-2299": 1, "MED-2300": 1, "MED-2301": 1, "MED-5303": 1, "MED-2303": 1, "MED-2304": 1, "MED-2305": 1, "MED-2306": 1, "MED-2313": 1, "MED-2315": 1, "MED-2815": 1, "MED-2322": 1, "MED-2823": 1, "MED-2323": 1, "MED-2332": 1, "MED-2333": 1, "MED-2335": 1, "MED-2336": 1, "MED-2526": 1, "MED-2714": 1, "MED-2781": 1, "MED-2782": 1, "MED-2818": 1, "MED-2785": 1, "MED-2791": 1, "MED-2825": 1, "MED-2819": 1, "MED-2808": 1, "MED-2811": 1, "MED-2812": 1, "MED-2817": 1, "MED-2820": 1, "MED-2821": 1, "MED-2824": 1, "MED-2831": 1, "MED-3144": 1, "MED-3459": 1, "MED-3515": 1, "MED-3516": 1, "MED-3517": 1, "MED-3536": 1, "MED-3537": 1, "MED-3538": 1, "MED-3539": 1, "MED-3540": 1, "MED-3541": 1, "MED-3542": 1, "MED-3543": 1, "MED-3544": 1, "MED-3545": 1, "MED-3546": 1, "MED-3547": 1, "MED-3658": 1, "MED-3659": 1, "MED-3660": 1, "MED-3661": 1, "MED-3662": 1, "MED-3663": 1, "MED-3664": 1, "MED-3688": 1, "MED-3689": 1, "MED-3690": 1, "MED-3691": 1, "MED-3692": 1, "MED-3693": 1, "MED-3694": 1, "MED-3695": 1, "MED-3749": 1, "MED-3750": 1, "MED-3751": 1, "MED-3752": 1, "MED-3753": 1, "MED-3754": 1, "MED-3755": 1, "MED-3756": 1, "MED-3757": 1, "MED-3758": 1, "MED-3759": 1, "MED-3760": 1, "MED-3761": 1, "MED-3796": 1, "MED-3778": 1, "MED-3779": 1, "MED-3791": 1, "MED-3792": 1, "MED-3793": 1, "MED-3794": 1, "MED-3795": 1, "MED-3797": 1, "MED-3798": 1, "MED-3799": 1, "MED-3800": 1, "MED-3801": 1, "MED-3866": 1, "MED-3867": 1, "MED-3868": 1, "MED-3869": 1, "MED-3876": 1, "MED-3877": 1, "MED-3874": 1, "MED-3878": 1, "MED-3875": 1, "MED-4233": 1, "MED-3904": 1, "MED-5175": 1, "MED-4156": 1, "MED-4157": 1, "MED-4158": 1, "MED-4159": 1, "MED-4160": 1, "MED-4161": 1, "MED-4162": 1, "MED-4188": 1, "MED-4193": 1, "MED-4192": 1, "MED-4309": 1, "MED-4312": 1, "MED-4328": 1, "MED-4329": 1, "MED-4330": 1, "MED-4365": 1, "MED-4366": 1, "MED-4370": 1, "MED-4371": 1, "MED-4369": 1, "MED-4372": 1, "MED-4373": 1, "MED-4374": 1, "MED-4533": 1, "MED-4529": 1, "MED-4535": 1, "MED-4531": 1, "MED-4532": 1, "MED-4534": 1, "MED-4536": 1, "MED-4539": 1, "MED-4540": 1, "MED-4541": 1, "MED-4543": 1, "MED-4544": 1, "MED-4563": 1, "MED-4564": 1, "MED-4565": 1, "MED-4595": 1, "MED-4669": 1, "MED-4671": 1, "MED-4711": 1, "MED-4840": 1, "MED-4842": 1, "MED-4855": 1, "MED-4856": 1, "MED-4864": 1, "MED-4870": 1, "MED-4872": 1, "MED-4873": 1, "MED-4874": 1, "MED-4876": 1, "MED-4930": 1, "MED-5022": 1, "MED-5023": 1, "MED-5024": 1, "MED-5025": 1, "MED-5029": 1, "MED-5062": 1, "MED-5063": 1, "MED-5122": 1, "MED-5120": 1, "MED-5121": 1, "MED-5157": 1, "MED-5158": 1, "MED-5172": 1, "MED-5173": 1, "MED-5176": 1, "MED-5177": 1, "MED-5178": 1, "MED-5179": 1, "MED-5184": 1}, "PLAIN-541": {"MED-3618": 1, "MED-3621": 1, "MED-3617": 1, "MED-3619": 1, "MED-3620": 1, "MED-3622": 1}, "PLAIN-551": {"MED-4380": 1, "MED-4381": 1}, "PLAIN-561": {"MED-4422": 1, "MED-3169": 1, "MED-3170": 1, "MED-3171": 1, "MED-3172": 1, "MED-3173": 1, "MED-3174": 1, "MED-3175": 1, "MED-3176": 1, "MED-3177": 1, "MED-3178": 1, "MED-3179": 1, "MED-3180": 1, "MED-3181": 1, "MED-3182": 1, "MED-3255": 1, "MED-3540": 1, "MED-3541": 1, "MED-3542": 1, "MED-3543": 1, "MED-3544": 1, "MED-3545": 1, "MED-3546": 1, "MED-3547": 1, "MED-4555": 1}, "PLAIN-571": {"MED-2337": 1, "MED-2338": 1, "MED-2340": 1, "MED-2355": 1, "MED-2349": 1, "MED-4742": 1, "MED-4743": 1}, "PLAIN-583": {"MED-2575": 1, "MED-2580": 1, "MED-2571": 1, "MED-2583": 1, "MED-2574": 1, "MED-2568": 1, "MED-2988": 1, "MED-2559": 1, "MED-4319": 1, "MED-2570": 1, "MED-2573": 1, "MED-2585": 1, "MED-2980": 1, "MED-2990": 1, "MED-2986": 1, "MED-2985": 1, "MED-2987": 1, "MED-2572": 1, "MED-2577": 1, "MED-2578": 1, "MED-2579": 1, "MED-2544": 1, "MED-2581": 1, "MED-2546": 1, "MED-2584": 1, "MED-2582": 1, "MED-2979": 1, "MED-2982": 1, "MED-2983": 1, "MED-2984": 1, "MED-2989": 1, "MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3089": 1, "MED-3090": 1, "MED-3091": 1, "MED-3092": 1, "MED-3093": 1, "MED-3094": 1, "MED-3095": 1, "MED-3096": 1, "MED-4320": 1}, "PLAIN-593": {"MED-4399": 1, "MED-4400": 1, "MED-4402": 1}, "PLAIN-603": {"MED-4413": 1}, "PLAIN-613": {"MED-1195": 1, "MED-1196": 1, "MED-5360": 1, "MED-1198": 1, "MED-1199": 1, "MED-1200": 1, "MED-1201": 1, "MED-1202": 1, "MED-1203": 1, "MED-4488": 1}, "PLAIN-623": {"MED-2522": 1, "MED-2586": 1, "MED-2587": 1, "MED-2588": 1, "MED-2591": 1, "MED-3780": 1, "MED-3784": 1, "MED-1551": 1, "MED-1552": 1, "MED-1553": 1, "MED-1554": 1, "MED-1555": 1, "MED-1556": 1, "MED-1609": 1, "MED-1610": 1, "MED-1611": 1, "MED-1612": 1, "MED-1613": 1, "MED-1614": 1, "MED-1615": 1, "MED-1616": 1, "MED-1617": 1, "MED-1618": 1, "MED-1619": 1, "MED-1620": 1, "MED-2523": 1, "MED-2589": 1, "MED-2590": 1, "MED-3786": 1, "MED-3781": 1, "MED-3789": 1, "MED-3193": 1, "MED-3783": 1, "MED-3785": 1, "MED-4514": 1, "MED-4515": 1, "MED-4880": 1, "MED-4881": 1, "MED-4882": 1, "MED-4883": 1, "MED-4884": 1}, "PLAIN-634": {"MED-1463": 1, "MED-1454": 1, "MED-1455": 1, "MED-1456": 1, "MED-1457": 1, "MED-1458": 1, "MED-1459": 1, "MED-1460": 1, "MED-1461": 1, "MED-1474": 1, "MED-4514": 1, "MED-4515": 1, "MED-4618": 1, "MED-4619": 1, "MED-4620": 1, "MED-4621": 1, "MED-4673": 1, "MED-4674": 1, "MED-5153": 1, "MED-5169": 1, "MED-5170": 1}, "PLAIN-645": {"MED-3146": 1, "MED-3145": 1, "MED-2033": 1, "MED-2013": 1, "MED-2014": 1, "MED-2027": 1, "MED-2016": 1, "MED-2037": 1, "MED-2018": 1, "MED-2019": 1, "MED-2020": 1, "MED-2021": 1, "MED-2022": 1, "MED-2024": 1, "MED-2034": 1, "MED-2026": 1, "MED-2028": 1, "MED-2029": 1, "MED-2030": 1, "MED-2031": 1, "MED-2032": 1, "MED-2035": 1, "MED-2036": 1, "MED-2038": 1, "MED-2039": 1, "MED-2040": 1, "MED-2041": 1, "MED-3144": 1, "MED-5154": 1}, "PLAIN-660": {"MED-2513": 1, "MED-2502": 1, "MED-2498": 1, "MED-2504": 1, "MED-2505": 1, "MED-2507": 1, "MED-5239": 1, "MED-2509": 1, "MED-2519": 1, "MED-2510": 1, "MED-2511": 1, "MED-2575": 1, "MED-2580": 1, "MED-2571": 1, "MED-2583": 1, "MED-2574": 1, "MED-2568": 1, "MED-2988": 1, "MED-2559": 1, "MED-4319": 1, "MED-2570": 1, "MED-2573": 1, "MED-2585": 1, "MED-2824": 1, "MED-2815": 1, "MED-2794": 1, "MED-2816": 1, "MED-2999": 1, "MED-2997": 1, "MED-2765": 1, "MED-3001": 1, "MED-2797": 1, "MED-2809": 1, "MED-2799": 1, "MED-2800": 1, "MED-2801": 1, "MED-2802": 1, "MED-2803": 1, "MED-2804": 1, "MED-2805": 1, "MED-2807": 1, "MED-2966": 1, "MED-2968": 1, "MED-2970": 1, "MED-2972": 1, "MED-4981": 1, "MED-2980": 1, "MED-2990": 1, "MED-2986": 1, "MED-2985": 1, "MED-2987": 1, "MED-717": 1, "MED-718": 1, "MED-719": 1, "MED-720": 1, "MED-721": 1, "MED-722": 1, "MED-723": 1, "MED-724": 1, "MED-5204": 1, "MED-1184": 1, "MED-1185": 1, "MED-1186": 1, "MED-1187": 1, "MED-1188": 1, "MED-1582": 1, "MED-1190": 1, "MED-5293": 1, "MED-1192": 1, "MED-1193": 1, "MED-1194": 1, "MED-1195": 1, "MED-1196": 1, "MED-5360": 1, "MED-1198": 1, "MED-1199": 1, "MED-1200": 1, "MED-1201": 1, "MED-1202": 1, "MED-1203": 1, "MED-1231": 1, "MED-1232": 1, "MED-1233": 1, "MED-1234": 1, "MED-1375": 1, "MED-1362": 1, "MED-1363": 1, "MED-1383": 1, "MED-1365": 1, "MED-1366": 1, "MED-1643": 1, "MED-1374": 1, "MED-1380": 1, "MED-5301": 1, "MED-1371": 1, "MED-5271": 1, "MED-1373": 1, "MED-1376": 1, "MED-1377": 1, "MED-1378": 1, "MED-1401": 1, "MED-1402": 1, "MED-1403": 1, "MED-1404": 1, "MED-1405": 1, "MED-1406": 1, "MED-1407": 1, "MED-1408": 1, "MED-1409": 1, "MED-1410": 1, "MED-1411": 1, "MED-1412": 1, "MED-1413": 1, "MED-1414": 1, "MED-1415": 1, "MED-1416": 1, "MED-1417": 1, "MED-1445": 1, "MED-1446": 1, "MED-1447": 1, "MED-1448": 1, "MED-1449": 1, "MED-1450": 1, "MED-1451": 1, "MED-1527": 1, "MED-1528": 1, "MED-1529": 1, "MED-1530": 1, "MED-1531": 1, "MED-1532": 1, "MED-1542": 1, "MED-1543": 1, "MED-1544": 1, "MED-1545": 1, "MED-1546": 1, "MED-2527": 1, "MED-1548": 1, "MED-1549": 1, "MED-1680": 1, "MED-1609": 1, "MED-1610": 1, "MED-1611": 1, "MED-1612": 1, "MED-1613": 1, "MED-1614": 1, "MED-1615": 1, "MED-1616": 1, "MED-1617": 1, "MED-1618": 1, "MED-1619": 1, "MED-1620": 1, "MED-1703": 1, "MED-1699": 1, "MED-1700": 1, "MED-1701": 1, "MED-1702": 1, "MED-1880": 1, "MED-1881": 1, "MED-2722": 1, "MED-2489": 1, "MED-1998": 1, "MED-1985": 1, "MED-1986": 1, "MED-1987": 1, "MED-1988": 1, "MED-1999": 1, "MED-1990": 1, "MED-1991": 1, "MED-1992": 1, "MED-1993": 1, "MED-1994": 1, "MED-2221": 1, "MED-1996": 1, "MED-1997": 1, "MED-2008": 1, "MED-2009": 1, "MED-2010": 1, "MED-2011": 1, "MED-2089": 1, "MED-2090": 1, "MED-2091": 1, "MED-2092": 1, "MED-2093": 1, "MED-2094": 1, "MED-2095": 1, "MED-2096": 1, "MED-2097": 1, "MED-2140": 1, "MED-2141": 1, "MED-2143": 1, "MED-2144": 1, "MED-2145": 1, "MED-2146": 1, "MED-2147": 1, "MED-2148": 1, "MED-2149": 1, "MED-2150": 1, "MED-4646": 1, "MED-2152": 1, "MED-2153": 1, "MED-2211": 1, "MED-2212": 1, "MED-2213": 1, "MED-2214": 1, "MED-2215": 1, "MED-2216": 1, "MED-2217": 1, "MED-2218": 1, "MED-2248": 1, "MED-2249": 1, "MED-2250": 1, "MED-2251": 1, "MED-2252": 1, "MED-2253": 1, "MED-2254": 1, "MED-2255": 1, "MED-2256": 1, "MED-2257": 1, "MED-2258": 1, "MED-2259": 1, "MED-2260": 1, "MED-2261": 1, "MED-2262": 1, "MED-2263": 1, "MED-2264": 1, "MED-2288": 1, "MED-3137": 1, "MED-2290": 1, "MED-2291": 1, "MED-2292": 1, "MED-2293": 1, "MED-2294": 1, "MED-2295": 1, "MED-2296": 1, "MED-2313": 1, "MED-2810": 1, "MED-2315": 1, "MED-2322": 1, "MED-2414": 1, "MED-4070": 1, "MED-2416": 1, "MED-2417": 1, "MED-2418": 1, "MED-3498": 1, "MED-2420": 1, "MED-2421": 1, "MED-2422": 1, "MED-2423": 1, "MED-2424": 1, "MED-4451": 1, "MED-2426": 1, "MED-2445": 1, "MED-2446": 1, "MED-2458": 1, "MED-2448": 1, "MED-2449": 1, "MED-2450": 1, "MED-2451": 1, "MED-2452": 1, "MED-2453": 1, "MED-2472": 1, "MED-2488": 1, "MED-2517": 1, "MED-2501": 1, "MED-2506": 1, "MED-2512": 1, "MED-2572": 1, "MED-2577": 1, "MED-2578": 1, "MED-2579": 1, "MED-2544": 1, "MED-2581": 1, "MED-2546": 1, "MED-2584": 1, "MED-2582": 1, "MED-2820": 1, "MED-3000": 1, "MED-2819": 1, "MED-4523": 1, "MED-2967": 1, "MED-2969": 1, "MED-2971": 1, "MED-2979": 1, "MED-2982": 1, "MED-2983": 1, "MED-2984": 1, "MED-2989": 1, "MED-3109": 1, "MED-3098": 1, "MED-3099": 1, "MED-3100": 1, "MED-3112": 1, "MED-3102": 1, "MED-3138": 1, "MED-3139": 1, "MED-3140": 1, "MED-3141": 1, "MED-3142": 1, "MED-3143": 1, "MED-3136": 1, "MED-3123": 1, "MED-3127": 1, "MED-3129": 1, "MED-3130": 1, "MED-3132": 1, "MED-3135": 1, "MED-3156": 1, "MED-3157": 1, "MED-3158": 1, "MED-3450": 1, "MED-3451": 1, "MED-3161": 1, "MED-3454": 1, "MED-3163": 1, "MED-3164": 1, "MED-3165": 1, "MED-3166": 1, "MED-3466": 1, "MED-3168": 1, "MED-3232": 1, "MED-3235": 1, "MED-3270": 1, "MED-3271": 1, "MED-3272": 1, "MED-3273": 1, "MED-3274": 1, "MED-3275": 1, "MED-3276": 1, "MED-3277": 1, "MED-3278": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3282": 1, "MED-3283": 1, "MED-3369": 1, "MED-3348": 1, "MED-3371": 1, "MED-3350": 1, "MED-3351": 1, "MED-3352": 1, "MED-3353": 1, "MED-3354": 1, "MED-3355": 1, "MED-3356": 1, "MED-3373": 1, "MED-3358": 1, "MED-3359": 1, "MED-3370": 1, "MED-3372": 1, "MED-3374": 1, "MED-3375": 1, "MED-3376": 1, "MED-3377": 1, "MED-3422": 1, "MED-3425": 1, "MED-3426": 1, "MED-3428": 1, "MED-3407": 1, "MED-3429": 1, "MED-3433": 1, "MED-3434": 1, "MED-3435": 1, "MED-3437": 1, "MED-3438": 1, "MED-3417": 1, "MED-3439": 1, "MED-3440": 1, "MED-3420": 1, "MED-3421": 1, "MED-3548": 1, "MED-3549": 1, "MED-3550": 1, "MED-3551": 1, "MED-3552": 1, "MED-3553": 1, "MED-3554": 1, "MED-3555": 1, "MED-3580": 1, "MED-3581": 1, "MED-3582": 1, "MED-3583": 1, "MED-3584": 1, "MED-3769": 1, "MED-3770": 1, "MED-3780": 1, "MED-3781": 1, "MED-3782": 1, "MED-3783": 1, "MED-3784": 1, "MED-3785": 1, "MED-3786": 1, "MED-3787": 1, "MED-3788": 1, "MED-3789": 1, "MED-3790": 1, "MED-3815": 1, "MED-3816": 1, "MED-3817": 1, "MED-3818": 1, "MED-3819": 1, "MED-3820": 1, "MED-3821": 1, "MED-3822": 1, "MED-3858": 1, "MED-4094": 1, "MED-3860": 1, "MED-4313": 1, "MED-3862": 1, "MED-3925": 1, "MED-3926": 1, "MED-3927": 1, "MED-3928": 1, "MED-3929": 1, "MED-3930": 1, "MED-3931": 1, "MED-3932": 1, "MED-3933": 1, "MED-4106": 1, "MED-4165": 1, "MED-4216": 1, "MED-4211": 1, "MED-4212": 1, "MED-4886": 1, "MED-5176": 1, "MED-4230": 1, "MED-4231": 1, "MED-4232": 1, "MED-4233": 1, "MED-4234": 1, "MED-4245": 1, "MED-4246": 1, "MED-4247": 1, "MED-4255": 1, "MED-4281": 1, "MED-4314": 1, "MED-4315": 1, "MED-4320": 1, "MED-4347": 1, "MED-4627": 1, "MED-4349": 1, "MED-4350": 1, "MED-4406": 1, "MED-4509": 1, "MED-4510": 1, "MED-4581": 1, "MED-4582": 1, "MED-4583": 1, "MED-4600": 1, "MED-4686": 1, "MED-4689": 1, "MED-4727": 1, "MED-4728": 1, "MED-4771": 1, "MED-4863": 1, "MED-4952": 1, "MED-4953": 1, "MED-4954": 1, "MED-4991": 1, "MED-4992": 1, "MED-5002": 1, "MED-5003": 1, "MED-5004": 1, "MED-5072": 1, "MED-5069": 1, "MED-5070": 1, "MED-5071": 1, "MED-5078": 1, "MED-5079": 1, "MED-5080": 1, "MED-5111": 1, "MED-5159": 1, "MED-5160": 1, "MED-5161": 1, "MED-5162": 1}, "PLAIN-671": {"MED-4857": 1}, "PLAIN-681": {"MED-5017": 1}, "PLAIN-691": {"MED-1847": 1, "MED-1848": 1, "MED-1849": 1, "MED-1850": 1, "MED-1851": 1, "MED-1852": 1, "MED-2185": 1, "MED-2200": 1, "MED-2201": 1, "MED-2188": 1, "MED-2189": 1, "MED-2202": 1, "MED-2191": 1, "MED-2203": 1, "MED-2204": 1, "MED-2194": 1, "MED-2195": 1, "MED-2208": 1, "MED-2207": 1, "MED-2209": 1, "MED-2210": 1}, "PLAIN-701": {"MED-3714": 1, "MED-3720": 1, "MED-3721": 1, "MED-3715": 1, "MED-3716": 1, "MED-3717": 1, "MED-3718": 1, "MED-3719": 1}, "PLAIN-711": {"MED-398": 1, "MED-3200": 1, "MED-3201": 1, "MED-3202": 1, "MED-3203": 1, "MED-3209": 1, "MED-1682": 1, "MED-1683": 1, "MED-1701": 1, "MED-1685": 1, "MED-1686": 1, "MED-1687": 1, "MED-1688": 1, "MED-1689": 1, "MED-1690": 1, "MED-1691": 1, "MED-2077": 1, "MED-1693": 1, "MED-2078": 1, "MED-1695": 1, "MED-1696": 1, "MED-1697": 1, "MED-3841": 1, "MED-3843": 1, "MED-3849": 1, "MED-3830": 1, "MED-1825": 1, "MED-1826": 1, "MED-1827": 1, "MED-1828": 1, "MED-1829": 1, "MED-1830": 1, "MED-2076": 1, "MED-2079": 1, "MED-2080": 1, "MED-5293": 1, "MED-2082": 1, "MED-2083": 1, "MED-5303": 1, "MED-2085": 1, "MED-3199": 1, "MED-3204": 1, "MED-3205": 1, "MED-3206": 1, "MED-3207": 1, "MED-3208": 1, "MED-3210": 1, "MED-4352": 1, "MED-4353": 1, "MED-4649": 1, "MED-4650": 1, "MED-4651": 1, "MED-4776": 1, "MED-4777": 1, "MED-4778": 1, "MED-4779": 1, "MED-4780": 1}, "PLAIN-721": {"MED-1265": 1, "MED-1266": 1, "MED-1267": 1, "MED-1268": 1, "MED-1278": 1, "MED-1282": 1, "MED-1271": 1, "MED-1272": 1, "MED-1273": 1, "MED-1274": 1, "MED-1287": 1, "MED-1276": 1, "MED-1277": 1, "MED-1279": 1, "MED-1280": 1, "MED-1281": 1, "MED-1283": 1, "MED-1284": 1, "MED-1285": 1, "MED-4876": 1, "MED-1288": 1, "MED-1289": 1, "MED-1290": 1, "MED-5172": 1, "MED-5173": 1}, "PLAIN-731": {"MED-2988": 1, "MED-2980": 1, "MED-2990": 1, "MED-2986": 1, "MED-2985": 1, "MED-2987": 1, "MED-332": 1, "MED-3090": 1, "MED-334": 1, "MED-335": 1, "MED-3092": 1, "MED-3094": 1, "MED-3226": 1, "MED-3227": 1, "MED-3230": 1, "MED-3229": 1, "MED-3232": 1, "MED-3233": 1, "MED-4722": 1, "MED-3237": 1, "MED-1337": 1, "MED-1338": 1, "MED-1339": 1, "MED-1340": 1, "MED-1341": 1, "MED-1486": 1, "MED-1487": 1, "MED-1488": 1, "MED-1489": 1, "MED-1490": 1, "MED-2979": 1, "MED-4319": 1, "MED-2982": 1, "MED-2983": 1, "MED-2984": 1, "MED-2989": 1, "MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3089": 1, "MED-3091": 1, "MED-3093": 1, "MED-3095": 1, "MED-3096": 1, "MED-3228": 1, "MED-3231": 1, "MED-3235": 1, "MED-3236": 1, "MED-4984": 1, "MED-4985": 1, "MED-4988": 1, "MED-4987": 1, "MED-5145": 1}, "PLAIN-741": {"MED-3874": 1, "MED-3875": 1, "MED-3876": 1, "MED-3877": 1, "MED-3878": 1, "MED-4233": 1, "MED-5176": 1, "MED-4230": 1, "MED-4231": 1, "MED-4232": 1, "MED-4234": 1, "MED-4235": 1, "MED-4236": 1, "MED-4237": 1, "MED-4239": 1, "MED-5177": 1, "MED-5178": 1, "MED-5179": 1, "MED-5184": 1}, "PLAIN-751": {"MED-3136": 1, "MED-3123": 1, "MED-3138": 1, "MED-3143": 1, "MED-3137": 1, "MED-3132": 1, "MED-3141": 1}, "PLAIN-761": {"MED-3796": 1, "MED-3778": 1, "MED-3801": 1, "MED-3794": 1, "MED-666": 1, "MED-3779": 1}, "PLAIN-771": {"MED-3385": 1, "MED-3387": 1, "MED-3386": 1}, "PLAIN-782": {"MED-4991": 1, "MED-4992": 1}, "PLAIN-792": {"MED-4546": 1, "MED-4547": 1, "MED-4958": 1, "MED-4959": 1, "MED-4960": 1, "MED-4961": 1}, "PLAIN-806": {"MED-2513": 1, "MED-2502": 1, "MED-2498": 1, "MED-2504": 1, "MED-2505": 1, "MED-2507": 1, "MED-5239": 1, "MED-2509": 1, "MED-2519": 1, "MED-2510": 1, "MED-2511": 1, "MED-5237": 1, "MED-2517": 1, "MED-2518": 1, "MED-2520": 1, "MED-2521": 1, "MED-3050": 1, "MED-3038": 1, "MED-3052": 1, "MED-3053": 1, "MED-3054": 1, "MED-3044": 1, "MED-3046": 1, "MED-3058": 1, "MED-3059": 1, "MED-3060": 1, "MED-3273": 1, "MED-3276": 1, "MED-3283": 1, "MED-1711": 1, "MED-1712": 1, "MED-2763": 1, "MED-1714": 1, "MED-1715": 1, "MED-1716": 1, "MED-1717": 1, "MED-1718": 1, "MED-1719": 1, "MED-1720": 1, "MED-1721": 1, "MED-1722": 1, "MED-1723": 1, "MED-1724": 1, "MED-2127": 1, "MED-2128": 1, "MED-2129": 1, "MED-2130": 1, "MED-2132": 1, "MED-2134": 1, "MED-2135": 1, "MED-2136": 1, "MED-2137": 1, "MED-2138": 1, "MED-2139": 1, "MED-2147": 1, "MED-2148": 1, "MED-2149": 1, "MED-2324": 1, "MED-2325": 1, "MED-2326": 1, "MED-2327": 1, "MED-2328": 1, "MED-2335": 1, "MED-2330": 1, "MED-2331": 1, "MED-2441": 1, "MED-2442": 1, "MED-2472": 1, "MED-2444": 1, "MED-2501": 1, "MED-2506": 1, "MED-2512": 1, "MED-2514": 1, "MED-2943": 1, "MED-3051": 1, "MED-3055": 1, "MED-3056": 1, "MED-3057": 1, "MED-3270": 1, "MED-3271": 1, "MED-3272": 1, "MED-3274": 1, "MED-3275": 1, "MED-3277": 1, "MED-3278": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3282": 1, "MED-4120": 1, "MED-4121": 1, "MED-4124": 1, "MED-4321": 1, "MED-4322": 1, "MED-4323": 1, "MED-4996": 1}, "PLAIN-817": {"MED-4841": 1}, "PLAIN-827": {"MED-2568": 1, "MED-2988": 1, "MED-2559": 1, "MED-4319": 1, "MED-2570": 1, "MED-2573": 1, "MED-2574": 1, "MED-2585": 1, "MED-2599": 1, "MED-2607": 1, "MED-2602": 1, "MED-2603": 1, "MED-2605": 1, "MED-2604": 1, "MED-2606": 1, "MED-2608": 1, "MED-2816": 1, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-2649": 1, "MED-2675": 1, "MED-2676": 1, "MED-2677": 1, "MED-2678": 1, "MED-2679": 1, "MED-2992": 1, "MED-2994": 1, "MED-3781": 1, "MED-3789": 1, "MED-3790": 1, "MED-3782": 1, "MED-3762": 1, "MED-3763": 1, "MED-3764": 1, "MED-1139": 1, "MED-1140": 1, "MED-1167": 1, "MED-1142": 1, "MED-1143": 1, "MED-1144": 1, "MED-1181": 1, "MED-1146": 1, "MED-1147": 1, "MED-1763": 1, "MED-1592": 1, "MED-1593": 1, "MED-1594": 1, "MED-1595": 1, "MED-1596": 1, "MED-1597": 1, "MED-1598": 1, "MED-1599": 1, "MED-1600": 1, "MED-1601": 1, "MED-1602": 1, "MED-1603": 1, "MED-1604": 1, "MED-1605": 1, "MED-1606": 1, "MED-1607": 1, "MED-4476": 1, "MED-1811": 1, "MED-1812": 1, "MED-1813": 1, "MED-1814": 1, "MED-5190": 1, "MED-4823": 1, "MED-1817": 1, "MED-1818": 1, "MED-1819": 1, "MED-1820": 1, "MED-2086": 1, "MED-2087": 1, "MED-2088": 1, "MED-2099": 1, "MED-2100": 1, "MED-2101": 1, "MED-2102": 1, "MED-2103": 1, "MED-2529": 1, "MED-2105": 1, "MED-2106": 1, "MED-2107": 1, "MED-2108": 1, "MED-2185": 1, "MED-2200": 1, "MED-2201": 1, "MED-2188": 1, "MED-2189": 1, "MED-2202": 1, "MED-2191": 1, "MED-2203": 1, "MED-2204": 1, "MED-2194": 1, "MED-2195": 1, "MED-2208": 1, "MED-2207": 1, "MED-2209": 1, "MED-2210": 1, "MED-2414": 1, "MED-4070": 1, "MED-2416": 1, "MED-2417": 1, "MED-2418": 1, "MED-3498": 1, "MED-2420": 1, "MED-2421": 1, "MED-2422": 1, "MED-2423": 1, "MED-2424": 1, "MED-4451": 1, "MED-2426": 1, "MED-2441": 1, "MED-2442": 1, "MED-2472": 1, "MED-2444": 1, "MED-2490": 1, "MED-2491": 1, "MED-2492": 1, "MED-2571": 1, "MED-2572": 1, "MED-2575": 1, "MED-2577": 1, "MED-2578": 1, "MED-2579": 1, "MED-2580": 1, "MED-2581": 1, "MED-2583": 1, "MED-2584": 1, "MED-2598": 1, "MED-2601": 1, "MED-2812": 1, "MED-2644": 1, "MED-2646": 1, "MED-2647": 1, "MED-2648": 1, "MED-2650": 1, "MED-2651": 1, "MED-2652": 1, "MED-2653": 1, "MED-2654": 1, "MED-2655": 1, "MED-2659": 1, "MED-2661": 1, "MED-2662": 1, "MED-2674": 1, "MED-2993": 1, "MED-3787": 1, "MED-3785": 1, "MED-3628": 1, "MED-3629": 1, "MED-3630": 1, "MED-3631": 1, "MED-3632": 1, "MED-3633": 1, "MED-3634": 1, "MED-3635": 1, "MED-3636": 1, "MED-4037": 1, "MED-4038": 1, "MED-4976": 1, "MED-4040": 1, "MED-5197": 1, "MED-4049": 1, "MED-4053": 1, "MED-4051": 1, "MED-5116": 1, "MED-4047": 1, "MED-4055": 1, "MED-4050": 1, "MED-4058": 1, "MED-4060": 1, "MED-4057": 1, "MED-4059": 1, "MED-4075": 1, "MED-4068": 1, "MED-4069": 1, "MED-4072": 1, "MED-4071": 1, "MED-4073": 1, "MED-4092": 1, "MED-4093": 1, "MED-4194": 1, "MED-4195": 1, "MED-4196": 1, "MED-4197": 1, "MED-4332": 1, "MED-4436": 1, "MED-4450": 1, "MED-4468": 1, "MED-4469": 1, "MED-4470": 1, "MED-4471": 1, "MED-4472": 1, "MED-4474": 1, "MED-4475": 1, "MED-4477": 1, "MED-4488": 1, "MED-4479": 1, "MED-4480": 1, "MED-4481": 1, "MED-4482": 1, "MED-4483": 1, "MED-4484": 1, "MED-4485": 1, "MED-4486": 1, "MED-4487": 1, "MED-4489": 1, "MED-4490": 1, "MED-4491": 1, "MED-4492": 1, "MED-4546": 1, "MED-4547": 1, "MED-4548": 1, "MED-4618": 1, "MED-4726": 1, "MED-4818": 1, "MED-4819": 1, "MED-4820": 1, "MED-4821": 1, "MED-4865": 1, "MED-4925": 1, "MED-4924": 1, "MED-4972": 1, "MED-4973": 1, "MED-4974": 1, "MED-4975": 1, "MED-4977": 1, "MED-4978": 1, "MED-5002": 1, "MED-5019": 1, "MED-5020": 1, "MED-5036": 1, "MED-5085": 1, "MED-5086": 1, "MED-5087": 1, "MED-5088": 1, "MED-5089": 1, "MED-5122": 1}, "PLAIN-838": {"MED-3494": 1, "MED-3501": 1, "MED-3496": 1, "MED-3497": 1, "MED-3502": 1, "MED-3500": 1, "MED-3495": 1, "MED-3498": 1, "MED-3499": 1}, "PLAIN-850": {"MED-3103": 1, "MED-3107": 1, "MED-3106": 1, "MED-3108": 1, "MED-3109": 1, "MED-3112": 1, "MED-3238": 1, "MED-3241": 1, "MED-3242": 1, "MED-3244": 1, "MED-3790": 1, "MED-3245": 1, "MED-5337": 1, "MED-3369": 1, "MED-3370": 1, "MED-3372": 1, "MED-3376": 1, "MED-3377": 1, "MED-717": 1, "MED-718": 1, "MED-719": 1, "MED-720": 1, "MED-721": 1, "MED-722": 1, "MED-723": 1, "MED-724": 1, "MED-2064": 1, "MED-2065": 1, "MED-2066": 1, "MED-2067": 1, "MED-2068": 1, "MED-2069": 1, "MED-2070": 1, "MED-2071": 1, "MED-4454": 1, "MED-2073": 1, "MED-2074": 1, "MED-2075": 1, "MED-2098": 1, "MED-3709": 1, "MED-3105": 1, "MED-3110": 1, "MED-3111": 1, "MED-3697": 1, "MED-3699": 1, "MED-3243": 1, "MED-3371": 1, "MED-3373": 1, "MED-3374": 1, "MED-3375": 1, "MED-4391": 1, "MED-4392": 1, "MED-4393": 1, "MED-4455": 1, "MED-4523": 1, "MED-4524": 1, "MED-4859": 1}, "PLAIN-872": {"MED-4650": 1}, "PLAIN-882": {"MED-3617": 1, "MED-3620": 1, "MED-3622": 1, "MED-3631": 1, "MED-3632": 1, "MED-3629": 1, "MED-3634": 1, "MED-3618": 1, "MED-3621": 1, "MED-3615": 1, "MED-3628": 1, "MED-3630": 1, "MED-3633": 1, "MED-3635": 1, "MED-3636": 1}, "PLAIN-892": {"MED-2988": 1, "MED-2577": 1, "MED-2578": 1, "MED-2572": 1, "MED-2581": 1, "MED-2582": 1, "MED-2584": 1, "MED-2824": 1, "MED-2815": 1, "MED-2794": 1, "MED-2816": 1, "MED-2797": 1, "MED-2809": 1, "MED-2799": 1, "MED-2800": 1, "MED-2801": 1, "MED-2802": 1, "MED-2803": 1, "MED-2804": 1, "MED-2805": 1, "MED-2807": 1, "MED-2980": 1, "MED-2990": 1, "MED-2986": 1, "MED-2985": 1, "MED-2987": 1, "MED-3137": 1, "MED-3130": 1, "MED-3139": 1, "MED-3140": 1, "MED-3127": 1, "MED-3142": 1, "MED-3135": 1, "MED-3129": 1, "MED-3136": 1, "MED-3138": 1, "MED-3141": 1, "MED-3584": 1, "MED-1880": 1, "MED-1881": 1, "MED-1942": 1, "MED-2786": 1, "MED-1944": 1, "MED-2217": 1, "MED-1947": 1, "MED-1948": 1, "MED-2008": 1, "MED-2009": 1, "MED-2010": 1, "MED-2011": 1, "MED-2140": 1, "MED-2141": 1, "MED-4319": 1, "MED-2143": 1, "MED-2144": 1, "MED-2145": 1, "MED-2146": 1, "MED-2147": 1, "MED-2148": 1, "MED-2149": 1, "MED-2568": 1, "MED-2570": 1, "MED-2571": 1, "MED-2573": 1, "MED-2574": 1, "MED-2575": 1, "MED-2579": 1, "MED-2580": 1, "MED-2583": 1, "MED-2585": 1, "MED-2820": 1, "MED-2819": 1, "MED-2979": 1, "MED-2982": 1, "MED-2983": 1, "MED-2984": 1, "MED-2989": 1, "MED-3143": 1, "MED-3123": 1, "MED-3132": 1, "MED-3580": 1, "MED-3581": 1, "MED-3582": 1, "MED-3583": 1, "MED-4149": 1, "MED-4316": 1, "MED-4317": 1, "MED-4318": 1, "MED-4347": 1, "MED-4627": 1, "MED-4349": 1, "MED-4350": 1}, "PLAIN-902": {"MED-4194": 1, "MED-4195": 1, "MED-4196": 1, "MED-4197": 1}, "PLAIN-913": {"MED-2827": 1, "MED-2830": 1, "MED-2826": 1, "MED-3545": 1, "MED-3173": 1, "MED-3541": 1, "MED-3546": 1, "MED-3542": 1, "MED-3543": 1, "MED-3547": 1, "MED-3811": 1, "MED-759": 1, "MED-760": 1, "MED-2831": 1, "MED-3540": 1, "MED-3544": 1, "MED-3812": 1, "MED-3813": 1, "MED-4389": 1, "MED-4415": 1, "MED-4416": 1, "MED-4417": 1, "MED-4418": 1, "MED-4539": 1, "MED-4540": 1, "MED-4541": 1, "MED-4998": 1, "MED-4999": 1, "MED-5000": 1, "MED-5001": 1, "MED-5052": 1, "MED-5083": 1, "MED-5084": 1, "MED-5156": 1}, "PLAIN-924": {"MED-2586": 1, "MED-2587": 1, "MED-2588": 1, "MED-2591": 1, "MED-3050": 1, "MED-3038": 1, "MED-3052": 1, "MED-3053": 1, "MED-3054": 1, "MED-3044": 1, "MED-3046": 1, "MED-3058": 1, "MED-3059": 1, "MED-3060": 1, "MED-3057": 1, "MED-3055": 1, "MED-3545": 1, "MED-3173": 1, "MED-3541": 1, "MED-3546": 1, "MED-3542": 1, "MED-3543": 1, "MED-3547": 1, "MED-2589": 1, "MED-2590": 1, "MED-3051": 1, "MED-3056": 1, "MED-3540": 1, "MED-3544": 1, "MED-4539": 1, "MED-4540": 1, "MED-4541": 1}, "PLAIN-934": {"MED-2675": 1, "MED-2676": 1, "MED-2677": 1, "MED-2678": 1, "MED-2679": 1, "MED-3639": 1, "MED-1621": 1, "MED-1622": 1, "MED-1623": 1, "MED-1624": 1, "MED-1625": 1, "MED-1626": 1, "MED-1627": 1, "MED-1628": 1, "MED-5054": 1, "MED-1630": 1, "MED-1631": 1, "MED-1634": 1, "MED-1635": 1, "MED-1636": 1, "MED-1637": 1, "MED-1638": 1, "MED-1639": 1, "MED-1640": 1, "MED-1641": 1, "MED-1642": 1, "MED-1643": 1, "MED-5258": 1, "MED-1645": 1, "MED-1646": 1, "MED-1647": 1, "MED-1648": 1, "MED-1649": 1, "MED-2154": 1, "MED-2155": 1, "MED-2156": 1, "MED-2157": 1, "MED-2158": 1, "MED-2159": 1, "MED-2160": 1, "MED-2161": 1, "MED-2162": 1, "MED-2163": 1, "MED-2164": 1, "MED-2165": 1, "MED-2166": 1, "MED-2167": 1, "MED-2168": 1, "MED-2169": 1, "MED-2170": 1, "MED-2171": 1, "MED-2172": 1, "MED-2173": 1, "MED-2174": 1, "MED-2175": 1, "MED-2176": 1, "MED-2177": 1, "MED-2178": 1, "MED-2674": 1, "MED-3637": 1, "MED-3638": 1, "MED-3640": 1, "MED-3641": 1, "MED-3654": 1, "MED-3655": 1, "MED-3656": 1, "MED-3657": 1, "MED-3796": 1, "MED-3778": 1, "MED-3779": 1, "MED-3925": 1, "MED-3926": 1, "MED-3927": 1, "MED-3928": 1, "MED-3929": 1, "MED-3930": 1, "MED-3931": 1, "MED-3932": 1, "MED-3933": 1, "MED-4040": 1, "MED-4976": 1, "MED-5197": 1, "MED-4156": 1, "MED-4316": 1, "MED-4317": 1, "MED-4318": 1, "MED-4438": 1, "MED-4439": 1, "MED-4774": 1, "MED-4857": 1, "MED-4877": 1, "MED-4878": 1, "MED-4879": 1, "MED-4973": 1, "MED-4974": 1, "MED-4975": 1, "MED-5047": 1, "MED-5048": 1, "MED-5049": 1, "MED-5050": 1, "MED-5123": 1}, "PLAIN-946": {"MED-2527": 1, "MED-3314": 1, "MED-3306": 1, "MED-3307": 1, "MED-3316": 1, "MED-3311": 1, "MED-3292": 1, "MED-3317": 1, "MED-3294": 1, "MED-3320": 1, "MED-3295": 1, "MED-3321": 1, "MED-2524": 1, "MED-2525": 1, "MED-2526": 1, "MED-2528": 1, "MED-2529": 1, "MED-2530": 1, "MED-3305": 1, "MED-3308": 1, "MED-3288": 1, "MED-3310": 1, "MED-3312": 1, "MED-3313": 1, "MED-3315": 1, "MED-4433": 1, "MED-3318": 1, "MED-3302": 1, "MED-4380": 1, "MED-4381": 1, "MED-4713": 1}, "PLAIN-956": {"MED-2675": 1, "MED-2676": 1, "MED-2677": 1, "MED-2678": 1, "MED-2679": 1, "MED-2736": 1, "MED-5169": 1, "MED-2745": 1, "MED-2742": 1, "MED-2827": 1, "MED-2830": 1, "MED-2826": 1, "MED-2896": 1, "MED-2899": 1, "MED-2900": 1, "MED-2901": 1, "MED-3137": 1, "MED-3146": 1, "MED-3145": 1, "MED-3178": 1, "MED-3169": 1, "MED-3172": 1, "MED-3181": 1, "MED-3174": 1, "MED-3175": 1, "MED-3176": 1, "MED-3182": 1, "MED-3369": 1, "MED-3370": 1, "MED-3372": 1, "MED-3376": 1, "MED-3377": 1, "MED-1157": 1, "MED-1158": 1, "MED-2494": 1, "MED-1160": 1, "MED-1161": 1, "MED-1162": 1, "MED-1181": 1, "MED-1164": 1, "MED-1165": 1, "MED-1568": 1, "MED-1569": 1, "MED-1570": 1, "MED-1571": 1, "MED-1572": 1, "MED-1573": 1, "MED-1880": 1, "MED-1881": 1, "MED-2064": 1, "MED-2065": 1, "MED-2066": 1, "MED-2067": 1, "MED-2068": 1, "MED-2069": 1, "MED-2070": 1, "MED-2071": 1, "MED-4454": 1, "MED-2073": 1, "MED-2074": 1, "MED-2075": 1, "MED-2098": 1, "MED-2179": 1, "MED-2180": 1, "MED-2181": 1, "MED-2182": 1, "MED-2183": 1, "MED-2184": 1, "MED-2185": 1, "MED-2200": 1, "MED-2201": 1, "MED-2188": 1, "MED-2189": 1, "MED-2202": 1, "MED-2191": 1, "MED-2203": 1, "MED-2204": 1, "MED-2194": 1, "MED-2195": 1, "MED-2208": 1, "MED-2207": 1, "MED-2209": 1, "MED-2210": 1, "MED-2414": 1, "MED-4070": 1, "MED-2416": 1, "MED-2417": 1, "MED-2418": 1, "MED-3498": 1, "MED-2420": 1, "MED-2421": 1, "MED-2422": 1, "MED-2423": 1, "MED-2424": 1, "MED-4451": 1, "MED-2426": 1, "MED-2490": 1, "MED-2491": 1, "MED-2492": 1, "MED-2674": 1, "MED-4131": 1, "MED-2738": 1, "MED-2740": 1, "MED-2741": 1, "MED-2744": 1, "MED-2831": 1, "MED-4384": 1, "MED-2898": 1, "MED-3138": 1, "MED-3139": 1, "MED-3140": 1, "MED-3141": 1, "MED-3142": 1, "MED-3143": 1, "MED-3144": 1, "MED-3170": 1, "MED-3171": 1, "MED-3173": 1, "MED-3177": 1, "MED-3179": 1, "MED-3180": 1, "MED-3348": 1, "MED-3371": 1, "MED-3350": 1, "MED-3351": 1, "MED-3352": 1, "MED-3353": 1, "MED-3354": 1, "MED-3355": 1, "MED-3356": 1, "MED-3373": 1, "MED-3358": 1, "MED-3359": 1, "MED-3374": 1, "MED-3375": 1, "MED-3628": 1, "MED-3629": 1, "MED-3630": 1, "MED-3631": 1, "MED-3632": 1, "MED-3633": 1, "MED-3634": 1, "MED-3635": 1, "MED-3636": 1, "MED-3648": 1, "MED-3649": 1, "MED-3651": 1, "MED-3652": 1, "MED-3653": 1, "MED-3880": 1, "MED-4132": 1, "MED-3882": 1, "MED-4136": 1, "MED-3884": 1, "MED-3885": 1, "MED-3886": 1, "MED-3887": 1, "MED-3888": 1, "MED-3889": 1, "MED-3890": 1, "MED-3891": 1, "MED-3892": 1, "MED-3982": 1, "MED-3983": 1, "MED-3984": 1, "MED-4037": 1, "MED-4038": 1, "MED-4976": 1, "MED-4040": 1, "MED-5197": 1, "MED-4049": 1, "MED-4053": 1, "MED-4051": 1, "MED-5116": 1, "MED-4047": 1, "MED-4055": 1, "MED-4050": 1, "MED-4058": 1, "MED-4060": 1, "MED-4057": 1, "MED-4059": 1, "MED-4075": 1, "MED-4068": 1, "MED-4069": 1, "MED-4072": 1, "MED-4071": 1, "MED-4073": 1, "MED-4332": 1, "MED-4436": 1, "MED-4450": 1, "MED-4452": 1, "MED-4553": 1, "MED-4596": 1, "MED-4858": 1, "MED-4859": 1, "MED-4908": 1, "MED-4909": 1, "MED-4973": 1, "MED-4974": 1, "MED-4975": 1, "MED-4978": 1, "MED-4991": 1, "MED-4992": 1, "MED-5074": 1, "MED-5075": 1, "MED-5076": 1}, "PLAIN-966": {"MED-4103": 1, "MED-4104": 1, "MED-4853": 1}, "PLAIN-977": {"MED-4399": 1, "MED-4400": 1, "MED-4402": 1}, "PLAIN-987": {"MED-2830": 1, "MED-735": 1, "MED-2831": 1}, "PLAIN-997": {"MED-4184": 1, "MED-5140": 1}, "PLAIN-1008": {"MED-4532": 1, "MED-4533": 1, "MED-4534": 1, "MED-4535": 1, "MED-4536": 1}, "PLAIN-1018": {"MED-2750": 1, "MED-2751": 1, "MED-2752": 1, "MED-2754": 1, "MED-2755": 1, "MED-3028": 1, "MED-2913": 1, "MED-3021": 1, "MED-3012": 1, "MED-3033": 1, "MED-2906": 1, "MED-3034": 1, "MED-1831": 1, "MED-1832": 1, "MED-1833": 1, "MED-1834": 1, "MED-5092": 1, "MED-2753": 1, "MED-2756": 1, "MED-3023": 1, "MED-2904": 1, "MED-2905": 1, "MED-2907": 1, "MED-3024": 1, "MED-3025": 1, "MED-2910": 1, "MED-3027": 1, "MED-3013": 1, "MED-3030": 1, "MED-2917": 1, "MED-3035": 1, "MED-2921": 1, "MED-3019": 1, "MED-3022": 1, "MED-3026": 1, "MED-3029": 1, "MED-3032": 1, "MED-4551": 1, "MED-4936": 1, "MED-4730": 1, "MED-4632": 1, "MED-4633": 1, "MED-4634": 1, "MED-4729": 1, "MED-4731": 1, "MED-4732": 1, "MED-4933": 1, "MED-4934": 1, "MED-4935": 1, "MED-4937": 1, "MED-4943": 1, "MED-5091": 1, "MED-5093": 1, "MED-5097": 1, "MED-5098": 1, "MED-5099": 1, "MED-5100": 1, "MED-5101": 1, "MED-5102": 1, "MED-5104": 1}, "PLAIN-1028": {"MED-4686": 1}, "PLAIN-1039": {"MED-4380": 1, "MED-4381": 1}, "PLAIN-1050": {"MED-2763": 1, "MED-3780": 1, "MED-3784": 1, "MED-3242": 1, "MED-3244": 1, "MED-3790": 1, "MED-3245": 1, "MED-5337": 1, "MED-3250": 1, "MED-3251": 1, "MED-3247": 1, "MED-3248": 1, "MED-3249": 1, "MED-3254": 1, "MED-3253": 1, "MED-1399": 1, "MED-1393": 1, "MED-1394": 1, "MED-1395": 1, "MED-1489": 1, "MED-1397": 1, "MED-1398": 1, "MED-1400": 1, "MED-1476": 1, "MED-2432": 1, "MED-1478": 1, "MED-1479": 1, "MED-1486": 1, "MED-1487": 1, "MED-1488": 1, "MED-1490": 1, "MED-1914": 1, "MED-1915": 1, "MED-1916": 1, "MED-1917": 1, "MED-1918": 1, "MED-1919": 1, "MED-1920": 1, "MED-1921": 1, "MED-1922": 1, "MED-1923": 1, "MED-1924": 1, "MED-4877": 1, "MED-1926": 1, "MED-4878": 1, "MED-1928": 1, "MED-1929": 1, "MED-1930": 1, "MED-1931": 1, "MED-1932": 1, "MED-1933": 1, "MED-1934": 1, "MED-1935": 1, "MED-1936": 1, "MED-2109": 1, "MED-2110": 1, "MED-2111": 1, "MED-2112": 1, "MED-4255": 1, "MED-3113": 1, "MED-4247": 1, "MED-3786": 1, "MED-3781": 1, "MED-3789": 1, "MED-3193": 1, "MED-3783": 1, "MED-3785": 1, "MED-3243": 1, "MED-3252": 1, "MED-3255": 1, "MED-3722": 1, "MED-3723": 1, "MED-3724": 1, "MED-3744": 1, "MED-3726": 1, "MED-4481": 1, "MED-3728": 1, "MED-3729": 1, "MED-3730": 1, "MED-3731": 1, "MED-3732": 1, "MED-4245": 1, "MED-4246": 1, "MED-4613": 1, "MED-4617": 1, "MED-4615": 1, "MED-4616": 1, "MED-4696": 1, "MED-4879": 1, "MED-4886": 1, "MED-4887": 1, "MED-4888": 1, "MED-4890": 1, "MED-4891": 1}, "PLAIN-1066": {"MED-4000": 1, "MED-4002": 1, "MED-5016": 1, "MED-4005": 1, "MED-4007": 1, "MED-4001": 1, "MED-4004": 1, "MED-4006": 1, "MED-4600": 1}, "PLAIN-1088": {"MED-2109": 1, "MED-2110": 1, "MED-2111": 1, "MED-2112": 1, "MED-4255": 1}, "PLAIN-1098": {"MED-3806": 1, "MED-3807": 1, "MED-3808": 1, "MED-3809": 1, "MED-3810": 1}, "PLAIN-1109": {"MED-2494": 1, "MED-2495": 1, "MED-2496": 1, "MED-2493": 1, "MED-2497": 1, "MED-2644": 1, "MED-2646": 1, "MED-2651": 1, "MED-118": 1, "MED-2652": 1, "MED-2653": 1, "MED-2655": 1, "MED-2659": 1, "MED-2661": 1, "MED-2662": 1, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-2649": 1, "MED-1098": 1, "MED-1099": 1, "MED-1100": 1, "MED-1101": 1, "MED-3590": 1, "MED-1763": 1, "MED-1592": 1, "MED-1593": 1, "MED-1594": 1, "MED-1595": 1, "MED-1596": 1, "MED-1597": 1, "MED-2127": 1, "MED-2128": 1, "MED-2129": 1, "MED-2130": 1, "MED-2507": 1, "MED-2132": 1, "MED-2517": 1, "MED-2134": 1, "MED-2135": 1, "MED-2136": 1, "MED-2137": 1, "MED-2138": 1, "MED-2139": 1, "MED-2461": 1, "MED-2464": 1, "MED-2468": 1, "MED-2469": 1, "MED-2471": 1, "MED-2472": 1, "MED-2474": 1, "MED-2475": 1, "MED-2476": 1, "MED-2479": 1, "MED-2482": 1, "MED-2484": 1, "MED-3687": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2650": 1, "MED-2654": 1, "MED-3585": 1, "MED-3586": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3591": 1, "MED-3592": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-3665": 1, "MED-3666": 1, "MED-3667": 1, "MED-3668": 1, "MED-3669": 1, "MED-3670": 1, "MED-3954": 1, "MED-3955": 1, "MED-3956": 1, "MED-4680": 1, "MED-3958": 1, "MED-3959": 1, "MED-4933": 1, "MED-4934": 1, "MED-4935": 1, "MED-4936": 1, "MED-4937": 1, "MED-4950": 1, "MED-4951": 1, "MED-5104": 1, "MED-5105": 1}, "PLAIN-1119": {"MED-3788": 1, "MED-3780": 1, "MED-3781": 1, "MED-3782": 1, "MED-3783": 1, "MED-3784": 1, "MED-3785": 1, "MED-3786": 1, "MED-3787": 1, "MED-3789": 1, "MED-3790": 1}, "PLAIN-1130": {"MED-4165": 1}, "PLAIN-1141": {"MED-1496": 1, "MED-1497": 1, "MED-1498": 1, "MED-1499": 1, "MED-1500": 1, "MED-1501": 1, "MED-1502": 1, "MED-1503": 1, "MED-1504": 1, "MED-1505": 1, "MED-1506": 1, "MED-2109": 1, "MED-2110": 1, "MED-2111": 1, "MED-2112": 1, "MED-4255": 1}, "PLAIN-1151": {"MED-2700": 1, "MED-2695": 1, "MED-2697": 1, "MED-2703": 1, "MED-2698": 1, "MED-2699": 1, "MED-2705": 1, "MED-4131": 1, "MED-2740": 1, "MED-2741": 1, "MED-2769": 1, "MED-2770": 1, "MED-2771": 1, "MED-2772": 1, "MED-2774": 1, "MED-2775": 1, "MED-3886": 1, "MED-3882": 1, "MED-3889": 1, "MED-3891": 1, "MED-3887": 1, "MED-729": 1, "MED-730": 1, "MED-731": 1, "MED-732": 1, "MED-1215": 1, "MED-1216": 1, "MED-1217": 1, "MED-1218": 1, "MED-1219": 1, "MED-1220": 1, "MED-1221": 1, "MED-4797": 1, "MED-1755": 1, "MED-1978": 1, "MED-1757": 1, "MED-1981": 1, "MED-1759": 1, "MED-1956": 1, "MED-1957": 1, "MED-1958": 1, "MED-1959": 1, "MED-1960": 1, "MED-1961": 1, "MED-1962": 1, "MED-1963": 1, "MED-1977": 1, "MED-1979": 1, "MED-1980": 1, "MED-1982": 1, "MED-1983": 1, "MED-2086": 1, "MED-2087": 1, "MED-2088": 1, "MED-2488": 1, "MED-2489": 1, "MED-2693": 1, "MED-2694": 1, "MED-2696": 1, "MED-3729": 1, "MED-2702": 1, "MED-2704": 1, "MED-2736": 1, "MED-2738": 1, "MED-5169": 1, "MED-2742": 1, "MED-2743": 1, "MED-2744": 1, "MED-2745": 1, "MED-2746": 1, "MED-2773": 1, "MED-3880": 1, "MED-4132": 1, "MED-3888": 1, "MED-4136": 1, "MED-3892": 1, "MED-3884": 1, "MED-3567": 1, "MED-3885": 1, "MED-3570": 1, "MED-3572": 1, "MED-3890": 1, "MED-3577": 1, "MED-3648": 1, "MED-3649": 1, "MED-3651": 1, "MED-3652": 1, "MED-3653": 1, "MED-4128": 1, "MED-4129": 1, "MED-4130": 1, "MED-4133": 1, "MED-4134": 1, "MED-4135": 1, "MED-4137": 1, "MED-4138": 1, "MED-4139": 1, "MED-4140": 1, "MED-4141": 1, "MED-4142": 1, "MED-4143": 1, "MED-4144": 1, "MED-4145": 1, "MED-4146": 1, "MED-4147": 1, "MED-4148": 1, "MED-4436": 1, "MED-4594": 1, "MED-4593": 1, "MED-4726": 1, "MED-4746": 1, "MED-4747": 1, "MED-4748": 1, "MED-4749": 1, "MED-4750": 1, "MED-4751": 1, "MED-4752": 1, "MED-4790": 1, "MED-4791": 1, "MED-4792": 1, "MED-4804": 1, "MED-4794": 1, "MED-4795": 1, "MED-4796": 1, "MED-4798": 1, "MED-4799": 1, "MED-4800": 1, "MED-4801": 1, "MED-4802": 1, "MED-4803": 1, "MED-4805": 1, "MED-4806": 1, "MED-4807": 1, "MED-4808": 1, "MED-4809": 1, "MED-4896": 1, "MED-4897": 1, "MED-4898": 1, "MED-4899": 1, "MED-4912": 1, "MED-4911": 1, "MED-4936": 1, "MED-4932": 1, "MED-4933": 1, "MED-4934": 1, "MED-4935": 1, "MED-4937": 1, "MED-4981": 1, "MED-4980": 1, "MED-5102": 1, "MED-5104": 1, "MED-5108": 1, "MED-5109": 1}, "PLAIN-1161": {"MED-4281": 1}, "PLAIN-1172": {"MED-3518": 1, "MED-3519": 1, "MED-3532": 1, "MED-3533": 1, "MED-3524": 1, "MED-3527": 1, "MED-3539": 1, "MED-3520": 1, "MED-3521": 1, "MED-3522": 1, "MED-3523": 1, "MED-3525": 1, "MED-3526": 1, "MED-3528": 1, "MED-3530": 1, "MED-3531": 1, "MED-3534": 1, "MED-3535": 1, "MED-4150": 1}, "PLAIN-1183": {"MED-1265": 1, "MED-1266": 1, "MED-1267": 1, "MED-1268": 1, "MED-1278": 1, "MED-1282": 1, "MED-1271": 1, "MED-1272": 1, "MED-1273": 1, "MED-1274": 1, "MED-1287": 1, "MED-1276": 1, "MED-4068": 1, "MED-4069": 1, "MED-4070": 1, "MED-4071": 1, "MED-4072": 1, "MED-4073": 1, "MED-5197": 1, "MED-4075": 1, "MED-4084": 1, "MED-4085": 1, "MED-4850": 1, "MED-4087": 1, "MED-4088": 1}, "PLAIN-1193": {"MED-4000": 1, "MED-4002": 1, "MED-5016": 1, "MED-4005": 1, "MED-4007": 1, "MED-4001": 1, "MED-4004": 1, "MED-4006": 1, "MED-4832": 1, "MED-4833": 1, "MED-4834": 1, "MED-4835": 1, "MED-5090": 1, "MED-5176": 1, "MED-5177": 1, "MED-5178": 1, "MED-5179": 1, "MED-5184": 1}, "PLAIN-1203": {"MED-1195": 1, "MED-1196": 1, "MED-5360": 1, "MED-1198": 1, "MED-1199": 1, "MED-1200": 1, "MED-1201": 1, "MED-1202": 1, "MED-1203": 1, "MED-4365": 1, "MED-4575": 1, "MED-4771": 1}, "PLAIN-1214": {"MED-2988": 1, "MED-2980": 1, "MED-2990": 1, "MED-2986": 1, "MED-2985": 1, "MED-2987": 1, "MED-1486": 1, "MED-1487": 1, "MED-1488": 1, "MED-1489": 1, "MED-1490": 1, "MED-2979": 1, "MED-4319": 1, "MED-2982": 1, "MED-2983": 1, "MED-2984": 1, "MED-2989": 1}, "PLAIN-1225": {"MED-1259": 1, "MED-2529": 1, "MED-1261": 1, "MED-1262": 1, "MED-1674": 1, "MED-1676": 1, "MED-1669": 1, "MED-1670": 1, "MED-1671": 1, "MED-1672": 1, "MED-1673": 1, "MED-1675": 1, "MED-1706": 1, "MED-1707": 1, "MED-1708": 1, "MED-1709": 1, "MED-1710": 1, "MED-2033": 1, "MED-2013": 1, "MED-2014": 1, "MED-2027": 1, "MED-2016": 1, "MED-2037": 1, "MED-4091": 1, "MED-4089": 1, "MED-5071": 1, "MED-5057": 1}, "PLAIN-1236": {"MED-1337": 1, "MED-1338": 1, "MED-1339": 1, "MED-1340": 1, "MED-1341": 1}, "PLAIN-1249": {"MED-4753": 1, "MED-4757": 1, "MED-4755": 1, "MED-4756": 1}, "PLAIN-1262": {"MED-2076": 1, "MED-2077": 1, "MED-2078": 1, "MED-2079": 1, "MED-2080": 1, "MED-5293": 1, "MED-2082": 1, "MED-2083": 1, "MED-5303": 1, "MED-2085": 1}, "PLAIN-1275": {"MED-2882": 1, "MED-2898": 1, "MED-2884": 1, "MED-2885": 1, "MED-2886": 1, "MED-2888": 1, "MED-2889": 1, "MED-2895": 1, "MED-2926": 1, "MED-2922": 1, "MED-3465": 1, "MED-2923": 1, "MED-2925": 1, "MED-3518": 1, "MED-3519": 1, "MED-3532": 1, "MED-3533": 1, "MED-3524": 1, "MED-3527": 1, "MED-3539": 1, "MED-3609": 1, "MED-3610": 1, "MED-4544": 1, "MED-2899": 1, "MED-2890": 1, "MED-2891": 1, "MED-2892": 1, "MED-2893": 1, "MED-2894": 1, "MED-2924": 1, "MED-2928": 1, "MED-3520": 1, "MED-3521": 1, "MED-3522": 1, "MED-3523": 1, "MED-3525": 1, "MED-3526": 1, "MED-3528": 1, "MED-3530": 1, "MED-3531": 1, "MED-3534": 1, "MED-3535": 1, "MED-3619": 1, "MED-3601": 1, "MED-3602": 1, "MED-3603": 1, "MED-3604": 1, "MED-3615": 1, "MED-3606": 1, "MED-3607": 1, "MED-3608": 1, "MED-4545": 1, "MED-4712": 1, "MED-5083": 1, "MED-5084": 1}, "PLAIN-1288": {"MED-2663": 1, "MED-2664": 1, "MED-2665": 1, "MED-4860": 1, "MED-2668": 1, "MED-2670": 1, "MED-2669": 1, "MED-2797": 1, "MED-2809": 1, "MED-2799": 1, "MED-2800": 1, "MED-2801": 1, "MED-2802": 1, "MED-2803": 1, "MED-2804": 1, "MED-2805": 1, "MED-2807": 1, "MED-2890": 1, "MED-2891": 1, "MED-2892": 1, "MED-3545": 1, "MED-3173": 1, "MED-3541": 1, "MED-3546": 1, "MED-3542": 1, "MED-3543": 1, "MED-3547": 1, "MED-3697": 1, "MED-3698": 1, "MED-3699": 1, "MED-3742": 1, "MED-1375": 1, "MED-1362": 1, "MED-1363": 1, "MED-1383": 1, "MED-1365": 1, "MED-1366": 1, "MED-1643": 1, "MED-1374": 1, "MED-1380": 1, "MED-5301": 1, "MED-1371": 1, "MED-5271": 1, "MED-1373": 1, "MED-1533": 1, "MED-1534": 1, "MED-1535": 1, "MED-3900": 1, "MED-5081": 1, "MED-1538": 1, "MED-1634": 1, "MED-1635": 1, "MED-1636": 1, "MED-1637": 1, "MED-1638": 1, "MED-1639": 1, "MED-1640": 1, "MED-1641": 1, "MED-1642": 1, "MED-5258": 1, "MED-1645": 1, "MED-1646": 1, "MED-1647": 1, "MED-1648": 1, "MED-1649": 1, "MED-1789": 1, "MED-1790": 1, "MED-1791": 1, "MED-1792": 1, "MED-1793": 1, "MED-1794": 1, "MED-1795": 1, "MED-2127": 1, "MED-2128": 1, "MED-2129": 1, "MED-2130": 1, "MED-2507": 1, "MED-2132": 1, "MED-2517": 1, "MED-2134": 1, "MED-2135": 1, "MED-2136": 1, "MED-2137": 1, "MED-2138": 1, "MED-2139": 1, "MED-2272": 1, "MED-2273": 1, "MED-2274": 1, "MED-3464": 1, "MED-2276": 1, "MED-3535": 1, "MED-2278": 1, "MED-2324": 1, "MED-2325": 1, "MED-2326": 1, "MED-2327": 1, "MED-2328": 1, "MED-2335": 1, "MED-2330": 1, "MED-2331": 1, "MED-2332": 1, "MED-2333": 1, "MED-2823": 1, "MED-2336": 1, "MED-2445": 1, "MED-2446": 1, "MED-2458": 1, "MED-2448": 1, "MED-2449": 1, "MED-2450": 1, "MED-2451": 1, "MED-2452": 1, "MED-2453": 1, "MED-2472": 1, "MED-2667": 1, "MED-2819": 1, "MED-2882": 1, "MED-2898": 1, "MED-2893": 1, "MED-2884": 1, "MED-2885": 1, "MED-2886": 1, "MED-2899": 1, "MED-2888": 1, "MED-2889": 1, "MED-2894": 1, "MED-2895": 1, "MED-4523": 1, "MED-2966": 1, "MED-2967": 1, "MED-2951": 1, "MED-2968": 1, "MED-2969": 1, "MED-2970": 1, "MED-2971": 1, "MED-2972": 1, "MED-3540": 1, "MED-3544": 1, "MED-3696": 1, "MED-3700": 1, "MED-3701": 1, "MED-3743": 1, "MED-3744": 1, "MED-3745": 1, "MED-3746": 1, "MED-3747": 1, "MED-3748": 1, "MED-4022": 1, "MED-4023": 1, "MED-4024": 1, "MED-4025": 1, "MED-4029": 1, "MED-4013": 1, "MED-4030": 1, "MED-4031": 1, "MED-4032": 1, "MED-4033": 1, "MED-4034": 1, "MED-4019": 1, "MED-4035": 1, "MED-4036": 1, "MED-4127": 1, "MED-4124": 1, "MED-4125": 1, "MED-4126": 1, "MED-4265": 1, "MED-4266": 1, "MED-4267": 1, "MED-4286": 1, "MED-4281": 1, "MED-4413": 1, "MED-4545": 1, "MED-4585": 1, "MED-5064": 1, "MED-5065": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-1299": {"MED-2769": 1, "MED-2770": 1, "MED-2771": 1, "MED-2772": 1, "MED-2774": 1, "MED-2775": 1, "MED-3563": 1, "MED-3557": 1, "MED-3558": 1, "MED-3887": 1, "MED-2086": 1, "MED-2087": 1, "MED-2088": 1, "MED-2773": 1, "MED-3880": 1, "MED-4132": 1, "MED-3882": 1, "MED-3567": 1, "MED-3884": 1, "MED-3885": 1, "MED-3570": 1, "MED-3886": 1, "MED-3572": 1, "MED-3888": 1, "MED-3889": 1, "MED-3890": 1, "MED-3577": 1, "MED-3891": 1, "MED-3892": 1, "MED-3954": 1, "MED-3955": 1, "MED-3956": 1, "MED-4680": 1, "MED-3958": 1, "MED-3959": 1, "MED-4144": 1, "MED-4145": 1, "MED-4146": 1, "MED-4147": 1, "MED-4148": 1, "MED-4747": 1, "MED-4748": 1, "MED-4749": 1, "MED-4750": 1, "MED-4751": 1, "MED-4791": 1, "MED-4792": 1, "MED-4804": 1, "MED-4794": 1, "MED-4795": 1, "MED-4796": 1, "MED-4797": 1, "MED-4798": 1, "MED-4799": 1, "MED-4800": 1, "MED-4801": 1, "MED-4802": 1, "MED-4803": 1, "MED-4912": 1, "MED-4911": 1, "MED-5341": 1}, "PLAIN-1309": {"MED-4182": 1, "MED-4183": 1, "MED-4380": 1, "MED-4381": 1}, "PLAIN-1320": {"MED-2757": 1, "MED-2758": 1, "MED-2762": 1, "MED-2759": 1, "MED-2760": 1, "MED-2761": 1}, "PLAIN-1331": {"MED-4375": 1}, "PLAIN-1342": {"MED-1401": 1, "MED-1402": 1, "MED-1366": 1, "MED-1403": 1, "MED-1404": 1, "MED-1405": 1, "MED-1406": 1, "MED-1407": 1, "MED-1408": 1, "MED-1409": 1, "MED-1410": 1, "MED-1411": 1, "MED-2582": 1, "MED-4316": 1, "MED-4317": 1, "MED-4318": 1}, "PLAIN-1353": {"MED-2997": 1, "MED-2999": 1, "MED-3000": 1, "MED-3001": 1}, "PLAIN-1363": {"MED-3617": 1, "MED-3618": 1, "MED-3620": 1, "MED-3621": 1, "MED-3615": 1, "MED-3622": 1, "MED-3623": 1, "MED-3624": 1, "MED-3625": 1, "MED-3626": 1, "MED-3627": 1}, "PLAIN-1374": {"MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3089": 1, "MED-3090": 1, "MED-3091": 1, "MED-3092": 1, "MED-3093": 1, "MED-3094": 1, "MED-3095": 1, "MED-3096": 1, "MED-4747": 1, "MED-4748": 1, "MED-4749": 1, "MED-4750": 1, "MED-4751": 1, "MED-4752": 1, "MED-4753": 1, "MED-4757": 1, "MED-4755": 1, "MED-4756": 1, "MED-4758": 1, "MED-4759": 1, "MED-4897": 1, "MED-4951": 1}, "PLAIN-1387": {"MED-3378": 1, "MED-3380": 1, "MED-5062": 1, "MED-3382": 1, "MED-3383": 1, "MED-3384": 1, "MED-3379": 1, "MED-3381": 1, "MED-4711": 1, "MED-4840": 1, "MED-5058": 1, "MED-5063": 1}, "PLAIN-1398": {"MED-2513": 1, "MED-2502": 1, "MED-2498": 1, "MED-2504": 1, "MED-2505": 1, "MED-2507": 1, "MED-5239": 1, "MED-2509": 1, "MED-2519": 1, "MED-2510": 1, "MED-2511": 1, "MED-3563": 1, "MED-3557": 1, "MED-3558": 1, "MED-1763": 1, "MED-1584": 1, "MED-1585": 1, "MED-1586": 1, "MED-1587": 1, "MED-1588": 1, "MED-1595": 1, "MED-1596": 1, "MED-1634": 1, "MED-1635": 1, "MED-1636": 1, "MED-1637": 1, "MED-1638": 1, "MED-1639": 1, "MED-1640": 1, "MED-1641": 1, "MED-1642": 1, "MED-1643": 1, "MED-5258": 1, "MED-1645": 1, "MED-1646": 1, "MED-1647": 1, "MED-1648": 1, "MED-1649": 1, "MED-1711": 1, "MED-1712": 1, "MED-2763": 1, "MED-1714": 1, "MED-1715": 1, "MED-1716": 1, "MED-1717": 1, "MED-1718": 1, "MED-1719": 1, "MED-1720": 1, "MED-1721": 1, "MED-1722": 1, "MED-1723": 1, "MED-1724": 1, "MED-2351": 1, "MED-2352": 1, "MED-2353": 1, "MED-2354": 1, "MED-2355": 1, "MED-2356": 1, "MED-2357": 1, "MED-3316": 1, "MED-2359": 1, "MED-2360": 1, "MED-2361": 1, "MED-2362": 1, "MED-2363": 1, "MED-2364": 1, "MED-2365": 1, "MED-2366": 1, "MED-2367": 1, "MED-2368": 1, "MED-2369": 1, "MED-2517": 1, "MED-2501": 1, "MED-2506": 1, "MED-2512": 1, "MED-3954": 1, "MED-3955": 1, "MED-3956": 1, "MED-4680": 1, "MED-3958": 1, "MED-3959": 1, "MED-4112": 1, "MED-4113": 1, "MED-4114": 1, "MED-4115": 1, "MED-4163": 1, "MED-4164": 1, "MED-4224": 1, "MED-4226": 1, "MED-4228": 1, "MED-4216": 1, "MED-4211": 1, "MED-4212": 1, "MED-4886": 1, "MED-4220": 1, "MED-4612": 1, "MED-4222": 1, "MED-4223": 1, "MED-4225": 1, "MED-4227": 1, "MED-4897": 1, "MED-4953": 1, "MED-4954": 1}, "PLAIN-1409": {"MED-2494": 1, "MED-2495": 1, "MED-2496": 1, "MED-2493": 1, "MED-2497": 1, "MED-2644": 1, "MED-2646": 1, "MED-2651": 1, "MED-118": 1, "MED-2652": 1, "MED-2653": 1, "MED-2655": 1, "MED-2659": 1, "MED-2661": 1, "MED-2662": 1, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-2649": 1, "MED-3028": 1, "MED-2913": 1, "MED-3021": 1, "MED-3012": 1, "MED-3033": 1, "MED-2906": 1, "MED-3034": 1, "MED-3109": 1, "MED-3098": 1, "MED-3099": 1, "MED-3100": 1, "MED-3112": 1, "MED-1098": 1, "MED-1099": 1, "MED-1100": 1, "MED-1101": 1, "MED-3590": 1, "MED-1763": 1, "MED-1592": 1, "MED-1593": 1, "MED-1594": 1, "MED-1595": 1, "MED-1596": 1, "MED-1597": 1, "MED-1747": 1, "MED-1748": 1, "MED-1749": 1, "MED-1750": 1, "MED-1751": 1, "MED-1752": 1, "MED-1753": 1, "MED-1754": 1, "MED-1762": 1, "MED-1764": 1, "MED-1765": 1, "MED-1766": 1, "MED-1778": 1, "MED-1768": 1, "MED-4951": 1, "MED-1770": 1, "MED-1771": 1, "MED-1781": 1, "MED-1773": 1, "MED-1774": 1, "MED-1831": 1, "MED-1832": 1, "MED-1833": 1, "MED-1834": 1, "MED-5092": 1, "MED-1956": 1, "MED-1957": 1, "MED-1958": 1, "MED-1959": 1, "MED-1960": 1, "MED-1961": 1, "MED-1962": 1, "MED-1963": 1, "MED-2042": 1, "MED-2043": 1, "MED-2044": 1, "MED-2045": 1, "MED-2046": 1, "MED-2047": 1, "MED-2048": 1, "MED-2049": 1, "MED-2050": 1, "MED-2051": 1, "MED-2052": 1, "MED-2099": 1, "MED-2100": 1, "MED-2101": 1, "MED-2102": 1, "MED-2103": 1, "MED-2529": 1, "MED-2105": 1, "MED-2106": 1, "MED-2107": 1, "MED-2108": 1, "MED-2385": 1, "MED-2386": 1, "MED-2395": 1, "MED-2388": 1, "MED-2389": 1, "MED-2396": 1, "MED-2391": 1, "MED-2407": 1, "MED-2405": 1, "MED-2394": 1, "MED-2397": 1, "MED-2398": 1, "MED-2400": 1, "MED-2402": 1, "MED-2411": 1, "MED-2404": 1, "MED-2406": 1, "MED-2409": 1, "MED-2410": 1, "MED-2412": 1, "MED-2413": 1, "MED-2461": 1, "MED-2464": 1, "MED-2468": 1, "MED-2469": 1, "MED-2471": 1, "MED-2472": 1, "MED-2474": 1, "MED-2475": 1, "MED-2476": 1, "MED-2479": 1, "MED-2482": 1, "MED-2484": 1, "MED-3687": 1, "MED-2490": 1, "MED-2491": 1, "MED-2492": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2650": 1, "MED-2654": 1, "MED-3023": 1, "MED-2904": 1, "MED-2905": 1, "MED-2907": 1, "MED-3024": 1, "MED-3025": 1, "MED-2910": 1, "MED-3027": 1, "MED-3013": 1, "MED-3030": 1, "MED-2917": 1, "MED-3035": 1, "MED-2921": 1, "MED-3102": 1, "MED-3585": 1, "MED-3586": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3591": 1, "MED-3592": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-4726": 1, "MED-3935": 1, "MED-3936": 1, "MED-3937": 1, "MED-3938": 1, "MED-3939": 1, "MED-3940": 1, "MED-3954": 1, "MED-3955": 1, "MED-3956": 1, "MED-4680": 1, "MED-3958": 1, "MED-3959": 1, "MED-4167": 1, "MED-4168": 1, "MED-4169": 1, "MED-4170": 1, "MED-4171": 1, "MED-4174": 1, "MED-4175": 1, "MED-4176": 1, "MED-4177": 1, "MED-4178": 1, "MED-4179": 1, "MED-4180": 1, "MED-4181": 1, "MED-4182": 1, "MED-4183": 1, "MED-4184": 1, "MED-4187": 1, "MED-4404": 1, "MED-4405": 1, "MED-4436": 1, "MED-4525": 1, "MED-4526": 1, "MED-4527": 1, "MED-4533": 1, "MED-4529": 1, "MED-4535": 1, "MED-4531": 1, "MED-4532": 1, "MED-4534": 1, "MED-4536": 1, "MED-4548": 1, "MED-4622": 1, "MED-4936": 1, "MED-4730": 1, "MED-4653": 1, "MED-4654": 1, "MED-4655": 1, "MED-4656": 1, "MED-4933": 1, "MED-4934": 1, "MED-4935": 1, "MED-4937": 1, "MED-4944": 1, "MED-4950": 1, "MED-4991": 1, "MED-4992": 1, "MED-5085": 1, "MED-5086": 1, "MED-5087": 1, "MED-5088": 1, "MED-5089": 1, "MED-5094": 1, "MED-5095": 1, "MED-5096": 1, "MED-5097": 1, "MED-5098": 1, "MED-5099": 1, "MED-5100": 1, "MED-5101": 1, "MED-5102": 1, "MED-5104": 1, "MED-5105": 1, "MED-5188": 1, "MED-5189": 1, "MED-5190": 1, "MED-5191": 1, "MED-5192": 1, "MED-5193": 1, "MED-5194": 1, "MED-5195": 1, "MED-5196": 1, "MED-5197": 1}, "PLAIN-1419": {"MED-3276": 1, "MED-3277": 1, "MED-3271": 1, "MED-3278": 1, "MED-3272": 1, "MED-3274": 1, "MED-3275": 1, "MED-3282": 1, "MED-1148": 1, "MED-1149": 1, "MED-1150": 1, "MED-1151": 1, "MED-1152": 1, "MED-1153": 1, "MED-1179": 1, "MED-1181": 1, "MED-1156": 1, "MED-2337": 1, "MED-2338": 1, "MED-2339": 1, "MED-2340": 1, "MED-2341": 1, "MED-2355": 1, "MED-2356": 1, "MED-2344": 1, "MED-2345": 1, "MED-2346": 1, "MED-2347": 1, "MED-2348": 1, "MED-2349": 1, "MED-2350": 1, "MED-2351": 1, "MED-2352": 1, "MED-2353": 1, "MED-2354": 1, "MED-2357": 1, "MED-3316": 1, "MED-2359": 1, "MED-2360": 1, "MED-2361": 1, "MED-2362": 1, "MED-2363": 1, "MED-2364": 1, "MED-2365": 1, "MED-2366": 1, "MED-2367": 1, "MED-2368": 1, "MED-2369": 1, "MED-3270": 1, "MED-3273": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3283": 1, "MED-4198": 1, "MED-4203": 1, "MED-4299": 1, "MED-4205": 1, "MED-4206": 1, "MED-4357": 1, "MED-4403": 1, "MED-4913": 1, "MED-4914": 1, "MED-4915": 1, "MED-5029": 1}, "PLAIN-1429": {"MED-2757": 1, "MED-2758": 1, "MED-2762": 1, "MED-2759": 1, "MED-2760": 1, "MED-2761": 1}, "PLAIN-1441": {"MED-2513": 1, "MED-2502": 1, "MED-2498": 1, "MED-2504": 1, "MED-2505": 1, "MED-2507": 1, "MED-5239": 1, "MED-2509": 1, "MED-2519": 1, "MED-2510": 1, "MED-2511": 1, "MED-2644": 1, "MED-2646": 1, "MED-2651": 1, "MED-118": 1, "MED-2652": 1, "MED-2653": 1, "MED-2655": 1, "MED-2659": 1, "MED-2661": 1, "MED-2662": 1, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-2649": 1, "MED-2999": 1, "MED-2997": 1, "MED-2765": 1, "MED-3001": 1, "MED-2769": 1, "MED-2770": 1, "MED-2771": 1, "MED-2772": 1, "MED-2774": 1, "MED-2775": 1, "MED-2882": 1, "MED-2899": 1, "MED-2894": 1, "MED-3136": 1, "MED-3138": 1, "MED-3141": 1, "MED-3140": 1, "MED-3443": 1, "MED-3444": 1, "MED-3441": 1, "MED-3445": 1, "MED-3446": 1, "MED-3442": 1, "MED-3447": 1, "MED-3448": 1, "MED-1318": 1, "MED-1319": 1, "MED-1320": 1, "MED-1321": 1, "MED-1322": 1, "MED-1323": 1, "MED-1324": 1, "MED-1612": 1, "MED-1326": 1, "MED-1327": 1, "MED-1328": 1, "MED-1329": 1, "MED-1330": 1, "MED-1331": 1, "MED-1332": 1, "MED-1333": 1, "MED-1334": 1, "MED-1335": 1, "MED-1795": 1, "MED-1418": 1, "MED-1419": 1, "MED-1420": 1, "MED-1421": 1, "MED-3964": 1, "MED-4342": 1, "MED-1581": 1, "MED-1425": 1, "MED-1426": 1, "MED-1527": 1, "MED-1528": 1, "MED-1529": 1, "MED-1530": 1, "MED-1531": 1, "MED-1532": 1, "MED-2114": 1, "MED-2115": 1, "MED-2116": 1, "MED-2117": 1, "MED-4896": 1, "MED-2120": 1, "MED-2121": 1, "MED-2122": 1, "MED-2123": 1, "MED-2124": 1, "MED-2136": 1, "MED-2126": 1, "MED-2445": 1, "MED-2446": 1, "MED-2458": 1, "MED-2448": 1, "MED-2449": 1, "MED-2450": 1, "MED-2451": 1, "MED-2452": 1, "MED-2453": 1, "MED-2472": 1, "MED-2517": 1, "MED-2501": 1, "MED-2506": 1, "MED-2512": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2650": 1, "MED-2654": 1, "MED-3000": 1, "MED-2773": 1, "MED-2898": 1, "MED-2884": 1, "MED-2885": 1, "MED-2886": 1, "MED-2888": 1, "MED-2889": 1, "MED-2890": 1, "MED-2891": 1, "MED-2892": 1, "MED-2893": 1, "MED-2895": 1, "MED-3139": 1, "MED-3142": 1, "MED-3143": 1, "MED-3623": 1, "MED-3624": 1, "MED-3625": 1, "MED-3626": 1, "MED-3627": 1, "MED-3628": 1, "MED-3629": 1, "MED-3630": 1, "MED-3631": 1, "MED-3632": 1, "MED-3633": 1, "MED-3634": 1, "MED-3635": 1, "MED-3636": 1, "MED-3863": 1, "MED-3864": 1, "MED-3865": 1, "MED-3874": 1, "MED-3875": 1, "MED-3876": 1, "MED-3877": 1, "MED-3878": 1, "MED-4233": 1, "MED-3971": 1, "MED-3972": 1, "MED-3973": 1, "MED-3974": 1, "MED-3975": 1, "MED-4127": 1, "MED-4124": 1, "MED-4125": 1, "MED-4126": 1, "MED-4184": 1, "MED-4216": 1, "MED-4211": 1, "MED-4212": 1, "MED-4886": 1, "MED-4466": 1, "MED-4625": 1, "MED-4626": 1, "MED-4635": 1, "MED-4640": 1, "MED-4747": 1, "MED-4748": 1, "MED-4749": 1, "MED-4750": 1, "MED-4751": 1, "MED-4867": 1, "MED-4868": 1, "MED-4869": 1, "MED-5030": 1}, "PLAIN-1453": {"MED-2797": 1, "MED-2809": 1, "MED-2799": 1, "MED-2800": 1, "MED-2801": 1, "MED-2802": 1, "MED-2803": 1, "MED-2804": 1, "MED-2805": 1, "MED-2807": 1, "MED-3050": 1, "MED-3038": 1, "MED-3052": 1, "MED-3053": 1, "MED-3054": 1, "MED-3044": 1, "MED-3046": 1, "MED-3058": 1, "MED-3059": 1, "MED-3060": 1, "MED-3087": 1, "MED-3088": 1, "MED-3089": 1, "MED-3093": 1, "MED-332": 1, "MED-3090": 1, "MED-334": 1, "MED-335": 1, "MED-3092": 1, "MED-3094": 1, "MED-3348": 1, "MED-3351": 1, "MED-3354": 1, "MED-3359": 1, "MED-3352": 1, "MED-3350": 1, "MED-3356": 1, "MED-3358": 1, "MED-3355": 1, "MED-3369": 1, "MED-3370": 1, "MED-3372": 1, "MED-3376": 1, "MED-3377": 1, "MED-3374": 1, "MED-3598": 1, "MED-3597": 1, "MED-1139": 1, "MED-1140": 1, "MED-1167": 1, "MED-1142": 1, "MED-1143": 1, "MED-1144": 1, "MED-1181": 1, "MED-1146": 1, "MED-1147": 1, "MED-1195": 1, "MED-1196": 1, "MED-5360": 1, "MED-1198": 1, "MED-1199": 1, "MED-1200": 1, "MED-1201": 1, "MED-1202": 1, "MED-1203": 1, "MED-1374": 1, "MED-1375": 1, "MED-1376": 1, "MED-1377": 1, "MED-1378": 1, "MED-2505": 1, "MED-1380": 1, "MED-1527": 1, "MED-1528": 1, "MED-1529": 1, "MED-1530": 1, "MED-1531": 1, "MED-1532": 1, "MED-1533": 1, "MED-1534": 1, "MED-1535": 1, "MED-3900": 1, "MED-5081": 1, "MED-1538": 1, "MED-1540": 1, "MED-1541": 1, "MED-1557": 1, "MED-1558": 1, "MED-1574": 1, "MED-1575": 1, "MED-1576": 1, "MED-1577": 1, "MED-1578": 1, "MED-1579": 1, "MED-1580": 1, "MED-1581": 1, "MED-1582": 1, "MED-1609": 1, "MED-1610": 1, "MED-1611": 1, "MED-1612": 1, "MED-1613": 1, "MED-1614": 1, "MED-1615": 1, "MED-1616": 1, "MED-1617": 1, "MED-1618": 1, "MED-1619": 1, "MED-1620": 1, "MED-1650": 1, "MED-1651": 1, "MED-2726": 1, "MED-2221": 1, "MED-4602": 1, "MED-1760": 1, "MED-1761": 1, "MED-1847": 1, "MED-1848": 1, "MED-1849": 1, "MED-1850": 1, "MED-1851": 1, "MED-1852": 1, "MED-1861": 1, "MED-1862": 1, "MED-1863": 1, "MED-1864": 1, "MED-1865": 1, "MED-1866": 1, "MED-1867": 1, "MED-1868": 1, "MED-1869": 1, "MED-1871": 1, "MED-1872": 1, "MED-1873": 1, "MED-1874": 1, "MED-1992": 1, "MED-1876": 1, "MED-1998": 1, "MED-1878": 1, "MED-1879": 1, "MED-2722": 1, "MED-2489": 1, "MED-2033": 1, "MED-2013": 1, "MED-2014": 1, "MED-2027": 1, "MED-2016": 1, "MED-2037": 1, "MED-2114": 1, "MED-2115": 1, "MED-2116": 1, "MED-2117": 1, "MED-4896": 1, "MED-2120": 1, "MED-2121": 1, "MED-2122": 1, "MED-2123": 1, "MED-2124": 1, "MED-2136": 1, "MED-2126": 1, "MED-2179": 1, "MED-2180": 1, "MED-2181": 1, "MED-2182": 1, "MED-2183": 1, "MED-2184": 1, "MED-2219": 1, "MED-2220": 1, "MED-2248": 1, "MED-2249": 1, "MED-2250": 1, "MED-2251": 1, "MED-2252": 1, "MED-2253": 1, "MED-2254": 1, "MED-2255": 1, "MED-2256": 1, "MED-2257": 1, "MED-2258": 1, "MED-2259": 1, "MED-2260": 1, "MED-2261": 1, "MED-2262": 1, "MED-2263": 1, "MED-2264": 1, "MED-2288": 1, "MED-3137": 1, "MED-2290": 1, "MED-2291": 1, "MED-2292": 1, "MED-2293": 1, "MED-2294": 1, "MED-2295": 1, "MED-2296": 1, "MED-2819": 1, "MED-3051": 1, "MED-3055": 1, "MED-3056": 1, "MED-3057": 1, "MED-3085": 1, "MED-3086": 1, "MED-3091": 1, "MED-3095": 1, "MED-3096": 1, "MED-3371": 1, "MED-3353": 1, "MED-3373": 1, "MED-3375": 1, "MED-3599": 1, "MED-3769": 1, "MED-3770": 1, "MED-4203": 1, "MED-4299": 1, "MED-4205": 1, "MED-4206": 1, "MED-4251": 1, "MED-4352": 1, "MED-4353": 1, "MED-4450": 1, "MED-4556": 1, "MED-4559": 1, "MED-4585": 1, "MED-4590": 1, "MED-4587": 1, "MED-4588": 1, "MED-4589": 1, "MED-4686": 1, "MED-4687": 1, "MED-4689": 1, "MED-4711": 1, "MED-4715": 1, "MED-4716": 1, "MED-4744": 1, "MED-4746": 1, "MED-4785": 1, "MED-4782": 1, "MED-5038": 1, "MED-5039": 1, "MED-5040": 1, "MED-5052": 1, "MED-5056": 1, "MED-5057": 1, "MED-5059": 1, "MED-5062": 1, "MED-5063": 1}, "PLAIN-1463": {"MED-2966": 1, "MED-2968": 1, "MED-2970": 1, "MED-2972": 1, "MED-4981": 1, "MED-2140": 1, "MED-2141": 1, "MED-4319": 1, "MED-2143": 1, "MED-2144": 1, "MED-2145": 1, "MED-2146": 1, "MED-4523": 1, "MED-2967": 1, "MED-2969": 1, "MED-2971": 1, "MED-3136": 1, "MED-3137": 1, "MED-3138": 1, "MED-3139": 1, "MED-3140": 1, "MED-3141": 1, "MED-3142": 1, "MED-3143": 1}, "PLAIN-1473": {"MED-4455": 1}, "PLAIN-1485": {"MED-2644": 1, "MED-2646": 1, "MED-2651": 1, "MED-118": 1, "MED-2652": 1, "MED-2653": 1, "MED-2655": 1, "MED-2659": 1, "MED-2661": 1, "MED-2662": 1, "MED-1249": 1, "MED-1250": 1, "MED-4831": 1, "MED-1252": 1, "MED-1253": 1, "MED-1254": 1, "MED-1801": 1, "MED-1256": 1, "MED-1257": 1, "MED-1258": 1, "MED-1381": 1, "MED-2593": 1, "MED-1383": 1, "MED-1393": 1, "MED-4710": 1, "MED-1386": 1, "MED-1387": 1, "MED-1388": 1, "MED-1389": 1, "MED-1390": 1, "MED-1394": 1, "MED-1491": 1, "MED-1492": 1, "MED-1493": 1, "MED-1494": 1, "MED-1495": 1, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2649": 1, "MED-2650": 1, "MED-2654": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-4349": 1, "MED-4520": 1, "MED-4450": 1, "MED-4451": 1, "MED-4686": 1}, "PLAIN-1496": {"MED-4859": 1}, "PLAIN-1506": {"MED-2513": 1, "MED-2502": 1, "MED-2498": 1, "MED-2504": 1, "MED-2505": 1, "MED-2507": 1, "MED-5239": 1, "MED-2509": 1, "MED-2519": 1, "MED-2510": 1, "MED-2511": 1, "MED-2517": 1, "MED-2501": 1, "MED-2506": 1, "MED-2512": 1}, "PLAIN-1516": {"MED-5102": 1, "MED-5104": 1}, "PLAIN-1527": {"MED-2527": 1, "MED-2675": 1, "MED-2676": 1, "MED-2677": 1, "MED-2678": 1, "MED-2679": 1, "MED-2780": 1, "MED-2810": 1, "MED-2787": 1, "MED-2788": 1, "MED-2777": 1, "MED-2816": 1, "MED-3315": 1, "MED-3305": 1, "MED-3308": 1, "MED-3310": 1, "MED-3312": 1, "MED-3318": 1, "MED-3313": 1, "MED-3486": 1, "MED-3492": 1, "MED-3487": 1, "MED-3493": 1, "MED-3488": 1, "MED-3489": 1, "MED-3490": 1, "MED-3491": 1, "MED-3811": 1, "MED-1301": 1, "MED-1302": 1, "MED-1303": 1, "MED-1304": 1, "MED-1305": 1, "MED-1313": 1, "MED-1307": 1, "MED-1308": 1, "MED-1309": 1, "MED-1327": 1, "MED-1669": 1, "MED-1670": 1, "MED-1671": 1, "MED-1672": 1, "MED-1673": 1, "MED-1674": 1, "MED-1675": 1, "MED-1676": 1, "MED-1706": 1, "MED-1707": 1, "MED-1708": 1, "MED-1709": 1, "MED-1710": 1, "MED-1853": 1, "MED-1871": 1, "MED-1864": 1, "MED-4027": 1, "MED-1857": 1, "MED-1858": 1, "MED-1859": 1, "MED-1860": 1, "MED-2008": 1, "MED-2009": 1, "MED-2010": 1, "MED-2011": 1, "MED-2042": 1, "MED-2043": 1, "MED-2044": 1, "MED-2045": 1, "MED-2046": 1, "MED-2047": 1, "MED-2048": 1, "MED-2049": 1, "MED-2050": 1, "MED-2051": 1, "MED-2052": 1, "MED-2140": 1, "MED-2141": 1, "MED-4319": 1, "MED-2143": 1, "MED-2144": 1, "MED-2145": 1, "MED-2146": 1, "MED-2154": 1, "MED-2155": 1, "MED-2156": 1, "MED-2157": 1, "MED-2158": 1, "MED-2159": 1, "MED-2160": 1, "MED-2161": 1, "MED-2162": 1, "MED-2163": 1, "MED-2164": 1, "MED-2165": 1, "MED-2166": 1, "MED-2167": 1, "MED-2168": 1, "MED-2169": 1, "MED-2170": 1, "MED-2171": 1, "MED-2172": 1, "MED-2173": 1, "MED-2174": 1, "MED-2175": 1, "MED-2176": 1, "MED-2177": 1, "MED-2178": 1, "MED-2351": 1, "MED-2352": 1, "MED-2353": 1, "MED-2354": 1, "MED-2355": 1, "MED-2356": 1, "MED-2357": 1, "MED-3316": 1, "MED-2359": 1, "MED-2360": 1, "MED-2361": 1, "MED-2362": 1, "MED-2363": 1, "MED-2364": 1, "MED-2365": 1, "MED-2366": 1, "MED-2367": 1, "MED-2368": 1, "MED-2369": 1, "MED-2524": 1, "MED-2525": 1, "MED-2526": 1, "MED-2528": 1, "MED-2529": 1, "MED-2530": 1, "MED-2616": 1, "MED-2617": 1, "MED-2618": 1, "MED-2619": 1, "MED-2674": 1, "MED-2815": 1, "MED-2781": 1, "MED-2782": 1, "MED-2783": 1, "MED-2818": 1, "MED-2785": 1, "MED-2786": 1, "MED-3306": 1, "MED-3307": 1, "MED-3309": 1, "MED-3311": 1, "MED-3314": 1, "MED-3317": 1, "MED-3319": 1, "MED-3320": 1, "MED-3321": 1, "MED-3733": 1, "MED-3734": 1, "MED-3744": 1, "MED-3745": 1, "MED-3742": 1, "MED-3738": 1, "MED-3747": 1, "MED-3743": 1, "MED-3748": 1, "MED-4100": 1, "MED-4101": 1, "MED-4102": 1, "MED-4116": 1, "MED-4117": 1, "MED-4118": 1, "MED-4119": 1, "MED-4165": 1, "MED-4969": 1, "MED-4363": 1, "MED-4364": 1, "MED-4373": 1, "MED-4374": 1, "MED-4415": 1, "MED-4416": 1, "MED-4417": 1, "MED-4418": 1, "MED-4543": 1, "MED-4544": 1, "MED-4804": 1, "MED-4812": 1, "MED-4813": 1, "MED-4814": 1, "MED-4815": 1, "MED-4816": 1, "MED-4817": 1, "MED-4870": 1, "MED-4872": 1, "MED-4873": 1, "MED-4874": 1, "MED-4930": 1, "MED-4991": 1, "MED-4992": 1, "MED-5025": 1, "MED-5079": 1, "MED-5080": 1, "MED-5157": 1, "MED-5158": 1, "MED-5163": 1, "MED-5169": 1, "MED-5170": 1}, "PLAIN-1537": {"MED-2586": 1, "MED-2587": 1, "MED-2588": 1, "MED-2591": 1, "MED-1463": 1, "MED-1464": 1, "MED-1465": 1, "MED-1466": 1, "MED-1467": 1, "MED-1468": 1, "MED-1474": 1, "MED-1470": 1, "MED-1471": 1, "MED-1472": 1, "MED-1473": 1, "MED-1475": 1, "MED-1476": 1, "MED-2432": 1, "MED-1478": 1, "MED-1479": 1, "MED-1609": 1, "MED-1610": 1, "MED-1611": 1, "MED-1612": 1, "MED-1613": 1, "MED-1614": 1, "MED-1615": 1, "MED-1616": 1, "MED-1617": 1, "MED-1618": 1, "MED-1619": 1, "MED-1620": 1, "MED-1655": 1, "MED-2211": 1, "MED-2212": 1, "MED-2213": 1, "MED-2214": 1, "MED-2215": 1, "MED-2216": 1, "MED-2217": 1, "MED-2218": 1, "MED-2589": 1, "MED-2590": 1, "MED-4514": 1, "MED-4515": 1}, "PLAIN-1547": {"MED-2355": 1, "MED-2362": 1, "MED-2363": 1, "MED-2364": 1, "MED-2365": 1, "MED-2369": 1}, "PLAIN-1557": {"MED-3137": 1, "MED-3148": 1, "MED-3149": 1, "MED-4313": 1, "MED-1133": 1, "MED-1134": 1, "MED-1135": 1, "MED-1136": 1, "MED-1137": 1, "MED-1138": 1, "MED-3138": 1, "MED-3139": 1, "MED-3140": 1, "MED-3141": 1, "MED-3142": 1, "MED-3143": 1, "MED-3918": 1, "MED-3919": 1, "MED-3920": 1, "MED-3921": 1, "MED-3922": 1, "MED-3923": 1, "MED-3924": 1, "MED-4314": 1, "MED-4315": 1, "MED-5145": 1}, "PLAIN-1568": {"MED-4150": 1, "MED-5056": 1}, "PLAIN-1579": {"MED-4667": 1, "MED-5109": 1, "MED-4665": 1, "MED-4666": 1, "MED-4668": 1}, "PLAIN-1590": {"MED-2527": 1, "MED-2524": 1, "MED-2525": 1, "MED-2526": 1, "MED-2528": 1, "MED-2529": 1, "MED-2530": 1, "MED-3619": 1, "MED-3601": 1, "MED-3602": 1, "MED-3603": 1, "MED-3604": 1, "MED-3615": 1, "MED-3606": 1, "MED-3607": 1, "MED-3608": 1, "MED-3609": 1, "MED-3610": 1}, "PLAIN-1601": {"MED-2644": 1, "MED-2646": 1, "MED-2651": 1, "MED-118": 1, "MED-2652": 1, "MED-2653": 1, "MED-2655": 1, "MED-2659": 1, "MED-2661": 1, "MED-2662": 1, "MED-2663": 1, "MED-2664": 1, "MED-2665": 1, "MED-4860": 1, "MED-2668": 1, "MED-2670": 1, "MED-2669": 1, "MED-3536": 1, "MED-3537": 1, "MED-3539": 1, "MED-5030": 1, "MED-3771": 1, "MED-3774": 1, "MED-3775": 1, "MED-3776": 1, "MED-1098": 1, "MED-1099": 1, "MED-1100": 1, "MED-1101": 1, "MED-3590": 1, "MED-1166": 1, "MED-1167": 1, "MED-1178": 1, "MED-1169": 1, "MED-1170": 1, "MED-1171": 1, "MED-1172": 1, "MED-1173": 1, "MED-1174": 1, "MED-1175": 1, "MED-1176": 1, "MED-1177": 1, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2649": 1, "MED-2650": 1, "MED-2654": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-2667": 1, "MED-3538": 1, "MED-4380": 1, "MED-4381": 1, "MED-4404": 1, "MED-4405": 1, "MED-4438": 1, "MED-4439": 1, "MED-4581": 1, "MED-4582": 1, "MED-4583": 1, "MED-4711": 1, "MED-4790": 1, "MED-4789": 1, "MED-4946": 1, "MED-4996": 1, "MED-5002": 1, "MED-5003": 1, "MED-5004": 1}, "PLAIN-1611": {"MED-2675": 1, "MED-2676": 1, "MED-2677": 1, "MED-2678": 1, "MED-2679": 1, "MED-2674": 1}, "PLAIN-1621": {"MED-4832": 1, "MED-4833": 1, "MED-4834": 1, "MED-4835": 1}, "PLAIN-1635": {"MED-2644": 1, "MED-2646": 1, "MED-2651": 1, "MED-118": 1, "MED-2652": 1, "MED-2653": 1, "MED-2655": 1, "MED-2659": 1, "MED-2661": 1, "MED-2662": 1, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-2649": 1, "MED-2769": 1, "MED-2770": 1, "MED-2771": 1, "MED-2772": 1, "MED-2774": 1, "MED-2775": 1, "MED-2827": 1, "MED-2830": 1, "MED-2826": 1, "MED-3146": 1, "MED-3145": 1, "MED-3781": 1, "MED-3789": 1, "MED-3790": 1, "MED-3782": 1, "MED-3780": 1, "MED-3784": 1, "MED-3220": 1, "MED-3221": 1, "MED-3231": 1, "MED-3230": 1, "MED-3236": 1, "MED-3217": 1, "MED-3237": 1, "MED-3250": 1, "MED-3251": 1, "MED-3247": 1, "MED-3248": 1, "MED-3249": 1, "MED-3276": 1, "MED-3277": 1, "MED-3271": 1, "MED-3278": 1, "MED-3272": 1, "MED-3274": 1, "MED-3275": 1, "MED-3282": 1, "MED-1148": 1, "MED-1149": 1, "MED-1150": 1, "MED-1151": 1, "MED-1152": 1, "MED-1153": 1, "MED-1179": 1, "MED-1181": 1, "MED-1156": 1, "MED-1223": 1, "MED-1224": 1, "MED-5239": 1, "MED-1226": 1, "MED-1227": 1, "MED-2126": 1, "MED-1229": 1, "MED-1230": 1, "MED-1337": 1, "MED-1338": 1, "MED-1339": 1, "MED-1340": 1, "MED-1341": 1, "MED-1463": 1, "MED-1454": 1, "MED-1455": 1, "MED-1456": 1, "MED-1457": 1, "MED-1458": 1, "MED-1459": 1, "MED-1460": 1, "MED-1461": 1, "MED-1474": 1, "MED-1551": 1, "MED-1552": 1, "MED-1553": 1, "MED-1554": 1, "MED-1555": 1, "MED-1556": 1, "MED-1557": 1, "MED-1558": 1, "MED-1763": 1, "MED-1584": 1, "MED-1585": 1, "MED-1586": 1, "MED-1587": 1, "MED-1588": 1, "MED-1595": 1, "MED-1596": 1, "MED-1592": 1, "MED-1593": 1, "MED-1594": 1, "MED-1597": 1, "MED-1634": 1, "MED-1635": 1, "MED-1636": 1, "MED-1637": 1, "MED-1638": 1, "MED-1639": 1, "MED-1640": 1, "MED-1641": 1, "MED-1642": 1, "MED-1643": 1, "MED-5258": 1, "MED-1645": 1, "MED-1646": 1, "MED-1647": 1, "MED-1648": 1, "MED-1649": 1, "MED-1762": 1, "MED-1764": 1, "MED-1765": 1, "MED-1766": 1, "MED-1778": 1, "MED-1768": 1, "MED-4951": 1, "MED-1770": 1, "MED-1771": 1, "MED-1781": 1, "MED-1773": 1, "MED-1774": 1, "MED-1956": 1, "MED-1957": 1, "MED-1958": 1, "MED-1959": 1, "MED-1960": 1, "MED-1961": 1, "MED-1962": 1, "MED-1963": 1, "MED-1998": 1, "MED-1985": 1, "MED-1986": 1, "MED-1987": 1, "MED-1988": 1, "MED-1999": 1, "MED-1990": 1, "MED-1991": 1, "MED-1992": 1, "MED-1993": 1, "MED-1994": 1, "MED-2221": 1, "MED-1996": 1, "MED-1997": 1, "MED-2031": 1, "MED-2032": 1, "MED-2033": 1, "MED-2034": 1, "MED-2035": 1, "MED-2036": 1, "MED-2037": 1, "MED-2038": 1, "MED-2039": 1, "MED-2040": 1, "MED-2041": 1, "MED-2053": 1, "MED-2054": 1, "MED-2055": 1, "MED-2056": 1, "MED-2057": 1, "MED-2058": 1, "MED-2059": 1, "MED-2060": 1, "MED-2061": 1, "MED-2062": 1, "MED-2063": 1, "MED-2114": 1, "MED-2115": 1, "MED-2116": 1, "MED-2117": 1, "MED-4896": 1, "MED-2120": 1, "MED-2121": 1, "MED-2122": 1, "MED-2123": 1, "MED-2124": 1, "MED-2136": 1, "MED-2127": 1, "MED-2128": 1, "MED-2129": 1, "MED-2130": 1, "MED-2507": 1, "MED-2132": 1, "MED-2517": 1, "MED-2134": 1, "MED-2135": 1, "MED-2137": 1, "MED-2138": 1, "MED-2139": 1, "MED-2222": 1, "MED-2223": 1, "MED-2224": 1, "MED-2225": 1, "MED-2226": 1, "MED-2227": 1, "MED-2228": 1, "MED-2488": 1, "MED-2489": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2650": 1, "MED-2654": 1, "MED-2773": 1, "MED-2831": 1, "MED-3144": 1, "MED-3787": 1, "MED-3785": 1, "MED-3786": 1, "MED-3193": 1, "MED-3783": 1, "MED-3226": 1, "MED-3228": 1, "MED-4722": 1, "MED-3215": 1, "MED-3216": 1, "MED-3227": 1, "MED-3229": 1, "MED-3233": 1, "MED-3252": 1, "MED-3270": 1, "MED-3273": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3283": 1, "MED-3353": 1, "MED-3369": 1, "MED-3348": 1, "MED-3371": 1, "MED-3350": 1, "MED-3355": 1, "MED-3356": 1, "MED-3373": 1, "MED-3351": 1, "MED-3352": 1, "MED-3358": 1, "MED-3359": 1, "MED-3449": 1, "MED-3450": 1, "MED-3451": 1, "MED-3452": 1, "MED-3453": 1, "MED-3454": 1, "MED-3455": 1, "MED-3456": 1, "MED-3457": 1, "MED-3458": 1, "MED-3467": 1, "MED-3473": 1, "MED-3469": 1, "MED-3481": 1, "MED-3471": 1, "MED-3478": 1, "MED-3494": 1, "MED-3495": 1, "MED-3496": 1, "MED-3497": 1, "MED-3498": 1, "MED-3499": 1, "MED-3500": 1, "MED-3501": 1, "MED-3502": 1, "MED-3585": 1, "MED-3586": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3590": 1, "MED-3591": 1, "MED-3592": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-3597": 1, "MED-3598": 1, "MED-3599": 1, "MED-3628": 1, "MED-3629": 1, "MED-3630": 1, "MED-3631": 1, "MED-3632": 1, "MED-3633": 1, "MED-3634": 1, "MED-3635": 1, "MED-3636": 1, "MED-3654": 1, "MED-3655": 1, "MED-3656": 1, "MED-3657": 1, "MED-3679": 1, "MED-3749": 1, "MED-3750": 1, "MED-3751": 1, "MED-3752": 1, "MED-3753": 1, "MED-3754": 1, "MED-3755": 1, "MED-3756": 1, "MED-3757": 1, "MED-3758": 1, "MED-3759": 1, "MED-3760": 1, "MED-3761": 1, "MED-3769": 1, "MED-3770": 1, "MED-3771": 1, "MED-3772": 1, "MED-3773": 1, "MED-3774": 1, "MED-3775": 1, "MED-3776": 1, "MED-3788": 1, "MED-3791": 1, "MED-3792": 1, "MED-3793": 1, "MED-3794": 1, "MED-3795": 1, "MED-3796": 1, "MED-3797": 1, "MED-3798": 1, "MED-3799": 1, "MED-3800": 1, "MED-3801": 1, "MED-3925": 1, "MED-3926": 1, "MED-3927": 1, "MED-3928": 1, "MED-3929": 1, "MED-3930": 1, "MED-3931": 1, "MED-3932": 1, "MED-3933": 1, "MED-4726": 1, "MED-3935": 1, "MED-3936": 1, "MED-3937": 1, "MED-3938": 1, "MED-3939": 1, "MED-3940": 1, "MED-4167": 1, "MED-4168": 1, "MED-4174": 1, "MED-4175": 1, "MED-4176": 1, "MED-4177": 1, "MED-4178": 1, "MED-4179": 1, "MED-4182": 1, "MED-4183": 1, "MED-4305": 1, "MED-4306": 1, "MED-4308": 1, "MED-4320": 1, "MED-4349": 1, "MED-4520": 1, "MED-4352": 1, "MED-4353": 1, "MED-4394": 1, "MED-4398": 1, "MED-4396": 1, "MED-5106": 1, "MED-4399": 1, "MED-4400": 1, "MED-4402": 1, "MED-4450": 1, "MED-4451": 1, "MED-4602": 1, "MED-4665": 1, "MED-4666": 1, "MED-4667": 1, "MED-4668": 1, "MED-4677": 1, "MED-4682": 1, "MED-4687": 1, "MED-4689": 1, "MED-4752": 1, "MED-4753": 1, "MED-4757": 1, "MED-4755": 1, "MED-4756": 1, "MED-4837": 1, "MED-4841": 1, "MED-4894": 1, "MED-4897": 1, "MED-4898": 1, "MED-4899": 1, "MED-4900": 1, "MED-4901": 1, "MED-4902": 1, "MED-4903": 1, "MED-4912": 1, "MED-4911": 1, "MED-4938": 1, "MED-4939": 1, "MED-4940": 1, "MED-4941": 1, "MED-4942": 1, "MED-5003": 1, "MED-5004": 1, "MED-5107": 1, "MED-5108": 1, "MED-5109": 1, "MED-5143": 1, "MED-5144": 1, "MED-5145": 1, "MED-5155": 1, "MED-5188": 1, "MED-5189": 1, "MED-5190": 1, "MED-5191": 1, "MED-5192": 1, "MED-5193": 1, "MED-5194": 1, "MED-5195": 1, "MED-5196": 1, "MED-5197": 1}, "PLAIN-1645": {"MED-1491": 1, "MED-1492": 1, "MED-1493": 1, "MED-1494": 1, "MED-1495": 1, "MED-5056": 1}, "PLAIN-1656": {"MED-2200": 1, "MED-2201": 1, "MED-2202": 1, "MED-2203": 1, "MED-2204": 1, "MED-2205": 1, "MED-2206": 1, "MED-2207": 1, "MED-2208": 1, "MED-2209": 1, "MED-2210": 1, "MED-2239": 1, "MED-2240": 1, "MED-2245": 1, "MED-2814": 1, "MED-2243": 1, "MED-4037": 1, "MED-4038": 1, "MED-4976": 1, "MED-4433": 1, "MED-4819": 1, "MED-4739": 1, "MED-4740": 1, "MED-4741": 1, "MED-4818": 1, "MED-4842": 1, "MED-5019": 1, "MED-5020": 1}, "PLAIN-1667": {"MED-2527": 1, "MED-3454": 1, "MED-3163": 1, "MED-3156": 1, "MED-3164": 1, "MED-3157": 1, "MED-3158": 1, "MED-3450": 1, "MED-3451": 1, "MED-3165": 1, "MED-3161": 1, "MED-3166": 1, "MED-3168": 1, "MED-3466": 1, "MED-3178": 1, "MED-3169": 1, "MED-3172": 1, "MED-3181": 1, "MED-3174": 1, "MED-3175": 1, "MED-3176": 1, "MED-3182": 1, "MED-3220": 1, "MED-3221": 1, "MED-3231": 1, "MED-3230": 1, "MED-3236": 1, "MED-3217": 1, "MED-3237": 1, "MED-3455": 1, "MED-3456": 1, "MED-3449": 1, "MED-3457": 1, "MED-3458": 1, "MED-3453": 1, "MED-3534": 1, "MED-3464": 1, "MED-3460": 1, "MED-3465": 1, "MED-3461": 1, "MED-3473": 1, "MED-3469": 1, "MED-3481": 1, "MED-3471": 1, "MED-3518": 1, "MED-3519": 1, "MED-3532": 1, "MED-3533": 1, "MED-3524": 1, "MED-3527": 1, "MED-3539": 1, "MED-1342": 1, "MED-1343": 1, "MED-1344": 1, "MED-1345": 1, "MED-1355": 1, "MED-1347": 1, "MED-1348": 1, "MED-1349": 1, "MED-1350": 1, "MED-1351": 1, "MED-1352": 1, "MED-1353": 1, "MED-1354": 1, "MED-1463": 1, "MED-1454": 1, "MED-1455": 1, "MED-1456": 1, "MED-1457": 1, "MED-1458": 1, "MED-1459": 1, "MED-1460": 1, "MED-1461": 1, "MED-1474": 1, "MED-1470": 1, "MED-1471": 1, "MED-1472": 1, "MED-1473": 1, "MED-1475": 1, "MED-2279": 1, "MED-2280": 1, "MED-2281": 1, "MED-2282": 1, "MED-2284": 1, "MED-2285": 1, "MED-3535": 1, "MED-2287": 1, "MED-2524": 1, "MED-2525": 1, "MED-2526": 1, "MED-2528": 1, "MED-2529": 1, "MED-2530": 1, "MED-3170": 1, "MED-3171": 1, "MED-3173": 1, "MED-3177": 1, "MED-3179": 1, "MED-3180": 1, "MED-3226": 1, "MED-3228": 1, "MED-4722": 1, "MED-3215": 1, "MED-3216": 1, "MED-3227": 1, "MED-3229": 1, "MED-3233": 1, "MED-3452": 1, "MED-3459": 1, "MED-3462": 1, "MED-3467": 1, "MED-3478": 1, "MED-3520": 1, "MED-3521": 1, "MED-3522": 1, "MED-3523": 1, "MED-3525": 1, "MED-3526": 1, "MED-3528": 1, "MED-3530": 1, "MED-3531": 1, "MED-3771": 1, "MED-3772": 1, "MED-3773": 1, "MED-3774": 1, "MED-3775": 1, "MED-3776": 1, "MED-3941": 1, "MED-3942": 1, "MED-3943": 1, "MED-3944": 1, "MED-3945": 1, "MED-3946": 1, "MED-3947": 1, "MED-4407": 1, "MED-3949": 1, "MED-3950": 1, "MED-3951": 1, "MED-3952": 1, "MED-3953": 1, "MED-4224": 1, "MED-4226": 1, "MED-4228": 1, "MED-4404": 1, "MED-4405": 1, "MED-4508": 1, "MED-4504": 1, "MED-4505": 1, "MED-4506": 1, "MED-4553": 1, "MED-4554": 1, "MED-4828": 1, "MED-4829": 1, "MED-4830": 1, "MED-4958": 1, "MED-4959": 1, "MED-4960": 1, "MED-4961": 1, "MED-5025": 1}, "PLAIN-1679": {"MED-5142": 1}, "PLAIN-1690": {"MED-3597": 1, "MED-3598": 1, "MED-3599": 1, "MED-4804": 1}, "PLAIN-1700": {"MED-2999": 1, "MED-2997": 1, "MED-2765": 1, "MED-3001": 1, "MED-3000": 1}, "PLAIN-1710": {"MED-3178": 1, "MED-3169": 1, "MED-3172": 1, "MED-3181": 1, "MED-3174": 1, "MED-3175": 1, "MED-3176": 1, "MED-3182": 1, "MED-3170": 1, "MED-3171": 1, "MED-3173": 1, "MED-3177": 1, "MED-3179": 1, "MED-3180": 1, "MED-4356": 1, "MED-4672": 1}, "PLAIN-1721": {"MED-1598": 1, "MED-1599": 1, "MED-1600": 1, "MED-1601": 1, "MED-1602": 1, "MED-1603": 1, "MED-1604": 1, "MED-1605": 1, "MED-1606": 1, "MED-1607": 1, "MED-4476": 1, "MED-1621": 1, "MED-1622": 1, "MED-1623": 1, "MED-1624": 1, "MED-1625": 1, "MED-1626": 1, "MED-1627": 1, "MED-1628": 1, "MED-5054": 1, "MED-1630": 1, "MED-1631": 1, "MED-4820": 1, "MED-4821": 1, "MED-4822": 1, "MED-4823": 1, "MED-4824": 1, "MED-4825": 1, "MED-4826": 1, "MED-4985": 1, "MED-4983": 1}, "PLAIN-1731": {"MED-2747": 1, "MED-2748": 1, "MED-2749": 1}, "PLAIN-1741": {"MED-2575": 1, "MED-2580": 1, "MED-2571": 1, "MED-2583": 1, "MED-2574": 1, "MED-2568": 1, "MED-2988": 1, "MED-2559": 1, "MED-4319": 1, "MED-2570": 1, "MED-2573": 1, "MED-2585": 1, "MED-2577": 1, "MED-2578": 1, "MED-2572": 1, "MED-2581": 1, "MED-2582": 1, "MED-2584": 1, "MED-2882": 1, "MED-2898": 1, "MED-2884": 1, "MED-2885": 1, "MED-2886": 1, "MED-2888": 1, "MED-2889": 1, "MED-2895": 1, "MED-2896": 1, "MED-2899": 1, "MED-2900": 1, "MED-2901": 1, "MED-2980": 1, "MED-2990": 1, "MED-2986": 1, "MED-2985": 1, "MED-2987": 1, "MED-3276": 1, "MED-3277": 1, "MED-3271": 1, "MED-3278": 1, "MED-3272": 1, "MED-3274": 1, "MED-3275": 1, "MED-3282": 1, "MED-3427": 1, "MED-3429": 1, "MED-3432": 1, "MED-3433": 1, "MED-3394": 1, "MED-3396": 1, "MED-3397": 1, "MED-3399": 1, "MED-3423": 1, "MED-3436": 1, "MED-3424": 1, "MED-3430": 1, "MED-1190": 1, "MED-5293": 1, "MED-1192": 1, "MED-1193": 1, "MED-1194": 1, "MED-1208": 1, "MED-1209": 1, "MED-1210": 1, "MED-1211": 1, "MED-1212": 1, "MED-1213": 1, "MED-5303": 1, "MED-1231": 1, "MED-1232": 1, "MED-1233": 1, "MED-1234": 1, "MED-1375": 1, "MED-1362": 1, "MED-1363": 1, "MED-1383": 1, "MED-1365": 1, "MED-1366": 1, "MED-1643": 1, "MED-1374": 1, "MED-1380": 1, "MED-5301": 1, "MED-1371": 1, "MED-5271": 1, "MED-1373": 1, "MED-1376": 1, "MED-1377": 1, "MED-1378": 1, "MED-2505": 1, "MED-1381": 1, "MED-2593": 1, "MED-1393": 1, "MED-4710": 1, "MED-1386": 1, "MED-1387": 1, "MED-1388": 1, "MED-1389": 1, "MED-1390": 1, "MED-1394": 1, "MED-1399": 1, "MED-1395": 1, "MED-1489": 1, "MED-1397": 1, "MED-1398": 1, "MED-1400": 1, "MED-1401": 1, "MED-1402": 1, "MED-1403": 1, "MED-1404": 1, "MED-1405": 1, "MED-1406": 1, "MED-1407": 1, "MED-1408": 1, "MED-1409": 1, "MED-1410": 1, "MED-1411": 1, "MED-1463": 1, "MED-1454": 1, "MED-1455": 1, "MED-1456": 1, "MED-1457": 1, "MED-1458": 1, "MED-1459": 1, "MED-1460": 1, "MED-1461": 1, "MED-1474": 1, "MED-1476": 1, "MED-2432": 1, "MED-1478": 1, "MED-1479": 1, "MED-1507": 1, "MED-1508": 1, "MED-1509": 1, "MED-1510": 1, "MED-1511": 1, "MED-1512": 1, "MED-1513": 1, "MED-1514": 1, "MED-1515": 1, "MED-1516": 1, "MED-1517": 1, "MED-1518": 1, "MED-1527": 1, "MED-1528": 1, "MED-1529": 1, "MED-1530": 1, "MED-1531": 1, "MED-1532": 1, "MED-1609": 1, "MED-1610": 1, "MED-1611": 1, "MED-1612": 1, "MED-1613": 1, "MED-1614": 1, "MED-1615": 1, "MED-1616": 1, "MED-1617": 1, "MED-1618": 1, "MED-1619": 1, "MED-1620": 1, "MED-1703": 1, "MED-1699": 1, "MED-1700": 1, "MED-1701": 1, "MED-1702": 1, "MED-1775": 1, "MED-1776": 1, "MED-1777": 1, "MED-1778": 1, "MED-1779": 1, "MED-1780": 1, "MED-1781": 1, "MED-1782": 1, "MED-1783": 1, "MED-1784": 1, "MED-1785": 1, "MED-1786": 1, "MED-1787": 1, "MED-1788": 1, "MED-2529": 1, "MED-2524": 1, "MED-2525": 1, "MED-2527": 1, "MED-2530": 1, "MED-2528": 1, "MED-1998": 1, "MED-1985": 1, "MED-1986": 1, "MED-1987": 1, "MED-1988": 1, "MED-1999": 1, "MED-1990": 1, "MED-1991": 1, "MED-1992": 1, "MED-1993": 1, "MED-1994": 1, "MED-2221": 1, "MED-1996": 1, "MED-1997": 1, "MED-2150": 1, "MED-4646": 1, "MED-2152": 1, "MED-2153": 1, "MED-2248": 1, "MED-2249": 1, "MED-2250": 1, "MED-2251": 1, "MED-2252": 1, "MED-2253": 1, "MED-2254": 1, "MED-2255": 1, "MED-2256": 1, "MED-2257": 1, "MED-2258": 1, "MED-2259": 1, "MED-2260": 1, "MED-2261": 1, "MED-2262": 1, "MED-2263": 1, "MED-2264": 1, "MED-2376": 1, "MED-2378": 1, "MED-2379": 1, "MED-2380": 1, "MED-2381": 1, "MED-2382": 1, "MED-2383": 1, "MED-2384": 1, "MED-2427": 1, "MED-2428": 1, "MED-2429": 1, "MED-2430": 1, "MED-2431": 1, "MED-2440": 1, "MED-2445": 1, "MED-2446": 1, "MED-2458": 1, "MED-2448": 1, "MED-2449": 1, "MED-2450": 1, "MED-2451": 1, "MED-2452": 1, "MED-2453": 1, "MED-2472": 1, "MED-2579": 1, "MED-2544": 1, "MED-2546": 1, "MED-2592": 1, "MED-2594": 1, "MED-2595": 1, "MED-2596": 1, "MED-2597": 1, "MED-2890": 1, "MED-2891": 1, "MED-2892": 1, "MED-2893": 1, "MED-2894": 1, "MED-4384": 1, "MED-2979": 1, "MED-2982": 1, "MED-2983": 1, "MED-2984": 1, "MED-2989": 1, "MED-3270": 1, "MED-3273": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3283": 1, "MED-3422": 1, "MED-3391": 1, "MED-3395": 1, "MED-3398": 1, "MED-3425": 1, "MED-3426": 1, "MED-3428": 1, "MED-3407": 1, "MED-3434": 1, "MED-3435": 1, "MED-3437": 1, "MED-3438": 1, "MED-3417": 1, "MED-3439": 1, "MED-3440": 1, "MED-3420": 1, "MED-3421": 1, "MED-3518": 1, "MED-3519": 1, "MED-3520": 1, "MED-3521": 1, "MED-3522": 1, "MED-3523": 1, "MED-3524": 1, "MED-3525": 1, "MED-3526": 1, "MED-3527": 1, "MED-3528": 1, "MED-3539": 1, "MED-3530": 1, "MED-3531": 1, "MED-3532": 1, "MED-3533": 1, "MED-3534": 1, "MED-3535": 1, "MED-3617": 1, "MED-3618": 1, "MED-3620": 1, "MED-3621": 1, "MED-3615": 1, "MED-3622": 1, "MED-3815": 1, "MED-3816": 1, "MED-3817": 1, "MED-3818": 1, "MED-3819": 1, "MED-3820": 1, "MED-3821": 1, "MED-3822": 1, "MED-3841": 1, "MED-3842": 1, "MED-3843": 1, "MED-3844": 1, "MED-3845": 1, "MED-3846": 1, "MED-3847": 1, "MED-3848": 1, "MED-3849": 1, "MED-3850": 1, "MED-4094": 1, "MED-3852": 1, "MED-3853": 1, "MED-3854": 1, "MED-3855": 1, "MED-3856": 1, "MED-3857": 1, "MED-3896": 1, "MED-3897": 1, "MED-3906": 1, "MED-3908": 1, "MED-3900": 1, "MED-4289": 1, "MED-3910": 1, "MED-4715": 1, "MED-4165": 1, "MED-4203": 1, "MED-4299": 1, "MED-4205": 1, "MED-4206": 1, "MED-4249": 1, "MED-4250": 1, "MED-4258": 1, "MED-4290": 1, "MED-4261": 1, "MED-4286": 1, "MED-4281": 1, "MED-4284": 1, "MED-4292": 1, "MED-4287": 1, "MED-4288": 1, "MED-4291": 1, "MED-4295": 1, "MED-4846": 1, "MED-4296": 1, "MED-4298": 1, "MED-4300": 1, "MED-4301": 1, "MED-4302": 1, "MED-4313": 1, "MED-4314": 1, "MED-4315": 1, "MED-4320": 1, "MED-4324": 1, "MED-4349": 1, "MED-4345": 1, "MED-4346": 1, "MED-4406": 1, "MED-4514": 1, "MED-4515": 1, "MED-4555": 1, "MED-4600": 1, "MED-4619": 1, "MED-4620": 1, "MED-4621": 1, "MED-4645": 1, "MED-4647": 1, "MED-4686": 1, "MED-4690": 1, "MED-4691": 1, "MED-4692": 1, "MED-4693": 1, "MED-4694": 1, "MED-4695": 1, "MED-4696": 1, "MED-4705": 1, "MED-4706": 1, "MED-4707": 1, "MED-4708": 1, "MED-4709": 1, "MED-4727": 1, "MED-4728": 1, "MED-4832": 1, "MED-4833": 1, "MED-4834": 1, "MED-4835": 1, "MED-4925": 1, "MED-4924": 1, "MED-4952": 1, "MED-5009": 1, "MED-5010": 1, "MED-5011": 1, "MED-5017": 1, "MED-5064": 1, "MED-5065": 1, "MED-5072": 1, "MED-5073": 1, "MED-5083": 1, "MED-5084": 1, "MED-5111": 1, "MED-5136": 1, "MED-5153": 1, "MED-5155": 1}, "PLAIN-1752": {"MED-2098": 1}, "PLAIN-1762": {"MED-3714": 1, "MED-3715": 1, "MED-3716": 1, "MED-3717": 1, "MED-3718": 1, "MED-3719": 1, "MED-3720": 1, "MED-3721": 1}, "PLAIN-1772": {"MED-4727": 1, "MED-4728": 1}, "PLAIN-1784": {"MED-4687": 1, "MED-4689": 1}, "PLAIN-1794": {"MED-5041": 1, "MED-5042": 1}, "PLAIN-1805": {"MED-1166": 1, "MED-1167": 1, "MED-1178": 1, "MED-1169": 1, "MED-1170": 1, "MED-1171": 1, "MED-1172": 1, "MED-1173": 1, "MED-1174": 1, "MED-1175": 1, "MED-1176": 1, "MED-1177": 1, "MED-1265": 1, "MED-1266": 1, "MED-1267": 1, "MED-1268": 1, "MED-1278": 1, "MED-1282": 1, "MED-1271": 1, "MED-1272": 1, "MED-1273": 1, "MED-1274": 1, "MED-1287": 1, "MED-1276": 1, "MED-1277": 1, "MED-1279": 1, "MED-1280": 1, "MED-1281": 1, "MED-1283": 1, "MED-1284": 1, "MED-1285": 1, "MED-4876": 1, "MED-1288": 1, "MED-1289": 1, "MED-1290": 1, "MED-1725": 1, "MED-1726": 1, "MED-2763": 1, "MED-1728": 1, "MED-1729": 1, "MED-1730": 1, "MED-1731": 1, "MED-1732": 1, "MED-1733": 1, "MED-1738": 1, "MED-1880": 1, "MED-1881": 1, "MED-2031": 1, "MED-2032": 1, "MED-2033": 1, "MED-2034": 1, "MED-2035": 1, "MED-2036": 1, "MED-2037": 1, "MED-2038": 1, "MED-2039": 1, "MED-2040": 1, "MED-2041": 1, "MED-2164": 1, "MED-2165": 1, "MED-2166": 1, "MED-2167": 1, "MED-2168": 1, "MED-2169": 1, "MED-2170": 1, "MED-2171": 1, "MED-2172": 1, "MED-2173": 1, "MED-2174": 1, "MED-2175": 1, "MED-2176": 1, "MED-2177": 1, "MED-2178": 1, "MED-3925": 1, "MED-3926": 1, "MED-3927": 1, "MED-3928": 1, "MED-3929": 1, "MED-3930": 1, "MED-3931": 1, "MED-3932": 1, "MED-3933": 1, "MED-4726": 1, "MED-3935": 1, "MED-3936": 1, "MED-3937": 1, "MED-3938": 1, "MED-3939": 1, "MED-3940": 1, "MED-4316": 1, "MED-4317": 1, "MED-4318": 1, "MED-4488": 1, "MED-4539": 1, "MED-4540": 1, "MED-4541": 1, "MED-4671": 1, "MED-4938": 1, "MED-4939": 1, "MED-4940": 1, "MED-4941": 1, "MED-4942": 1, "MED-4955": 1, "MED-4956": 1, "MED-5172": 1, "MED-5173": 1, "MED-5188": 1, "MED-5189": 1, "MED-5190": 1, "MED-5191": 1, "MED-5192": 1, "MED-5193": 1, "MED-5194": 1, "MED-5195": 1, "MED-5196": 1, "MED-5197": 1}, "PLAIN-1817": {"MED-3369": 1, "MED-3370": 1, "MED-3372": 1, "MED-3376": 1, "MED-3377": 1, "MED-2150": 1, "MED-4646": 1, "MED-2152": 1, "MED-2153": 1, "MED-3371": 1, "MED-3373": 1, "MED-3374": 1, "MED-3375": 1, "MED-4094": 1, "MED-4194": 1, "MED-4195": 1, "MED-4196": 1, "MED-4197": 1, "MED-4281": 1, "MED-4286": 1, "MED-4284": 1, "MED-4705": 1, "MED-4706": 1, "MED-4707": 1, "MED-4708": 1, "MED-4709": 1, "MED-4710": 1}, "PLAIN-1827": {"MED-4539": 1, "MED-4540": 1, "MED-4541": 1}, "PLAIN-1837": {"MED-2494": 1, "MED-2495": 1, "MED-2496": 1, "MED-2493": 1, "MED-2497": 1, "MED-2747": 1, "MED-2748": 1, "MED-2749": 1, "MED-3276": 1, "MED-3277": 1, "MED-3271": 1, "MED-3278": 1, "MED-3272": 1, "MED-3274": 1, "MED-3275": 1, "MED-3282": 1, "MED-3585": 1, "MED-3586": 1, "MED-3592": 1, "MED-1139": 1, "MED-1140": 1, "MED-1167": 1, "MED-1142": 1, "MED-1143": 1, "MED-1144": 1, "MED-1181": 1, "MED-1146": 1, "MED-1147": 1, "MED-1148": 1, "MED-1149": 1, "MED-1150": 1, "MED-1151": 1, "MED-1152": 1, "MED-1153": 1, "MED-1179": 1, "MED-1156": 1, "MED-1157": 1, "MED-1158": 1, "MED-1160": 1, "MED-1161": 1, "MED-1162": 1, "MED-1164": 1, "MED-1165": 1, "MED-1166": 1, "MED-1178": 1, "MED-1169": 1, "MED-1170": 1, "MED-1171": 1, "MED-1172": 1, "MED-1173": 1, "MED-1174": 1, "MED-1175": 1, "MED-1176": 1, "MED-1177": 1, "MED-1180": 1, "MED-1182": 1, "MED-1725": 1, "MED-1726": 1, "MED-2763": 1, "MED-1728": 1, "MED-1729": 1, "MED-1730": 1, "MED-1731": 1, "MED-1732": 1, "MED-1733": 1, "MED-1738": 1, "MED-1743": 1, "MED-1736": 1, "MED-1737": 1, "MED-1739": 1, "MED-1740": 1, "MED-1741": 1, "MED-1749": 1, "MED-1744": 1, "MED-1745": 1, "MED-1746": 1, "MED-1747": 1, "MED-1748": 1, "MED-1750": 1, "MED-1751": 1, "MED-1752": 1, "MED-1753": 1, "MED-1754": 1, "MED-1956": 1, "MED-1957": 1, "MED-1958": 1, "MED-1959": 1, "MED-1960": 1, "MED-1961": 1, "MED-1962": 1, "MED-1963": 1, "MED-2405": 1, "MED-2406": 1, "MED-2407": 1, "MED-3028": 1, "MED-2409": 1, "MED-2410": 1, "MED-2411": 1, "MED-2412": 1, "MED-2413": 1, "MED-2490": 1, "MED-2491": 1, "MED-2492": 1, "MED-3270": 1, "MED-3273": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3283": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3590": 1, "MED-3591": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-4726": 1, "MED-3935": 1, "MED-3936": 1, "MED-3937": 1, "MED-3938": 1, "MED-3939": 1, "MED-3940": 1, "MED-4174": 1, "MED-4175": 1, "MED-4176": 1, "MED-4177": 1, "MED-4178": 1, "MED-4179": 1, "MED-4181": 1, "MED-4187": 1, "MED-4532": 1, "MED-4533": 1, "MED-4534": 1, "MED-4535": 1, "MED-4536": 1, "MED-4548": 1, "MED-4729": 1, "MED-4730": 1, "MED-4731": 1, "MED-4732": 1, "MED-4804": 1, "MED-4913": 1, "MED-4914": 1, "MED-4915": 1, "MED-4938": 1, "MED-4939": 1, "MED-4940": 1, "MED-4941": 1, "MED-4942": 1, "MED-4943": 1, "MED-4951": 1, "MED-5029": 1, "MED-5053": 1, "MED-5054": 1, "MED-5055": 1, "MED-5102": 1, "MED-5104": 1, "MED-5108": 1, "MED-5109": 1, "MED-5110": 1, "MED-5167": 1, "MED-5168": 1, "MED-5188": 1, "MED-5189": 1, "MED-5190": 1, "MED-5191": 1, "MED-5192": 1, "MED-5193": 1, "MED-5194": 1, "MED-5195": 1, "MED-5196": 1, "MED-5197": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-1847": {"MED-3982": 1, "MED-3983": 1, "MED-3984": 1, "MED-4000": 1, "MED-4002": 1, "MED-5016": 1, "MED-4005": 1, "MED-4007": 1, "MED-4001": 1, "MED-4004": 1, "MED-4006": 1}, "PLAIN-1857": {"MED-2575": 1, "MED-2580": 1, "MED-2571": 1, "MED-2583": 1, "MED-2574": 1, "MED-2568": 1, "MED-2988": 1, "MED-2559": 1, "MED-4319": 1, "MED-2570": 1, "MED-2573": 1, "MED-2585": 1, "MED-2577": 1, "MED-2578": 1, "MED-2572": 1, "MED-2581": 1, "MED-2582": 1, "MED-2584": 1, "MED-2579": 1, "MED-2544": 1, "MED-2546": 1, "MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3089": 1, "MED-3090": 1, "MED-3091": 1, "MED-3092": 1, "MED-3093": 1, "MED-3094": 1, "MED-3095": 1, "MED-3096": 1}, "PLAIN-1867": {"MED-3742": 1, "MED-3743": 1, "MED-3744": 1, "MED-3745": 1, "MED-3746": 1, "MED-3747": 1, "MED-3748": 1}, "PLAIN-1877": {"MED-3585": 1, "MED-3586": 1, "MED-3592": 1, "MED-3598": 1, "MED-3597": 1, "MED-3671": 1, "MED-3676": 1, "MED-4481": 1, "MED-3728": 1, "MED-3730": 1, "MED-3722": 1, "MED-3723": 1, "MED-3788": 1, "MED-3796": 1, "MED-3778": 1, "MED-3801": 1, "MED-3794": 1, "MED-666": 1, "MED-3811": 1, "MED-1116": 1, "MED-4851": 1, "MED-1118": 1, "MED-3852": 1, "MED-1120": 1, "MED-1121": 1, "MED-1122": 1, "MED-1123": 1, "MED-1124": 1, "MED-1125": 1, "MED-1126": 1, "MED-1127": 1, "MED-1128": 1, "MED-1129": 1, "MED-1130": 1, "MED-1131": 1, "MED-4849": 1}, "PLAIN-1887": {"MED-5159": 1, "MED-5160": 1, "MED-5161": 1, "MED-5162": 1}, "PLAIN-1897": {"MED-4991": 1, "MED-4992": 1}, "PLAIN-1909": {"MED-3886": 1, "MED-3882": 1, "MED-3889": 1, "MED-3891": 1, "MED-2940": 1, "MED-2941": 1, "MED-2943": 1, "MED-2944": 1, "MED-2966": 1, "MED-2968": 1, "MED-2970": 1, "MED-2972": 1, "MED-4981": 1, "MED-2991": 1, "MED-332": 1, "MED-3090": 1, "MED-334": 1, "MED-335": 1, "MED-3092": 1, "MED-3094": 1, "MED-3109": 1, "MED-3098": 1, "MED-3099": 1, "MED-3100": 1, "MED-3112": 1, "MED-3148": 1, "MED-3149": 1, "MED-4313": 1, "MED-3197": 1, "MED-3198": 1, "MED-3220": 1, "MED-3221": 1, "MED-3231": 1, "MED-3230": 1, "MED-3236": 1, "MED-3217": 1, "MED-3237": 1, "MED-729": 1, "MED-730": 1, "MED-731": 1, "MED-732": 1, "MED-1208": 1, "MED-1209": 1, "MED-1210": 1, "MED-1211": 1, "MED-1212": 1, "MED-1213": 1, "MED-5303": 1, "MED-1215": 1, "MED-1216": 1, "MED-1217": 1, "MED-1218": 1, "MED-1219": 1, "MED-1220": 1, "MED-1221": 1, "MED-4797": 1, "MED-1318": 1, "MED-1319": 1, "MED-1320": 1, "MED-1321": 1, "MED-1322": 1, "MED-1323": 1, "MED-1324": 1, "MED-1612": 1, "MED-1326": 1, "MED-1327": 1, "MED-1328": 1, "MED-1329": 1, "MED-1330": 1, "MED-1331": 1, "MED-1332": 1, "MED-1333": 1, "MED-1334": 1, "MED-1335": 1, "MED-1795": 1, "MED-1431": 1, "MED-1432": 1, "MED-1433": 1, "MED-1434": 1, "MED-1435": 1, "MED-1436": 1, "MED-1437": 1, "MED-1438": 1, "MED-1439": 1, "MED-1440": 1, "MED-1476": 1, "MED-2432": 1, "MED-1478": 1, "MED-1479": 1, "MED-1598": 1, "MED-1599": 1, "MED-1600": 1, "MED-1601": 1, "MED-1602": 1, "MED-1603": 1, "MED-1604": 1, "MED-1605": 1, "MED-1606": 1, "MED-1607": 1, "MED-4476": 1, "MED-1609": 1, "MED-1610": 1, "MED-1611": 1, "MED-1613": 1, "MED-1614": 1, "MED-1615": 1, "MED-1616": 1, "MED-1617": 1, "MED-1618": 1, "MED-1619": 1, "MED-1620": 1, "MED-1966": 1, "MED-2967": 1, "MED-1968": 1, "MED-1977": 1, "MED-1978": 1, "MED-1979": 1, "MED-1980": 1, "MED-1981": 1, "MED-1982": 1, "MED-1983": 1, "MED-2000": 1, "MED-2001": 1, "MED-2002": 1, "MED-2003": 1, "MED-2004": 1, "MED-2005": 1, "MED-2006": 1, "MED-2007": 1, "MED-2033": 1, "MED-2013": 1, "MED-2014": 1, "MED-2027": 1, "MED-2016": 1, "MED-2037": 1, "MED-2164": 1, "MED-2165": 1, "MED-2166": 1, "MED-2167": 1, "MED-2168": 1, "MED-2169": 1, "MED-2170": 1, "MED-2171": 1, "MED-2172": 1, "MED-2173": 1, "MED-2174": 1, "MED-2175": 1, "MED-2176": 1, "MED-2177": 1, "MED-2178": 1, "MED-2337": 1, "MED-2338": 1, "MED-2339": 1, "MED-2340": 1, "MED-2341": 1, "MED-2355": 1, "MED-2356": 1, "MED-2344": 1, "MED-2345": 1, "MED-2346": 1, "MED-2347": 1, "MED-2348": 1, "MED-2349": 1, "MED-2350": 1, "MED-2351": 1, "MED-2352": 1, "MED-2353": 1, "MED-2354": 1, "MED-2357": 1, "MED-3316": 1, "MED-2359": 1, "MED-2360": 1, "MED-2361": 1, "MED-2362": 1, "MED-2363": 1, "MED-2364": 1, "MED-2365": 1, "MED-2366": 1, "MED-2367": 1, "MED-2368": 1, "MED-2369": 1, "MED-2488": 1, "MED-2489": 1, "MED-2490": 1, "MED-2491": 1, "MED-2492": 1, "MED-3880": 1, "MED-3887": 1, "MED-4132": 1, "MED-3888": 1, "MED-4136": 1, "MED-3892": 1, "MED-3884": 1, "MED-2843": 1, "MED-2844": 1, "MED-2845": 1, "MED-2846": 1, "MED-2847": 1, "MED-2848": 1, "MED-2849": 1, "MED-2850": 1, "MED-2851": 1, "MED-2852": 1, "MED-2853": 1, "MED-2938": 1, "MED-2939": 1, "MED-2942": 1, "MED-2945": 1, "MED-2946": 1, "MED-2947": 1, "MED-4523": 1, "MED-2969": 1, "MED-2971": 1, "MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3089": 1, "MED-3091": 1, "MED-3093": 1, "MED-3095": 1, "MED-3096": 1, "MED-3102": 1, "MED-3169": 1, "MED-3170": 1, "MED-3171": 1, "MED-3172": 1, "MED-3173": 1, "MED-3174": 1, "MED-3175": 1, "MED-3176": 1, "MED-3177": 1, "MED-3178": 1, "MED-3179": 1, "MED-3180": 1, "MED-3181": 1, "MED-3182": 1, "MED-3226": 1, "MED-3228": 1, "MED-4722": 1, "MED-3215": 1, "MED-3216": 1, "MED-3227": 1, "MED-3229": 1, "MED-3233": 1, "MED-3232": 1, "MED-3235": 1, "MED-3238": 1, "MED-3697": 1, "MED-3699": 1, "MED-3241": 1, "MED-3247": 1, "MED-3248": 1, "MED-3249": 1, "MED-3250": 1, "MED-3251": 1, "MED-3252": 1, "MED-3369": 1, "MED-3348": 1, "MED-3371": 1, "MED-3350": 1, "MED-3351": 1, "MED-3352": 1, "MED-3353": 1, "MED-3354": 1, "MED-3355": 1, "MED-3356": 1, "MED-3373": 1, "MED-3358": 1, "MED-3359": 1, "MED-3449": 1, "MED-3450": 1, "MED-3451": 1, "MED-3452": 1, "MED-3453": 1, "MED-3454": 1, "MED-3455": 1, "MED-3456": 1, "MED-3457": 1, "MED-3458": 1, "MED-3567": 1, "MED-3885": 1, "MED-3570": 1, "MED-3572": 1, "MED-3890": 1, "MED-3577": 1, "MED-3585": 1, "MED-3586": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3590": 1, "MED-3591": 1, "MED-3592": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-3597": 1, "MED-3598": 1, "MED-3599": 1, "MED-3648": 1, "MED-3649": 1, "MED-3651": 1, "MED-3652": 1, "MED-3653": 1, "MED-3722": 1, "MED-3723": 1, "MED-3724": 1, "MED-3744": 1, "MED-3726": 1, "MED-4481": 1, "MED-3728": 1, "MED-3729": 1, "MED-3730": 1, "MED-3731": 1, "MED-3732": 1, "MED-3769": 1, "MED-3770": 1, "MED-3791": 1, "MED-3792": 1, "MED-3793": 1, "MED-3794": 1, "MED-3795": 1, "MED-3796": 1, "MED-3797": 1, "MED-3798": 1, "MED-3799": 1, "MED-3800": 1, "MED-3801": 1, "MED-3925": 1, "MED-3926": 1, "MED-3927": 1, "MED-3928": 1, "MED-3929": 1, "MED-3930": 1, "MED-3931": 1, "MED-3932": 1, "MED-3933": 1, "MED-3960": 1, "MED-3961": 1, "MED-3962": 1, "MED-3963": 1, "MED-3964": 1, "MED-3965": 1, "MED-3966": 1, "MED-3967": 1, "MED-3968": 1, "MED-3969": 1, "MED-3970": 1, "MED-4037": 1, "MED-4038": 1, "MED-4976": 1, "MED-4040": 1, "MED-5197": 1, "MED-4049": 1, "MED-4053": 1, "MED-4051": 1, "MED-5116": 1, "MED-4047": 1, "MED-4055": 1, "MED-4050": 1, "MED-4058": 1, "MED-4060": 1, "MED-4057": 1, "MED-4059": 1, "MED-4075": 1, "MED-4068": 1, "MED-4069": 1, "MED-4070": 1, "MED-4072": 1, "MED-4071": 1, "MED-4073": 1, "MED-4091": 1, "MED-4089": 1, "MED-5071": 1, "MED-4128": 1, "MED-4129": 1, "MED-4130": 1, "MED-4131": 1, "MED-4133": 1, "MED-4134": 1, "MED-4135": 1, "MED-4137": 1, "MED-4138": 1, "MED-4139": 1, "MED-4140": 1, "MED-4141": 1, "MED-4142": 1, "MED-4143": 1, "MED-4167": 1, "MED-4168": 1, "MED-4182": 1, "MED-4183": 1, "MED-4261": 1, "MED-4300": 1, "MED-4301": 1, "MED-4319": 1, "MED-4320": 1, "MED-4335": 1, "MED-4336": 1, "MED-4337": 1, "MED-4356": 1, "MED-4357": 1, "MED-4969": 1, "MED-4363": 1, "MED-4364": 1, "MED-4436": 1, "MED-4437": 1, "MED-4450": 1, "MED-4451": 1, "MED-4470": 1, "MED-4471": 1, "MED-4472": 1, "MED-4474": 1, "MED-4475": 1, "MED-4477": 1, "MED-4479": 1, "MED-4480": 1, "MED-4482": 1, "MED-4483": 1, "MED-4484": 1, "MED-4485": 1, "MED-4486": 1, "MED-4487": 1, "MED-4488": 1, "MED-4489": 1, "MED-4490": 1, "MED-4491": 1, "MED-4492": 1, "MED-4493": 1, "MED-4494": 1, "MED-4556": 1, "MED-4559": 1, "MED-4594": 1, "MED-4593": 1, "MED-4672": 1, "MED-4719": 1, "MED-4812": 1, "MED-4813": 1, "MED-4814": 1, "MED-4815": 1, "MED-4816": 1, "MED-4817": 1, "MED-4818": 1, "MED-4819": 1, "MED-4820": 1, "MED-4821": 1, "MED-4912": 1, "MED-4911": 1, "MED-4973": 1, "MED-4974": 1, "MED-4975": 1, "MED-4980": 1, "MED-5072": 1, "MED-5069": 1, "MED-5070": 1, "MED-5094": 1, "MED-5095": 1, "MED-5096": 1, "MED-5102": 1, "MED-5104": 1, "MED-5110": 1}, "PLAIN-1919": {"MED-3314": 1, "MED-3306": 1, "MED-3307": 1, "MED-3316": 1, "MED-3311": 1, "MED-3292": 1, "MED-3317": 1, "MED-3294": 1, "MED-3320": 1, "MED-3295": 1, "MED-3321": 1, "MED-3315": 1, "MED-3305": 1, "MED-3308": 1, "MED-3310": 1, "MED-3312": 1, "MED-3318": 1, "MED-3313": 1, "MED-3288": 1, "MED-4433": 1, "MED-3302": 1, "MED-3309": 1, "MED-3319": 1, "MED-4431": 1, "MED-4819": 1, "MED-4436": 1, "MED-4818": 1}, "PLAIN-1929": {"MED-4511": 1, "MED-4512": 1, "MED-4513": 1, "MED-4665": 1, "MED-4666": 1, "MED-4667": 1, "MED-4668": 1, "MED-4685": 1, "MED-4772": 1, "MED-4773": 1}, "PLAIN-1940": {"MED-3801": 1, "MED-3794": 1, "MED-666": 1, "MED-3791": 1, "MED-3792": 1, "MED-3793": 1, "MED-3795": 1, "MED-3796": 1, "MED-3797": 1, "MED-3798": 1, "MED-3799": 1, "MED-3800": 1}, "PLAIN-1950": {"MED-3896": 1, "MED-3897": 1, "MED-3906": 1, "MED-3908": 1, "MED-3900": 1, "MED-4289": 1, "MED-3910": 1, "MED-4715": 1, "MED-3904": 1, "MED-5175": 1, "MED-3907": 1, "MED-4250": 1, "MED-4603": 1}, "PLAIN-1962": {"MED-2827": 1, "MED-2830": 1, "MED-2826": 1, "MED-2831": 1, "MED-2999": 1, "MED-3000": 1, "MED-3001": 1, "MED-3617": 1, "MED-3618": 1, "MED-3620": 1, "MED-3621": 1, "MED-3615": 1, "MED-3622": 1}, "PLAIN-1972": {"MED-2790": 1, "MED-2792": 1, "MED-2793": 1, "MED-2794": 1, "MED-4390": 1, "MED-2800": 1, "MED-2791": 1, "MED-2825": 1, "MED-4539": 1, "MED-4540": 1, "MED-4541": 1}, "PLAIN-1983": {"MED-2513": 1, "MED-2502": 1, "MED-2498": 1, "MED-2504": 1, "MED-2505": 1, "MED-2507": 1, "MED-5239": 1, "MED-2509": 1, "MED-2519": 1, "MED-2510": 1, "MED-2511": 1, "MED-5237": 1, "MED-2517": 1, "MED-2518": 1, "MED-2520": 1, "MED-2521": 1, "MED-2127": 1, "MED-2128": 1, "MED-2129": 1, "MED-2130": 1, "MED-2132": 1, "MED-2134": 1, "MED-2135": 1, "MED-2136": 1, "MED-2137": 1, "MED-2138": 1, "MED-2139": 1, "MED-2501": 1, "MED-2506": 1, "MED-2512": 1, "MED-2514": 1, "MED-2943": 1}, "PLAIN-1995": {"MED-3918": 1, "MED-3919": 1, "MED-3920": 1, "MED-3921": 1, "MED-3922": 1, "MED-3923": 1, "MED-3924": 1, "MED-4864": 1, "MED-5122": 1, "MED-5120": 1, "MED-5121": 1}, "PLAIN-2009": {"MED-4828": 1, "MED-4829": 1, "MED-4830": 1, "MED-4958": 1, "MED-4959": 1, "MED-4960": 1, "MED-4961": 1, "MED-5025": 1}, "PLAIN-2019": {"MED-4575": 1, "MED-4573": 1, "MED-4574": 1}, "PLAIN-2030": {"MED-4049": 1, "MED-4053": 1, "MED-4051": 1, "MED-5116": 1, "MED-4047": 1, "MED-4055": 1}, "PLAIN-2040": {"MED-2644": 1, "MED-2646": 1, "MED-2651": 1, "MED-118": 1, "MED-2652": 1, "MED-2653": 1, "MED-2655": 1, "MED-2659": 1, "MED-2661": 1, "MED-2662": 1, "MED-2672": 1, "MED-2673": 1, "MED-2675": 1, "MED-2676": 1, "MED-2677": 1, "MED-2678": 1, "MED-2679": 1, "MED-301": 1, "MED-3012": 1, "MED-3013": 1, "MED-3030": 1, "MED-3029": 1, "MED-306": 1, "MED-3035": 1, "MED-3631": 1, "MED-3632": 1, "MED-3629": 1, "MED-3634": 1, "MED-1133": 1, "MED-1134": 1, "MED-1135": 1, "MED-1136": 1, "MED-1137": 1, "MED-1138": 1, "MED-1609": 1, "MED-1610": 1, "MED-1611": 1, "MED-1612": 1, "MED-1613": 1, "MED-1614": 1, "MED-1615": 1, "MED-1616": 1, "MED-1617": 1, "MED-1618": 1, "MED-1619": 1, "MED-1620": 1, "MED-1747": 1, "MED-1748": 1, "MED-1749": 1, "MED-1750": 1, "MED-1751": 1, "MED-1752": 1, "MED-1753": 1, "MED-1754": 1, "MED-2164": 1, "MED-2165": 1, "MED-2166": 1, "MED-2167": 1, "MED-2168": 1, "MED-2169": 1, "MED-2170": 1, "MED-2171": 1, "MED-2172": 1, "MED-2173": 1, "MED-2174": 1, "MED-2175": 1, "MED-2176": 1, "MED-2177": 1, "MED-2178": 1, "MED-2385": 1, "MED-2386": 1, "MED-2395": 1, "MED-2388": 1, "MED-2389": 1, "MED-2396": 1, "MED-2391": 1, "MED-2407": 1, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2649": 1, "MED-2650": 1, "MED-2654": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-2671": 1, "MED-2674": 1, "MED-3019": 1, "MED-3021": 1, "MED-3022": 1, "MED-3023": 1, "MED-3024": 1, "MED-3025": 1, "MED-3026": 1, "MED-3027": 1, "MED-3028": 1, "MED-3032": 1, "MED-3033": 1, "MED-3034": 1, "MED-4183": 1, "MED-4377": 1, "MED-4936": 1, "MED-4730": 1, "MED-4687": 1, "MED-4689": 1, "MED-4932": 1, "MED-4933": 1, "MED-4934": 1, "MED-4935": 1, "MED-4937": 1, "MED-4958": 1, "MED-4959": 1, "MED-4960": 1, "MED-4961": 1, "MED-4973": 1, "MED-4974": 1, "MED-4975": 1, "MED-5091": 1, "MED-5092": 1, "MED-5093": 1, "MED-5097": 1, "MED-5098": 1, "MED-5099": 1, "MED-5100": 1, "MED-5101": 1, "MED-5169": 1, "MED-5170": 1}, "PLAIN-2051": {"MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-2649": 1, "MED-2797": 1, "MED-2809": 1, "MED-2799": 1, "MED-2800": 1, "MED-2801": 1, "MED-2802": 1, "MED-2803": 1, "MED-2804": 1, "MED-2805": 1, "MED-2807": 1, "MED-3148": 1, "MED-3149": 1, "MED-4313": 1, "MED-3780": 1, "MED-3784": 1, "MED-3197": 1, "MED-3198": 1, "MED-3250": 1, "MED-3251": 1, "MED-3247": 1, "MED-3248": 1, "MED-3249": 1, "MED-3254": 1, "MED-3253": 1, "MED-3454": 1, "MED-3455": 1, "MED-3456": 1, "MED-3449": 1, "MED-3450": 1, "MED-3457": 1, "MED-3458": 1, "MED-3451": 1, "MED-3453": 1, "MED-3585": 1, "MED-3586": 1, "MED-3592": 1, "MED-1204": 1, "MED-1205": 1, "MED-1489": 1, "MED-1207": 1, "MED-1249": 1, "MED-1250": 1, "MED-4831": 1, "MED-1252": 1, "MED-1253": 1, "MED-1254": 1, "MED-1801": 1, "MED-1256": 1, "MED-1257": 1, "MED-1258": 1, "MED-1375": 1, "MED-1362": 1, "MED-1363": 1, "MED-1383": 1, "MED-1365": 1, "MED-1366": 1, "MED-1643": 1, "MED-1374": 1, "MED-1380": 1, "MED-5301": 1, "MED-1371": 1, "MED-5271": 1, "MED-1373": 1, "MED-1399": 1, "MED-1393": 1, "MED-1394": 1, "MED-1395": 1, "MED-1397": 1, "MED-1398": 1, "MED-1400": 1, "MED-1445": 1, "MED-1446": 1, "MED-1447": 1, "MED-1448": 1, "MED-1449": 1, "MED-1450": 1, "MED-1451": 1, "MED-1463": 1, "MED-1454": 1, "MED-1455": 1, "MED-1456": 1, "MED-1457": 1, "MED-1458": 1, "MED-1459": 1, "MED-1460": 1, "MED-1461": 1, "MED-1474": 1, "MED-1470": 1, "MED-1471": 1, "MED-1472": 1, "MED-1473": 1, "MED-1475": 1, "MED-1491": 1, "MED-1492": 1, "MED-1493": 1, "MED-1494": 1, "MED-1495": 1, "MED-1496": 1, "MED-1497": 1, "MED-1498": 1, "MED-1499": 1, "MED-1500": 1, "MED-1501": 1, "MED-1502": 1, "MED-1503": 1, "MED-1504": 1, "MED-1505": 1, "MED-1506": 1, "MED-1551": 1, "MED-1552": 1, "MED-1553": 1, "MED-1554": 1, "MED-1555": 1, "MED-1556": 1, "MED-1557": 1, "MED-1558": 1, "MED-1762": 1, "MED-1763": 1, "MED-1764": 1, "MED-1765": 1, "MED-1766": 1, "MED-1778": 1, "MED-1768": 1, "MED-4951": 1, "MED-1770": 1, "MED-1771": 1, "MED-1781": 1, "MED-1773": 1, "MED-1774": 1, "MED-1775": 1, "MED-1776": 1, "MED-1777": 1, "MED-1779": 1, "MED-1780": 1, "MED-1782": 1, "MED-1783": 1, "MED-1784": 1, "MED-1785": 1, "MED-1786": 1, "MED-1787": 1, "MED-1788": 1, "MED-1992": 1, "MED-1876": 1, "MED-1998": 1, "MED-1878": 1, "MED-1879": 1, "MED-1914": 1, "MED-1915": 1, "MED-1916": 1, "MED-1917": 1, "MED-1918": 1, "MED-1919": 1, "MED-1920": 1, "MED-1921": 1, "MED-1922": 1, "MED-1923": 1, "MED-1924": 1, "MED-4877": 1, "MED-1926": 1, "MED-4878": 1, "MED-1928": 1, "MED-1929": 1, "MED-1930": 1, "MED-1931": 1, "MED-1932": 1, "MED-1933": 1, "MED-1934": 1, "MED-1935": 1, "MED-1936": 1, "MED-1949": 1, "MED-1950": 1, "MED-1951": 1, "MED-1952": 1, "MED-1953": 1, "MED-1954": 1, "MED-1955": 1, "MED-1966": 1, "MED-2967": 1, "MED-1968": 1, "MED-2000": 1, "MED-2001": 1, "MED-2002": 1, "MED-2003": 1, "MED-2004": 1, "MED-2005": 1, "MED-2006": 1, "MED-2007": 1, "MED-2008": 1, "MED-2009": 1, "MED-2010": 1, "MED-2011": 1, "MED-2076": 1, "MED-2077": 1, "MED-2078": 1, "MED-2079": 1, "MED-2080": 1, "MED-5293": 1, "MED-2082": 1, "MED-2083": 1, "MED-5303": 1, "MED-2085": 1, "MED-2179": 1, "MED-2180": 1, "MED-2181": 1, "MED-2182": 1, "MED-2183": 1, "MED-2184": 1, "MED-2211": 1, "MED-2212": 1, "MED-2213": 1, "MED-2214": 1, "MED-2215": 1, "MED-2216": 1, "MED-2217": 1, "MED-2218": 1, "MED-2374": 1, "MED-2237": 1, "MED-2238": 1, "MED-2288": 1, "MED-3137": 1, "MED-2290": 1, "MED-2291": 1, "MED-2292": 1, "MED-2293": 1, "MED-2294": 1, "MED-2295": 1, "MED-2296": 1, "MED-2461": 1, "MED-2464": 1, "MED-2644": 1, "MED-2646": 1, "MED-2468": 1, "MED-2469": 1, "MED-2471": 1, "MED-2472": 1, "MED-2652": 1, "MED-2474": 1, "MED-2475": 1, "MED-2476": 1, "MED-2655": 1, "MED-2479": 1, "MED-2482": 1, "MED-2659": 1, "MED-2484": 1, "MED-2661": 1, "MED-2662": 1, "MED-3687": 1, "MED-2647": 1, "MED-2648": 1, "MED-2650": 1, "MED-2651": 1, "MED-2653": 1, "MED-2654": 1, "MED-2819": 1, "MED-2999": 1, "MED-3000": 1, "MED-3001": 1, "MED-3786": 1, "MED-3781": 1, "MED-3789": 1, "MED-3193": 1, "MED-3783": 1, "MED-3785": 1, "MED-3252": 1, "MED-3255": 1, "MED-3452": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3590": 1, "MED-3591": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-3654": 1, "MED-3655": 1, "MED-3656": 1, "MED-3657": 1, "MED-3782": 1, "MED-3787": 1, "MED-3788": 1, "MED-3790": 1, "MED-4000": 1, "MED-4002": 1, "MED-5016": 1, "MED-4005": 1, "MED-4007": 1, "MED-4001": 1, "MED-4004": 1, "MED-4006": 1, "MED-4022": 1, "MED-4023": 1, "MED-4024": 1, "MED-4025": 1, "MED-4026": 1, "MED-4027": 1, "MED-4028": 1, "MED-4029": 1, "MED-4030": 1, "MED-4031": 1, "MED-4032": 1, "MED-4033": 1, "MED-4034": 1, "MED-4035": 1, "MED-4036": 1, "MED-4853": 1, "MED-4198": 1, "MED-4200": 1, "MED-4201": 1, "MED-4202": 1, "MED-4302": 1, "MED-4319": 1, "MED-4320": 1, "MED-4333": 1, "MED-4337": 1, "MED-4335": 1, "MED-4336": 1, "MED-4349": 1, "MED-4520": 1, "MED-4352": 1, "MED-4353": 1, "MED-4450": 1, "MED-4451": 1, "MED-4555": 1, "MED-4556": 1, "MED-4559": 1, "MED-4676": 1, "MED-4837": 1, "MED-4944": 1, "MED-5012": 1, "MED-5013": 1, "MED-5014": 1, "MED-5015": 1, "MED-5038": 1, "MED-5039": 1, "MED-5040": 1, "MED-5090": 1, "MED-5108": 1, "MED-5109": 1, "MED-5146": 1, "MED-5147": 1, "MED-5148": 1, "MED-5149": 1, "MED-5150": 1, "MED-5151": 1, "MED-5152": 1}, "PLAIN-2061": {"MED-2494": 1, "MED-2495": 1, "MED-2496": 1, "MED-2493": 1, "MED-2497": 1, "MED-2644": 1, "MED-2646": 1, "MED-2651": 1, "MED-118": 1, "MED-2652": 1, "MED-2653": 1, "MED-2655": 1, "MED-2659": 1, "MED-2661": 1, "MED-2662": 1, "MED-2750": 1, "MED-2751": 1, "MED-2752": 1, "MED-2754": 1, "MED-2755": 1, "MED-3028": 1, "MED-2913": 1, "MED-3021": 1, "MED-3012": 1, "MED-3033": 1, "MED-2906": 1, "MED-3034": 1, "MED-2940": 1, "MED-2941": 1, "MED-2943": 1, "MED-2944": 1, "MED-2966": 1, "MED-2968": 1, "MED-2970": 1, "MED-2972": 1, "MED-4981": 1, "MED-301": 1, "MED-3013": 1, "MED-3030": 1, "MED-3029": 1, "MED-306": 1, "MED-3035": 1, "MED-3023": 1, "MED-3031": 1, "MED-3020": 1, "MED-3109": 1, "MED-3098": 1, "MED-3099": 1, "MED-3100": 1, "MED-3112": 1, "MED-1265": 1, "MED-1266": 1, "MED-1267": 1, "MED-1268": 1, "MED-1278": 1, "MED-1282": 1, "MED-1271": 1, "MED-1272": 1, "MED-1273": 1, "MED-1274": 1, "MED-1287": 1, "MED-1276": 1, "MED-1277": 1, "MED-1279": 1, "MED-1280": 1, "MED-1281": 1, "MED-1283": 1, "MED-1284": 1, "MED-1285": 1, "MED-4876": 1, "MED-1288": 1, "MED-1289": 1, "MED-1290": 1, "MED-1318": 1, "MED-1319": 1, "MED-1320": 1, "MED-1321": 1, "MED-1322": 1, "MED-1323": 1, "MED-1324": 1, "MED-1612": 1, "MED-1326": 1, "MED-1327": 1, "MED-1328": 1, "MED-1329": 1, "MED-1330": 1, "MED-1331": 1, "MED-1332": 1, "MED-1333": 1, "MED-1334": 1, "MED-1335": 1, "MED-1795": 1, "MED-1374": 1, "MED-1375": 1, "MED-1376": 1, "MED-1377": 1, "MED-1378": 1, "MED-1380": 1, "MED-1568": 1, "MED-1569": 1, "MED-1570": 1, "MED-1571": 1, "MED-1572": 1, "MED-1573": 1, "MED-2248": 1, "MED-2249": 1, "MED-2250": 1, "MED-2251": 1, "MED-2252": 1, "MED-2253": 1, "MED-2254": 1, "MED-2255": 1, "MED-2256": 1, "MED-2257": 1, "MED-2258": 1, "MED-2259": 1, "MED-2260": 1, "MED-2261": 1, "MED-2262": 1, "MED-2263": 1, "MED-2264": 1, "MED-2272": 1, "MED-2273": 1, "MED-2274": 1, "MED-3464": 1, "MED-2276": 1, "MED-3535": 1, "MED-2278": 1, "MED-2337": 1, "MED-2338": 1, "MED-2339": 1, "MED-2340": 1, "MED-2341": 1, "MED-2355": 1, "MED-2356": 1, "MED-2344": 1, "MED-2345": 1, "MED-2346": 1, "MED-2347": 1, "MED-2348": 1, "MED-2349": 1, "MED-2350": 1, "MED-2405": 1, "MED-2394": 1, "MED-2395": 1, "MED-2396": 1, "MED-2397": 1, "MED-2398": 1, "MED-2407": 1, "MED-2400": 1, "MED-2402": 1, "MED-2411": 1, "MED-2404": 1, "MED-2406": 1, "MED-2409": 1, "MED-2410": 1, "MED-2412": 1, "MED-2413": 1, "MED-2488": 1, "MED-2489": 1, "MED-4551": 1, "MED-2643": 1, "MED-2645": 1, "MED-2647": 1, "MED-2648": 1, "MED-2627": 1, "MED-2649": 1, "MED-2650": 1, "MED-2654": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2660": 1, "MED-2753": 1, "MED-2756": 1, "MED-2904": 1, "MED-2905": 1, "MED-2907": 1, "MED-3024": 1, "MED-3025": 1, "MED-2910": 1, "MED-3027": 1, "MED-2917": 1, "MED-2921": 1, "MED-2938": 1, "MED-2939": 1, "MED-2942": 1, "MED-2945": 1, "MED-2946": 1, "MED-2947": 1, "MED-4523": 1, "MED-2967": 1, "MED-2951": 1, "MED-2969": 1, "MED-2971": 1, "MED-3019": 1, "MED-3022": 1, "MED-3026": 1, "MED-3032": 1, "MED-3102": 1, "MED-4313": 1, "MED-3148": 1, "MED-3149": 1, "MED-3786": 1, "MED-3780": 1, "MED-3781": 1, "MED-3789": 1, "MED-3193": 1, "MED-3783": 1, "MED-3784": 1, "MED-3785": 1, "MED-3197": 1, "MED-3198": 1, "MED-3226": 1, "MED-3228": 1, "MED-3231": 1, "MED-4722": 1, "MED-3215": 1, "MED-3216": 1, "MED-3217": 1, "MED-3237": 1, "MED-3227": 1, "MED-3220": 1, "MED-3221": 1, "MED-3229": 1, "MED-3230": 1, "MED-3233": 1, "MED-3236": 1, "MED-3305": 1, "MED-3308": 1, "MED-3288": 1, "MED-3310": 1, "MED-3312": 1, "MED-3313": 1, "MED-3315": 1, "MED-4433": 1, "MED-3318": 1, "MED-3302": 1, "MED-3449": 1, "MED-3450": 1, "MED-3451": 1, "MED-3452": 1, "MED-3453": 1, "MED-3454": 1, "MED-3455": 1, "MED-3456": 1, "MED-3457": 1, "MED-3458": 1, "MED-3585": 1, "MED-3586": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3590": 1, "MED-3591": 1, "MED-3592": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-3617": 1, "MED-3618": 1, "MED-3620": 1, "MED-3621": 1, "MED-3615": 1, "MED-3622": 1, "MED-3628": 1, "MED-3629": 1, "MED-3630": 1, "MED-3631": 1, "MED-3632": 1, "MED-3633": 1, "MED-3634": 1, "MED-3635": 1, "MED-3636": 1, "MED-3782": 1, "MED-3787": 1, "MED-3788": 1, "MED-3790": 1, "MED-3880": 1, "MED-4132": 1, "MED-3882": 1, "MED-4136": 1, "MED-3884": 1, "MED-3885": 1, "MED-3886": 1, "MED-3887": 1, "MED-3888": 1, "MED-3889": 1, "MED-3890": 1, "MED-3891": 1, "MED-3892": 1, "MED-3925": 1, "MED-3926": 1, "MED-3927": 1, "MED-3928": 1, "MED-3929": 1, "MED-3930": 1, "MED-3931": 1, "MED-3932": 1, "MED-3933": 1, "MED-3954": 1, "MED-3955": 1, "MED-3956": 1, "MED-4680": 1, "MED-3958": 1, "MED-3959": 1, "MED-3960": 1, "MED-3961": 1, "MED-3962": 1, "MED-3963": 1, "MED-3964": 1, "MED-3965": 1, "MED-3966": 1, "MED-3967": 1, "MED-3968": 1, "MED-3969": 1, "MED-3970": 1, "MED-4174": 1, "MED-4175": 1, "MED-4176": 1, "MED-4177": 1, "MED-4178": 1, "MED-4179": 1, "MED-4182": 1, "MED-4183": 1, "MED-4349": 1, "MED-4520": 1, "MED-4342": 1, "MED-4345": 1, "MED-4346": 1, "MED-4354": 1, "MED-4359": 1, "MED-4360": 1, "MED-4375": 1, "MED-4376": 1, "MED-4377": 1, "MED-4378": 1, "MED-4379": 1, "MED-4380": 1, "MED-4381": 1, "MED-4719": 1, "MED-4727": 1, "MED-4728": 1, "MED-4733": 1, "MED-4736": 1, "MED-4735": 1, "MED-4739": 1, "MED-4738": 1, "MED-4740": 1, "MED-4741": 1, "MED-4742": 1, "MED-4743": 1, "MED-4791": 1, "MED-4792": 1, "MED-4811": 1, "MED-4822": 1, "MED-4823": 1, "MED-4824": 1, "MED-4825": 1, "MED-4826": 1, "MED-4880": 1, "MED-4881": 1, "MED-4882": 1, "MED-4883": 1, "MED-4884": 1, "MED-4908": 1, "MED-4909": 1, "MED-4936": 1, "MED-4932": 1, "MED-4933": 1, "MED-4934": 1, "MED-4935": 1, "MED-4937": 1, "MED-4943": 1, "MED-4944": 1, "MED-4946": 1, "MED-4947": 1, "MED-4948": 1, "MED-4949": 1, "MED-4950": 1, "MED-4951": 1, "MED-4958": 1, "MED-4959": 1, "MED-4960": 1, "MED-4961": 1, "MED-4962": 1, "MED-4963": 1, "MED-4964": 1, "MED-4969": 1, "MED-4966": 1, "MED-4967": 1, "MED-4968": 1, "MED-4988": 1, "MED-4989": 1, "MED-5091": 1, "MED-5092": 1, "MED-5093": 1, "MED-5094": 1, "MED-5095": 1, "MED-5096": 1, "MED-5097": 1, "MED-5098": 1, "MED-5099": 1, "MED-5100": 1, "MED-5101": 1, "MED-5102": 1, "MED-5104": 1, "MED-5169": 1, "MED-5170": 1}, "PLAIN-2071": {"MED-3518": 1, "MED-3519": 1, "MED-3532": 1, "MED-3533": 1, "MED-3524": 1, "MED-3527": 1, "MED-3539": 1, "MED-3536": 1, "MED-3537": 1, "MED-5030": 1, "MED-3545": 1, "MED-3173": 1, "MED-3541": 1, "MED-3546": 1, "MED-3542": 1, "MED-3543": 1, "MED-3547": 1, "MED-1621": 1, "MED-1622": 1, "MED-1623": 1, "MED-1624": 1, "MED-1625": 1, "MED-1626": 1, "MED-1627": 1, "MED-1628": 1, "MED-5054": 1, "MED-1630": 1, "MED-1631": 1, "MED-3520": 1, "MED-3521": 1, "MED-3522": 1, "MED-3523": 1, "MED-3525": 1, "MED-3526": 1, "MED-3528": 1, "MED-3530": 1, "MED-3531": 1, "MED-3534": 1, "MED-3535": 1, "MED-3538": 1, "MED-3540": 1, "MED-3544": 1, "MED-4305": 1, "MED-4306": 1, "MED-4308": 1}, "PLAIN-2081": {"MED-4303": 1}, "PLAIN-2092": {"MED-1431": 1, "MED-1432": 1, "MED-1433": 1, "MED-1434": 1, "MED-1435": 1, "MED-1436": 1, "MED-1437": 1, "MED-1438": 1, "MED-1439": 1, "MED-1440": 1}, "PLAIN-2102": {"MED-2522": 1, "MED-2606": 1, "MED-2607": 1, "MED-2608": 1, "MED-2816": 1, "MED-2675": 1, "MED-2676": 1, "MED-2677": 1, "MED-2678": 1, "MED-2679": 1, "MED-2763": 1, "MED-2966": 1, "MED-2968": 1, "MED-2970": 1, "MED-2972": 1, "MED-4981": 1, "MED-3050": 1, "MED-3038": 1, "MED-3052": 1, "MED-3053": 1, "MED-3054": 1, "MED-3044": 1, "MED-3046": 1, "MED-3058": 1, "MED-3059": 1, "MED-3060": 1, "MED-3109": 1, "MED-3098": 1, "MED-3099": 1, "MED-3100": 1, "MED-3112": 1, "MED-1139": 1, "MED-1140": 1, "MED-1167": 1, "MED-1142": 1, "MED-1143": 1, "MED-1144": 1, "MED-1181": 1, "MED-1146": 1, "MED-1147": 1, "MED-1208": 1, "MED-1209": 1, "MED-1210": 1, "MED-1211": 1, "MED-1212": 1, "MED-1213": 1, "MED-5303": 1, "MED-1301": 1, "MED-1302": 1, "MED-1303": 1, "MED-1304": 1, "MED-1305": 1, "MED-1313": 1, "MED-1307": 1, "MED-1308": 1, "MED-1309": 1, "MED-1327": 1, "MED-1399": 1, "MED-1393": 1, "MED-1394": 1, "MED-1395": 1, "MED-1489": 1, "MED-1397": 1, "MED-1398": 1, "MED-1400": 1, "MED-2370": 1, "MED-1428": 1, "MED-1429": 1, "MED-1430": 1, "MED-1496": 1, "MED-1497": 1, "MED-1498": 1, "MED-1499": 1, "MED-1500": 1, "MED-1501": 1, "MED-1502": 1, "MED-1503": 1, "MED-1504": 1, "MED-1505": 1, "MED-1506": 1, "MED-1519": 1, "MED-1520": 1, "MED-1521": 1, "MED-1522": 1, "MED-1523": 1, "MED-1524": 1, "MED-1525": 1, "MED-1526": 1, "MED-1527": 1, "MED-1528": 1, "MED-1529": 1, "MED-1530": 1, "MED-1531": 1, "MED-1532": 1, "MED-1542": 1, "MED-1543": 1, "MED-1544": 1, "MED-1545": 1, "MED-1546": 1, "MED-2527": 1, "MED-1548": 1, "MED-1549": 1, "MED-1680": 1, "MED-1559": 1, "MED-1560": 1, "MED-3557": 1, "MED-3242": 1, "MED-1563": 1, "MED-1564": 1, "MED-1565": 1, "MED-3699": 1, "MED-1567": 1, "MED-1598": 1, "MED-1599": 1, "MED-1600": 1, "MED-1601": 1, "MED-1602": 1, "MED-1603": 1, "MED-1604": 1, "MED-1605": 1, "MED-1606": 1, "MED-1607": 1, "MED-4476": 1, "MED-1650": 1, "MED-1651": 1, "MED-2726": 1, "MED-2221": 1, "MED-4602": 1, "MED-1655": 1, "MED-1677": 1, "MED-1678": 1, "MED-1679": 1, "MED-1681": 1, "MED-1703": 1, "MED-1704": 1, "MED-1705": 1, "MED-1711": 1, "MED-1712": 1, "MED-1714": 1, "MED-1715": 1, "MED-1716": 1, "MED-1717": 1, "MED-1718": 1, "MED-1719": 1, "MED-1720": 1, "MED-1721": 1, "MED-1722": 1, "MED-1723": 1, "MED-1724": 1, "MED-1760": 1, "MED-1761": 1, "MED-1796": 1, "MED-1797": 1, "MED-1798": 1, "MED-1799": 1, "MED-1800": 1, "MED-1801": 1, "MED-1802": 1, "MED-1803": 1, "MED-1804": 1, "MED-4767": 1, "MED-1806": 1, "MED-1807": 1, "MED-1808": 1, "MED-4769": 1, "MED-1810": 1, "MED-2722": 1, "MED-2489": 1, "MED-1966": 1, "MED-2967": 1, "MED-1968": 1, "MED-2089": 1, "MED-2090": 1, "MED-2091": 1, "MED-2092": 1, "MED-2093": 1, "MED-2094": 1, "MED-2095": 1, "MED-2096": 1, "MED-2097": 1, "MED-2164": 1, "MED-2165": 1, "MED-2166": 1, "MED-2167": 1, "MED-2168": 1, "MED-2169": 1, "MED-2170": 1, "MED-2171": 1, "MED-2172": 1, "MED-2173": 1, "MED-2174": 1, "MED-2175": 1, "MED-2176": 1, "MED-2177": 1, "MED-2178": 1, "MED-2219": 1, "MED-2220": 1, "MED-2374": 1, "MED-2237": 1, "MED-2238": 1, "MED-2244": 1, "MED-2245": 1, "MED-2246": 1, "MED-2247": 1, "MED-2248": 1, "MED-2249": 1, "MED-2250": 1, "MED-2251": 1, "MED-2252": 1, "MED-2253": 1, "MED-2254": 1, "MED-2255": 1, "MED-2256": 1, "MED-2257": 1, "MED-2258": 1, "MED-2259": 1, "MED-2260": 1, "MED-2261": 1, "MED-2262": 1, "MED-2263": 1, "MED-2264": 1, "MED-2297": 1, "MED-2298": 1, "MED-2299": 1, "MED-2300": 1, "MED-2301": 1, "MED-2303": 1, "MED-2304": 1, "MED-2305": 1, "MED-2306": 1, "MED-2371": 1, "MED-2372": 1, "MED-4324": 1, "MED-2441": 1, "MED-2442": 1, "MED-2472": 1, "MED-2444": 1, "MED-2461": 1, "MED-4551": 1, "MED-2643": 1, "MED-2464": 1, "MED-2645": 1, "MED-2644": 1, "MED-2646": 1, "MED-2468": 1, "MED-2469": 1, "MED-2649": 1, "MED-2471": 1, "MED-2652": 1, "MED-2474": 1, "MED-2475": 1, "MED-2476": 1, "MED-2655": 1, "MED-2656": 1, "MED-2479": 1, "MED-2657": 1, "MED-2658": 1, "MED-2482": 1, "MED-2659": 1, "MED-2484": 1, "MED-2661": 1, "MED-2662": 1, "MED-3687": 1, "MED-2523": 1, "MED-2592": 1, "MED-2593": 1, "MED-2594": 1, "MED-2595": 1, "MED-2596": 1, "MED-2597": 1, "MED-2812": 1, "MED-2674": 1, "MED-4523": 1, "MED-2951": 1, "MED-2969": 1, "MED-2971": 1, "MED-3051": 1, "MED-3055": 1, "MED-3056": 1, "MED-3057": 1, "MED-3102": 1, "MED-3113": 1, "MED-4247": 1, "MED-3197": 1, "MED-3198": 1, "MED-3253": 1, "MED-3254": 1, "MED-3255": 1, "MED-3270": 1, "MED-3271": 1, "MED-3272": 1, "MED-3273": 1, "MED-3274": 1, "MED-3275": 1, "MED-3276": 1, "MED-3277": 1, "MED-3278": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3282": 1, "MED-3283": 1, "MED-3305": 1, "MED-3306": 1, "MED-3307": 1, "MED-3308": 1, "MED-3309": 1, "MED-3310": 1, "MED-3311": 1, "MED-3312": 1, "MED-3313": 1, "MED-3314": 1, "MED-3315": 1, "MED-3316": 1, "MED-3317": 1, "MED-3318": 1, "MED-3319": 1, "MED-3320": 1, "MED-3321": 1, "MED-3385": 1, "MED-3386": 1, "MED-3387": 1, "MED-3540": 1, "MED-3541": 1, "MED-3542": 1, "MED-3543": 1, "MED-3544": 1, "MED-3545": 1, "MED-3546": 1, "MED-3547": 1, "MED-3623": 1, "MED-3624": 1, "MED-3625": 1, "MED-3626": 1, "MED-3627": 1, "MED-3628": 1, "MED-3629": 1, "MED-3630": 1, "MED-3631": 1, "MED-3632": 1, "MED-3633": 1, "MED-3634": 1, "MED-3635": 1, "MED-3636": 1, "MED-3696": 1, "MED-3697": 1, "MED-3698": 1, "MED-3700": 1, "MED-3701": 1, "MED-3714": 1, "MED-3715": 1, "MED-3716": 1, "MED-3717": 1, "MED-3718": 1, "MED-3719": 1, "MED-3720": 1, "MED-3721": 1, "MED-3762": 1, "MED-3763": 1, "MED-3764": 1, "MED-3765": 1, "MED-3766": 1, "MED-3767": 1, "MED-3768": 1, "MED-3941": 1, "MED-3942": 1, "MED-3943": 1, "MED-3944": 1, "MED-3945": 1, "MED-3946": 1, "MED-3947": 1, "MED-4407": 1, "MED-3949": 1, "MED-3950": 1, "MED-3951": 1, "MED-3952": 1, "MED-3953": 1, "MED-4053": 1, "MED-4058": 1, "MED-4055": 1, "MED-4060": 1, "MED-4068": 1, "MED-4069": 1, "MED-4070": 1, "MED-4072": 1, "MED-5197": 1, "MED-4075": 1, "MED-4071": 1, "MED-4073": 1, "MED-4095": 1, "MED-4100": 1, "MED-4101": 1, "MED-4102": 1, "MED-4200": 1, "MED-4201": 1, "MED-4202": 1, "MED-4261": 1, "MED-4382": 1, "MED-4408": 1, "MED-4411": 1, "MED-4410": 1, "MED-4412": 1, "MED-4466": 1, "MED-4474": 1, "MED-4475": 1, "MED-4477": 1, "MED-4488": 1, "MED-4489": 1, "MED-4490": 1, "MED-4491": 1, "MED-4492": 1, "MED-4555": 1, "MED-4676": 1, "MED-4719": 1, "MED-4720": 1, "MED-4721": 1, "MED-4776": 1, "MED-4777": 1, "MED-4778": 1, "MED-4779": 1, "MED-4780": 1}, "PLAIN-2113": {"MED-4771": 1}, "PLAIN-2124": {"MED-1240": 1, "MED-1241": 1, "MED-1242": 1, "MED-1243": 1, "MED-1244": 1, "MED-1245": 1, "MED-1246": 1, "MED-1247": 1, "MED-1248": 1, "MED-1519": 1, "MED-1520": 1, "MED-1521": 1, "MED-1522": 1, "MED-1523": 1, "MED-1524": 1, "MED-1525": 1, "MED-1526": 1}, "PLAIN-2134": {"MED-1574": 1, "MED-1575": 1, "MED-1576": 1, "MED-1577": 1, "MED-1578": 1, "MED-1579": 1, "MED-1580": 1, "MED-1581": 1, "MED-1582": 1, "MED-5053": 1, "MED-5054": 1, "MED-5055": 1}, "PLAIN-2145": {"MED-3918": 1, "MED-3919": 1, "MED-3920": 1, "MED-3921": 1, "MED-3922": 1, "MED-3923": 1, "MED-3924": 1, "MED-4371": 1, "MED-4372": 1}, "PLAIN-2156": {"MED-4867": 1, "MED-4868": 1, "MED-4869": 1, "MED-5053": 1, "MED-5054": 1, "MED-5055": 1}, "PLAIN-2167": {"MED-2722": 1}, "PLAIN-2177": {"MED-717": 1, "MED-718": 1, "MED-719": 1, "MED-720": 1, "MED-721": 1, "MED-722": 1, "MED-723": 1, "MED-724": 1, "MED-5204": 1, "MED-1184": 1, "MED-1185": 1, "MED-1186": 1, "MED-1187": 1, "MED-1188": 1, "MED-1582": 1, "MED-1418": 1, "MED-1419": 1, "MED-1420": 1, "MED-1421": 1, "MED-3964": 1, "MED-4342": 1, "MED-1581": 1, "MED-1425": 1, "MED-1426": 1}, "PLAIN-2187": {"MED-1240": 1, "MED-1241": 1, "MED-1242": 1, "MED-1243": 1, "MED-1244": 1, "MED-1245": 1, "MED-1246": 1, "MED-1247": 1, "MED-1248": 1}, "PLAIN-2197": {"MED-1621": 1, "MED-1622": 1, "MED-1623": 1, "MED-1624": 1, "MED-1625": 1, "MED-1626": 1, "MED-1627": 1, "MED-1628": 1, "MED-5054": 1, "MED-1630": 1, "MED-1631": 1, "MED-1634": 1, "MED-1635": 1, "MED-1636": 1, "MED-1637": 1, "MED-1638": 1, "MED-1639": 1, "MED-1640": 1, "MED-1641": 1, "MED-1642": 1, "MED-1643": 1, "MED-5258": 1, "MED-1645": 1, "MED-1646": 1, "MED-1647": 1, "MED-1648": 1, "MED-1649": 1, "MED-3050": 1, "MED-3051": 1, "MED-3038": 1, "MED-3052": 1, "MED-3053": 1, "MED-3054": 1, "MED-3055": 1, "MED-3056": 1, "MED-3044": 1, "MED-3057": 1, "MED-3046": 1, "MED-3058": 1, "MED-3059": 1, "MED-3060": 1, "MED-4091": 1, "MED-4089": 1, "MED-5071": 1, "MED-4120": 1, "MED-4121": 1, "MED-4124": 1, "MED-4127": 1, "MED-4125": 1, "MED-4126": 1, "MED-4686": 1, "MED-4867": 1, "MED-4868": 1, "MED-4869": 1, "MED-5053": 1, "MED-5055": 1, "MED-5056": 1, "MED-5057": 1}, "PLAIN-2209": {"MED-4635": 1, "MED-4640": 1}, "PLAIN-2220": {"MED-4440": 1, "MED-4785": 1, "MED-4471": 1, "MED-4472": 1, "MED-4976": 1, "MED-4546": 1, "MED-4547": 1, "MED-5002": 1, "MED-5078": 1}, "PLAIN-2230": {"MED-3148": 1, "MED-3149": 1, "MED-4313": 1, "MED-4609": 1, "MED-4613": 1, "MED-4617": 1, "MED-4612": 1}, "PLAIN-2240": {"MED-3960": 1, "MED-3961": 1, "MED-3962": 1, "MED-3963": 1, "MED-3964": 1, "MED-3965": 1, "MED-3966": 1, "MED-3967": 1, "MED-3968": 1, "MED-3969": 1, "MED-3970": 1}, "PLAIN-2250": {"MED-4359": 1, "MED-4360": 1}, "PLAIN-2261": {"MED-2726": 1, "MED-2725": 1, "MED-3374": 1, "MED-3376": 1, "MED-1463": 1, "MED-1454": 1, "MED-1455": 1, "MED-1456": 1, "MED-1457": 1, "MED-1458": 1, "MED-1459": 1, "MED-1460": 1, "MED-1461": 1, "MED-1474": 1, "MED-1476": 1, "MED-2432": 1, "MED-1478": 1, "MED-1479": 1, "MED-1557": 1, "MED-1558": 1, "MED-1992": 1, "MED-1876": 1, "MED-1998": 1, "MED-1878": 1, "MED-1879": 1, "MED-2076": 1, "MED-2077": 1, "MED-2078": 1, "MED-2079": 1, "MED-2080": 1, "MED-5293": 1, "MED-2082": 1, "MED-2083": 1, "MED-5303": 1, "MED-2085": 1, "MED-2999": 1, "MED-3000": 1, "MED-3001": 1, "MED-3369": 1, "MED-3370": 1, "MED-3371": 1, "MED-3372": 1, "MED-3373": 1, "MED-3375": 1, "MED-3377": 1, "MED-3597": 1, "MED-3598": 1, "MED-3599": 1, "MED-4200": 1, "MED-4201": 1, "MED-4202": 1, "MED-4295": 1, "MED-4352": 1, "MED-4353": 1, "MED-4984": 1, "MED-4388": 1, "MED-4389": 1, "MED-4450": 1, "MED-4556": 1, "MED-4559": 1}, "PLAIN-2271": {"MED-762": 1, "MED-4247": 1, "MED-4642": 1, "MED-4643": 1, "MED-4644": 1, "MED-4776": 1, "MED-4777": 1, "MED-4778": 1, "MED-4779": 1, "MED-4780": 1}, "PLAIN-2281": {"MED-2098": 1}, "PLAIN-2291": {"MED-1650": 1, "MED-1651": 1, "MED-2726": 1, "MED-2221": 1, "MED-4602": 1}, "PLAIN-2301": {"MED-3503": 1, "MED-3504": 1, "MED-3794": 1, "MED-557": 1, "MED-4753": 1, "MED-4757": 1, "MED-4755": 1, "MED-4756": 1, "MED-4938": 1, "MED-4939": 1, "MED-4940": 1, "MED-4941": 1, "MED-4942": 1}, "PLAIN-2311": {"MED-4594": 1, "MED-4593": 1}, "PLAIN-2321": {"MED-4216": 1, "MED-4211": 1, "MED-4212": 1, "MED-4514": 1, "MED-4515": 1}, "PLAIN-2332": {"MED-2747": 1, "MED-2748": 1, "MED-2749": 1, "MED-1103": 1, "MED-1104": 1, "MED-1105": 1, "MED-1106": 1, "MED-1107": 1, "MED-1108": 1, "MED-1109": 1, "MED-1110": 1, "MED-1111": 1, "MED-1112": 1, "MED-1113": 1, "MED-1114": 1, "MED-1115": 1, "MED-1215": 1, "MED-1216": 1, "MED-1217": 1, "MED-1218": 1, "MED-1219": 1, "MED-1220": 1, "MED-1221": 1, "MED-4797": 1, "MED-1796": 1, "MED-1797": 1, "MED-1798": 1, "MED-1799": 1, "MED-1800": 1, "MED-1801": 1, "MED-1802": 1, "MED-1803": 1, "MED-1804": 1, "MED-4767": 1, "MED-1806": 1, "MED-1807": 1, "MED-1808": 1, "MED-4769": 1, "MED-1810": 1, "MED-2018": 1, "MED-2019": 1, "MED-2020": 1, "MED-2021": 1, "MED-2022": 1, "MED-2033": 1, "MED-2024": 1, "MED-2034": 1, "MED-2026": 1, "MED-2027": 1, "MED-2028": 1, "MED-2029": 1, "MED-2030": 1, "MED-3305": 1, "MED-3306": 1, "MED-3307": 1, "MED-3308": 1, "MED-3288": 1, "MED-3310": 1, "MED-3311": 1, "MED-3312": 1, "MED-3292": 1, "MED-3313": 1, "MED-3294": 1, "MED-3295": 1, "MED-3314": 1, "MED-3315": 1, "MED-3316": 1, "MED-4433": 1, "MED-3317": 1, "MED-3318": 1, "MED-3302": 1, "MED-3320": 1, "MED-3321": 1, "MED-3309": 1, "MED-3319": 1, "MED-3982": 1, "MED-3983": 1, "MED-3984": 1, "MED-4969": 1, "MED-4363": 1, "MED-4364": 1, "MED-4765": 1, "MED-4766": 1, "MED-4768": 1, "MED-4812": 1, "MED-4813": 1, "MED-4814": 1, "MED-4815": 1, "MED-4816": 1, "MED-4817": 1, "MED-4818": 1, "MED-4819": 1, "MED-4820": 1, "MED-4821": 1, "MED-5031": 1, "MED-5060": 1, "MED-5061": 1}, "PLAIN-2343": {"MED-5108": 1, "MED-5109": 1}, "PLAIN-2354": {"MED-2150": 1, "MED-4646": 1, "MED-2152": 1, "MED-2153": 1, "MED-4303": 1}, "PLAIN-2364": {"MED-3866": 1, "MED-3869": 1, "MED-1342": 1, "MED-1343": 1, "MED-1344": 1, "MED-1345": 1, "MED-1355": 1, "MED-1347": 1, "MED-1348": 1, "MED-1349": 1, "MED-1350": 1, "MED-1351": 1, "MED-1352": 1, "MED-1353": 1, "MED-1354": 1}, "PLAIN-2375": {"MED-5169": 1, "MED-5170": 1}, "PLAIN-2386": {"MED-3169": 1, "MED-3170": 1, "MED-3171": 1, "MED-3172": 1, "MED-3173": 1, "MED-3174": 1, "MED-3175": 1, "MED-3176": 1, "MED-3177": 1, "MED-3178": 1, "MED-3179": 1, "MED-3180": 1, "MED-3181": 1, "MED-3182": 1, "MED-4359": 1, "MED-4360": 1, "MED-4403": 1, "MED-4742": 1, "MED-4743": 1}, "PLAIN-2396": {"MED-2763": 1, "MED-4324": 1, "MED-3688": 1, "MED-3689": 1, "MED-3690": 1, "MED-3691": 1, "MED-3692": 1, "MED-3693": 1, "MED-3694": 1, "MED-3695": 1, "MED-3858": 1, "MED-4094": 1, "MED-3860": 1, "MED-4313": 1, "MED-3862": 1, "MED-5038": 1, "MED-5039": 1, "MED-5040": 1}, "PLAIN-2408": {"MED-1356": 1, "MED-1357": 1, "MED-1358": 1, "MED-1359": 1, "MED-1360": 1}, "PLAIN-2430": {"MED-980": 2, "MED-981": 2, "MED-982": 2, "MED-983": 2, "MED-984": 2, "MED-985": 2, "MED-986": 2, "MED-987": 2, "MED-988": 2, "MED-3137": 2, "MED-990": 2, "MED-991": 2, "MED-992": 2, "MED-993": 2, "MED-994": 2, "MED-2663": 1, "MED-2664": 1, "MED-2665": 1, "MED-4860": 1, "MED-2667": 1, "MED-2668": 1, "MED-2669": 1, "MED-2670": 1, "MED-2938": 1, "MED-2939": 1, "MED-2940": 1, "MED-2941": 1, "MED-2942": 1, "MED-2943": 1, "MED-2945": 1, "MED-2946": 1, "MED-2947": 1, "MED-3270": 1, "MED-3271": 1, "MED-3272": 1, "MED-3273": 1, "MED-3274": 1, "MED-3275": 1, "MED-3276": 1, "MED-3277": 1, "MED-3278": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3282": 1, "MED-3283": 1}, "PLAIN-2440": {"MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-2450": {"MED-1166": 2, "MED-1167": 2, "MED-1178": 2, "MED-1169": 2, "MED-1170": 2, "MED-1171": 2, "MED-1172": 2, "MED-1173": 2, "MED-1174": 2, "MED-1175": 2, "MED-1176": 2, "MED-1177": 2, "MED-2493": 1, "MED-2494": 1, "MED-2495": 1, "MED-2496": 1, "MED-2497": 1, "MED-2747": 1, "MED-2748": 1, "MED-2749": 1, "MED-3886": 1, "MED-3880": 1, "MED-3887": 1, "MED-4132": 1, "MED-3882": 1, "MED-3888": 1, "MED-3889": 1, "MED-4136": 1, "MED-3891": 1, "MED-3892": 1, "MED-3884": 1, "MED-4726": 1, "MED-3935": 1, "MED-3936": 1, "MED-3937": 1, "MED-3938": 1, "MED-3939": 1, "MED-3940": 1}, "PLAIN-2460": {"MED-1716": 2, "MED-1236": 2, "MED-1237": 2, "MED-1238": 2, "MED-1474": 2, "MED-1992": 1, "MED-1876": 1, "MED-1998": 1, "MED-1878": 1, "MED-1879": 1, "MED-2405": 1, "MED-2406": 1, "MED-2407": 1, "MED-3028": 1, "MED-2409": 1, "MED-2410": 1, "MED-2411": 1, "MED-2412": 1, "MED-2413": 1, "MED-2973": 1, "MED-5124": 1, "MED-2975": 1, "MED-2976": 1, "MED-2977": 1, "MED-2978": 1, "MED-3866": 1, "MED-3867": 1, "MED-3868": 1, "MED-3869": 1, "MED-4543": 1, "MED-4538": 1}, "PLAIN-2470": {"MED-1337": 2, "MED-1338": 2, "MED-1339": 2, "MED-1340": 2, "MED-1341": 2, "MED-1557": 1, "MED-1558": 1, "MED-1762": 1, "MED-1763": 1, "MED-1764": 1, "MED-1765": 1, "MED-1766": 1, "MED-1778": 1, "MED-1768": 1, "MED-4951": 1, "MED-1770": 1, "MED-1771": 1, "MED-1781": 1, "MED-1773": 1, "MED-1774": 1, "MED-2769": 1, "MED-2770": 1, "MED-2771": 1, "MED-2772": 1, "MED-2773": 1, "MED-2774": 1, "MED-2775": 1, "MED-2979": 1, "MED-2980": 1, "MED-4319": 1, "MED-2982": 1, "MED-2983": 1, "MED-2984": 1, "MED-2985": 1, "MED-2986": 1, "MED-2987": 1, "MED-2988": 1, "MED-2989": 1, "MED-2990": 1, "MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3089": 1, "MED-3090": 1, "MED-3091": 1, "MED-3092": 1, "MED-3093": 1, "MED-3094": 1, "MED-3095": 1, "MED-3096": 1, "MED-3226": 1, "MED-3227": 1, "MED-3228": 1, "MED-3229": 1, "MED-3230": 1, "MED-3231": 1, "MED-3232": 1, "MED-3233": 1, "MED-4722": 1, "MED-3235": 1, "MED-3236": 1, "MED-3237": 1, "MED-3597": 1, "MED-3598": 1, "MED-3599": 1, "MED-4726": 1, "MED-3935": 1, "MED-3936": 1, "MED-3937": 1, "MED-3938": 1, "MED-3939": 1, "MED-3940": 1}, "PLAIN-2480": {"MED-1418": 2, "MED-1419": 2, "MED-1420": 2, "MED-1421": 2, "MED-3964": 2, "MED-4342": 2, "MED-1581": 2, "MED-1425": 2, "MED-1426": 2, "MED-1574": 1, "MED-1575": 1, "MED-1576": 1, "MED-1577": 1, "MED-1578": 1, "MED-1579": 1, "MED-1580": 1, "MED-1582": 1, "MED-4349": 1}, "PLAIN-2490": {"MED-1486": 2, "MED-1487": 2, "MED-1488": 2, "MED-1489": 2, "MED-1490": 2, "MED-2109": 1, "MED-2110": 1, "MED-2111": 1, "MED-2112": 1, "MED-4255": 1, "MED-2524": 1, "MED-2525": 1, "MED-2526": 1, "MED-2527": 1, "MED-2528": 1, "MED-2529": 1, "MED-2530": 1, "MED-3617": 1, "MED-3618": 1, "MED-3619": 1, "MED-3620": 1, "MED-3621": 1, "MED-3622": 1}, "PLAIN-2500": {"MED-1557": 2, "MED-1558": 2, "MED-3597": 1, "MED-3598": 1, "MED-3599": 1, "MED-4556": 1, "MED-4559": 1}, "PLAIN-2510": {"MED-1634": 2, "MED-1635": 2, "MED-1636": 2, "MED-1637": 2, "MED-1638": 2, "MED-1639": 2, "MED-1640": 2, "MED-1641": 2, "MED-1642": 2, "MED-1643": 2, "MED-5258": 2, "MED-1645": 2, "MED-1646": 2, "MED-1647": 2, "MED-1648": 2, "MED-1649": 2, "MED-2154": 1, "MED-2155": 1, "MED-2156": 1, "MED-2157": 1, "MED-2158": 1, "MED-2159": 1, "MED-2160": 1, "MED-2161": 1, "MED-2162": 1, "MED-2163": 1, "MED-2222": 1, "MED-2223": 1, "MED-2224": 1, "MED-2225": 1, "MED-2226": 1, "MED-2227": 1, "MED-2228": 1, "MED-2376": 1, "MED-2593": 1, "MED-2378": 1, "MED-2379": 1, "MED-2380": 1, "MED-2381": 1, "MED-2382": 1, "MED-2383": 1, "MED-2384": 1, "MED-4438": 1, "MED-4439": 1, "MED-4520": 1, "MED-4522": 1}, "PLAIN-2520": {"MED-1711": 2, "MED-1712": 2, "MED-2763": 2, "MED-1714": 2, "MED-1715": 2, "MED-1716": 2, "MED-1717": 2, "MED-1718": 2, "MED-1719": 2, "MED-1720": 2, "MED-1721": 2, "MED-1722": 2, "MED-1723": 2, "MED-1724": 2, "MED-2114": 1, "MED-2115": 1, "MED-2116": 1, "MED-2117": 1, "MED-4896": 1, "MED-3436": 1, "MED-2120": 1, "MED-2121": 1, "MED-2122": 1, "MED-2123": 1, "MED-2124": 1, "MED-2136": 1, "MED-2126": 1, "MED-2127": 1, "MED-2128": 1, "MED-2129": 1, "MED-2130": 1, "MED-2507": 1, "MED-2132": 1, "MED-2517": 1, "MED-2134": 1, "MED-2135": 1, "MED-2137": 1, "MED-2138": 1, "MED-2139": 1, "MED-2498": 1, "MED-2519": 1, "MED-2501": 1, "MED-2502": 1, "MED-2513": 1, "MED-2504": 1, "MED-2505": 1, "MED-2506": 1, "MED-5239": 1, "MED-2509": 1, "MED-2510": 1, "MED-2511": 1, "MED-2512": 1, "MED-2514": 1, "MED-5237": 1, "MED-2943": 1, "MED-2518": 1, "MED-2520": 1, "MED-2521": 1, "MED-4224": 1, "MED-4226": 1, "MED-4228": 1, "MED-4216": 1, "MED-4211": 1, "MED-4212": 1, "MED-4886": 1, "MED-5337": 1, "MED-4220": 1, "MED-4612": 1, "MED-5341": 1, "MED-4222": 1, "MED-4223": 1, "MED-4225": 1, "MED-4227": 1}, "PLAIN-2530": {"MED-1796": 2, "MED-1797": 2, "MED-1798": 2, "MED-1799": 2, "MED-1800": 2, "MED-1801": 2, "MED-1802": 2, "MED-1803": 2, "MED-1804": 2, "MED-4767": 2, "MED-1806": 2, "MED-1807": 2, "MED-1808": 2, "MED-4769": 2, "MED-1810": 2, "MED-1998": 1, "MED-1985": 1, "MED-1986": 1, "MED-1987": 1, "MED-1988": 1, "MED-1999": 1, "MED-1990": 1, "MED-1991": 1, "MED-1992": 1, "MED-1993": 1, "MED-1994": 1, "MED-2221": 1, "MED-1996": 1, "MED-1997": 1, "MED-2053": 1, "MED-2054": 1, "MED-2055": 1, "MED-2056": 1, "MED-2057": 1, "MED-2058": 1, "MED-2059": 1, "MED-2060": 1, "MED-2061": 1, "MED-2062": 1, "MED-2063": 1, "MED-3253": 1, "MED-3254": 1, "MED-3255": 1, "MED-3305": 1, "MED-3306": 1, "MED-3307": 1, "MED-3308": 1, "MED-3309": 1, "MED-3310": 1, "MED-3311": 1, "MED-3312": 1, "MED-3313": 1, "MED-3314": 1, "MED-3315": 1, "MED-3316": 1, "MED-3317": 1, "MED-3318": 1, "MED-3319": 1, "MED-3320": 1, "MED-3321": 1, "MED-4261": 1, "MED-4431": 1, "MED-4433": 1, "MED-4819": 1, "MED-4436": 1, "MED-4760": 1, "MED-4762": 1, "MED-4763": 1, "MED-4764": 1, "MED-4765": 1, "MED-4766": 1, "MED-4768": 1}, "PLAIN-2540": {"MED-1882": 2, "MED-2884": 2, "MED-1884": 2, "MED-1885": 2, "MED-1886": 2, "MED-1887": 2, "MED-1888": 2, "MED-1889": 2, "MED-1890": 2, "MED-2370": 1, "MED-2371": 1, "MED-2372": 1, "MED-4324": 1, "MED-2374": 1, "MED-3197": 1, "MED-3198": 1, "MED-3253": 1, "MED-3254": 1, "MED-3255": 1, "MED-4556": 1, "MED-4559": 1}, "PLAIN-2550": {"MED-2529": 2, "MED-2524": 2, "MED-2525": 2, "MED-2527": 2, "MED-2530": 2, "MED-2528": 2, "MED-2427": 1, "MED-2428": 1, "MED-2429": 1, "MED-2430": 1, "MED-2431": 1, "MED-2432": 1, "MED-2440": 1, "MED-2526": 1, "MED-2763": 1, "MED-3113": 1, "MED-4247": 1, "MED-4560": 1, "MED-4559": 1, "MED-4598": 1, "MED-4599": 1, "MED-4696": 1, "MED-4828": 1, "MED-4829": 1, "MED-4830": 1}, "PLAIN-2560": {"MED-2053": 2, "MED-2054": 2, "MED-2055": 2, "MED-2056": 2, "MED-2057": 2, "MED-2058": 2, "MED-2059": 2, "MED-2060": 2, "MED-2061": 2, "MED-2062": 2, "MED-2063": 2, "MED-3954": 1, "MED-3955": 1, "MED-3956": 1, "MED-4680": 1, "MED-3958": 1, "MED-3959": 1, "MED-4394": 1, "MED-4399": 1, "MED-4400": 1, "MED-4402": 1}, "PLAIN-2570": {"MED-2140": 2, "MED-2141": 2, "MED-4319": 2, "MED-2143": 2, "MED-2144": 2, "MED-2145": 2, "MED-2146": 2, "MED-2405": 1, "MED-2394": 1, "MED-2395": 1, "MED-2396": 1, "MED-2397": 1, "MED-2398": 1, "MED-2407": 1, "MED-2400": 1, "MED-3028": 1, "MED-2402": 1, "MED-2411": 1, "MED-2404": 1, "MED-2406": 1, "MED-2409": 1, "MED-2410": 1, "MED-2412": 1, "MED-2413": 1, "MED-2973": 1, "MED-5124": 1, "MED-2975": 1, "MED-2976": 1, "MED-2977": 1, "MED-2978": 1, "MED-3811": 1, "MED-3812": 1, "MED-3813": 1, "MED-4389": 1, "MED-3866": 1, "MED-3867": 1, "MED-3868": 1, "MED-3869": 1, "MED-4543": 1, "MED-4538": 1}, "PLAIN-2580": {"MED-2219": 2, "MED-2220": 2, "MED-2221": 2, "MED-2488": 1, "MED-2489": 1, "MED-2616": 1, "MED-2617": 1, "MED-2618": 1, "MED-2619": 1, "MED-2725": 1, "MED-2726": 1, "MED-2736": 1, "MED-4131": 1, "MED-2738": 1, "MED-5169": 1, "MED-2740": 1, "MED-2741": 1, "MED-2742": 1, "MED-2744": 1, "MED-2745": 1, "MED-3197": 1, "MED-3198": 1, "MED-4602": 1}, "PLAIN-2590": {"MED-2288": 2, "MED-3137": 2, "MED-2290": 2, "MED-2291": 2, "MED-2292": 2, "MED-2293": 2, "MED-2294": 2, "MED-2295": 2, "MED-2296": 2, "MED-2498": 1, "MED-2517": 1, "MED-2519": 1, "MED-2501": 1, "MED-2502": 1, "MED-2513": 1, "MED-2504": 1, "MED-2505": 1, "MED-2506": 1, "MED-2507": 1, "MED-5239": 1, "MED-2509": 1, "MED-2510": 1, "MED-2511": 1, "MED-2512": 1, "MED-3000": 1, "MED-2765": 1, "MED-2997": 1, "MED-3001": 1, "MED-2999": 1, "MED-4313": 1, "MED-3148": 1, "MED-3149": 1, "MED-3242": 1, "MED-3243": 1, "MED-3244": 1, "MED-3245": 1, "MED-3270": 1, "MED-3271": 1, "MED-3272": 1, "MED-3273": 1, "MED-3274": 1, "MED-3275": 1, "MED-3276": 1, "MED-3277": 1, "MED-3278": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3282": 1, "MED-3283": 1, "MED-3580": 1, "MED-3581": 1, "MED-3582": 1, "MED-3583": 1, "MED-3584": 1, "MED-3858": 1, "MED-4094": 1, "MED-3860": 1, "MED-3862": 1, "MED-4107": 1, "MED-4299": 1, "MED-4298": 1, "MED-4600": 1}, "PLAIN-2600": {"MED-2370": 2, "MED-2371": 2, "MED-2372": 2, "MED-4324": 2, "MED-3197": 1, "MED-3198": 1, "MED-4300": 1, "MED-4301": 1, "MED-4602": 1, "MED-4676": 1}, "PLAIN-2610": {"MED-2445": 2, "MED-2446": 2, "MED-2458": 2, "MED-2448": 2, "MED-2449": 2, "MED-2450": 2, "MED-2451": 2, "MED-2452": 2, "MED-2453": 2, "MED-2472": 2, "MED-4551": 1, "MED-2643": 1, "MED-2644": 1, "MED-2645": 1, "MED-2646": 1, "MED-2647": 1, "MED-2648": 1, "MED-2649": 1, "MED-2650": 1, "MED-2651": 1, "MED-2652": 1, "MED-2653": 1, "MED-2654": 1, "MED-2655": 1, "MED-2656": 1, "MED-2657": 1, "MED-2658": 1, "MED-2659": 1, "MED-2660": 1, "MED-2661": 1, "MED-2662": 1, "MED-2663": 1, "MED-2664": 1, "MED-2665": 1, "MED-4860": 1, "MED-2667": 1, "MED-2668": 1, "MED-2669": 1, "MED-2670": 1, "MED-2882": 1, "MED-2890": 1, "MED-2891": 1, "MED-2892": 1, "MED-2898": 1, "MED-2893": 1, "MED-2884": 1, "MED-2885": 1, "MED-2886": 1, "MED-2899": 1, "MED-2888": 1, "MED-2889": 1, "MED-2894": 1, "MED-2895": 1, "MED-3904": 1, "MED-5175": 1, "MED-4295": 1, "MED-4349": 1, "MED-4345": 1, "MED-4346": 1, "MED-4365": 1, "MED-4658": 1}, "PLAIN-2620": {"MED-2568": 2, "MED-4319": 2, "MED-2570": 2, "MED-2571": 2, "MED-2572": 2, "MED-2573": 2, "MED-2574": 2, "MED-2575": 2, "MED-2988": 2, "MED-2577": 2, "MED-2578": 2, "MED-2579": 2, "MED-2580": 2, "MED-2544": 2, "MED-2581": 2, "MED-2546": 2, "MED-2583": 2, "MED-2584": 2, "MED-2585": 2, "MED-2979": 1, "MED-2980": 1, "MED-2982": 1, "MED-2983": 1, "MED-2984": 1, "MED-2985": 1, "MED-2986": 1, "MED-2987": 1, "MED-2989": 1, "MED-2990": 1, "MED-3242": 1, "MED-3243": 1, "MED-3244": 1, "MED-3245": 1, "MED-5337": 1, "MED-3714": 1, "MED-3715": 1, "MED-3716": 1, "MED-3717": 1, "MED-3718": 1, "MED-3719": 1, "MED-3720": 1, "MED-3721": 1, "MED-3722": 1, "MED-3723": 1, "MED-3724": 1, "MED-3744": 1, "MED-3726": 1, "MED-4481": 1, "MED-3728": 1, "MED-3729": 1, "MED-3730": 1, "MED-3731": 1, "MED-3732": 1, "MED-4165": 1, "MED-4543": 1}, "PLAIN-2630": {"MED-4551": 2, "MED-2643": 2, "MED-2644": 2, "MED-2645": 2, "MED-2646": 2, "MED-2647": 2, "MED-2648": 2, "MED-2649": 2, "MED-2650": 2, "MED-2651": 2, "MED-2652": 2, "MED-2653": 2, "MED-2654": 2, "MED-2655": 2, "MED-2656": 2, "MED-2657": 2, "MED-2658": 2, "MED-2659": 2, "MED-2660": 2, "MED-2661": 2, "MED-2662": 2, "MED-3585": 1, "MED-3586": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3590": 1, "MED-3591": 1, "MED-3592": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-3954": 1, "MED-3955": 1, "MED-3956": 1, "MED-4680": 1, "MED-3958": 1, "MED-3959": 1, "MED-4550": 1, "MED-4653": 1, "MED-4654": 1, "MED-4655": 1, "MED-4656": 1, "MED-4727": 1, "MED-4728": 1, "MED-4950": 1, "MED-4951": 1}, "PLAIN-2640": {"MED-2736": 2, "MED-4131": 2, "MED-2738": 2, "MED-5169": 2, "MED-2740": 2, "MED-2741": 2, "MED-2742": 2, "MED-2744": 2, "MED-2745": 2, "MED-2725": 1, "MED-2726": 1, "MED-2743": 1, "MED-2746": 1, "MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3091": 1, "MED-3092": 1, "MED-3095": 1, "MED-3089": 1, "MED-3090": 1, "MED-3093": 1, "MED-3094": 1, "MED-3096": 1, "MED-3880": 1, "MED-4132": 1, "MED-3882": 1, "MED-4136": 1, "MED-3884": 1, "MED-3885": 1, "MED-3886": 1, "MED-3887": 1, "MED-3888": 1, "MED-3889": 1, "MED-3890": 1, "MED-3891": 1, "MED-3892": 1, "MED-4354": 1, "MED-4355": 1, "MED-4357": 1, "MED-4358": 1, "MED-4361": 1, "MED-4426": 1, "MED-4427": 1, "MED-4428": 1, "MED-4429": 1, "MED-4596": 1}, "PLAIN-2650": {"MED-2797": 2, "MED-2809": 2, "MED-2799": 2, "MED-2800": 2, "MED-2801": 2, "MED-2802": 2, "MED-2803": 2, "MED-2804": 2, "MED-2805": 2, "MED-2819": 2, "MED-2807": 2, "MED-3702": 1, "MED-3703": 1, "MED-3704": 1, "MED-3705": 1, "MED-3706": 1, "MED-3707": 1, "MED-3708": 1, "MED-3709": 1, "MED-3710": 1, "MED-3711": 1, "MED-3712": 1, "MED-3713": 1, "MED-4349": 1, "MED-4345": 1, "MED-4346": 1, "MED-4613": 1, "MED-4617": 1, "MED-4615": 1}, "PLAIN-2660": {"MED-3021": 2, "MED-3023": 2, "MED-2904": 2, "MED-2905": 2, "MED-2906": 2, "MED-2907": 2, "MED-3024": 2, "MED-3025": 2, "MED-2910": 2, "MED-3027": 2, "MED-3028": 2, "MED-2913": 2, "MED-3012": 2, "MED-3013": 2, "MED-3030": 2, "MED-2917": 2, "MED-3033": 2, "MED-3034": 2, "MED-3035": 2, "MED-2921": 2, "MED-4169": 1, "MED-4170": 1, "MED-4171": 1, "MED-4180": 1, "MED-4181": 1, "MED-4622": 1, "MED-4726": 1}, "PLAIN-2670": {"MED-2992": 2, "MED-2993": 2, "MED-2994": 2, "MED-3085": 1, "MED-3086": 1, "MED-3087": 1, "MED-3088": 1, "MED-3091": 1, "MED-3092": 1, "MED-3095": 1, "MED-3089": 1, "MED-3090": 1, "MED-3093": 1, "MED-3094": 1, "MED-3096": 1, "MED-3378": 1, "MED-3379": 1, "MED-3380": 1, "MED-3381": 1, "MED-3382": 1, "MED-3960": 1, "MED-3961": 1, "MED-3962": 1, "MED-3963": 1, "MED-3964": 1, "MED-3965": 1, "MED-3966": 1, "MED-3967": 1, "MED-3968": 1, "MED-3969": 1, "MED-3970": 1, "MED-4912": 1, "MED-4911": 1, "MED-4936": 1, "MED-4932": 1}, "PLAIN-2680": {"MED-3109": 2, "MED-3098": 2, "MED-3099": 2, "MED-3100": 2, "MED-3112": 2, "MED-3102": 2, "MED-4194": 1, "MED-4195": 1, "MED-4196": 1, "MED-4197": 1, "MED-4726": 1, "MED-4739": 1, "MED-4740": 1, "MED-4741": 1}, "PLAIN-2690": {"MED-3169": 2, "MED-3170": 2, "MED-3171": 2, "MED-3172": 2, "MED-3173": 2, "MED-3174": 2, "MED-3175": 2, "MED-3176": 2, "MED-3177": 2, "MED-3178": 2, "MED-3179": 2, "MED-3180": 2, "MED-3181": 2, "MED-3182": 2, "MED-3305": 1, "MED-3306": 1, "MED-3307": 1, "MED-3308": 1, "MED-3288": 1, "MED-3310": 1, "MED-3311": 1, "MED-3312": 1, "MED-3292": 1, "MED-3313": 1, "MED-3294": 1, "MED-3295": 1, "MED-3314": 1, "MED-3315": 1, "MED-3316": 1, "MED-4433": 1, "MED-3317": 1, "MED-3318": 1, "MED-3302": 1, "MED-3320": 1, "MED-3321": 1, "MED-4356": 1, "MED-4403": 1, "MED-4672": 1, "MED-4742": 1, "MED-4743": 1, "MED-4955": 1, "MED-4956": 1, "MED-4957": 1}, "PLAIN-2700": {"MED-3253": 2, "MED-3254": 2, "MED-3255": 2, "MED-4556": 1, "MED-4559": 1, "MED-4560": 1, "MED-4613": 1, "MED-4617": 1, "MED-4615": 1, "MED-4616": 1}, "PLAIN-2710": {"MED-3378": 2, "MED-3379": 2, "MED-3380": 2, "MED-3381": 2, "MED-3382": 2, "MED-5062": 1, "MED-3369": 1, "MED-3370": 1, "MED-3371": 1, "MED-3372": 1, "MED-3373": 1, "MED-3374": 1, "MED-3375": 1, "MED-3376": 1, "MED-3377": 1, "MED-3385": 1, "MED-3386": 1, "MED-3387": 1, "MED-4936": 1, "MED-4932": 1, "MED-5063": 1}, "PLAIN-2720": {"MED-3473": 2, "MED-3474": 2, "MED-3475": 2, "MED-3476": 2, "MED-3477": 2, "MED-3478": 2, "MED-3479": 2, "MED-3480": 2, "MED-3481": 2, "MED-3482": 2, "MED-3483": 2, "MED-3484": 2, "MED-3485": 2, "MED-3469": 1, "MED-3471": 1, "MED-4390": 1, "MED-4022": 1, "MED-4023": 1, "MED-4024": 1, "MED-4025": 1, "MED-4029": 1, "MED-4013": 1, "MED-4030": 1, "MED-4031": 1, "MED-4032": 1, "MED-4033": 1, "MED-4034": 1, "MED-4019": 1, "MED-4035": 1, "MED-4036": 1, "MED-4391": 1, "MED-4392": 1, "MED-4393": 1, "MED-4581": 1, "MED-4582": 1, "MED-4583": 1}, "PLAIN-2730": {"MED-3548": 2, "MED-3549": 2, "MED-3550": 2, "MED-3551": 2, "MED-3552": 2, "MED-3553": 2, "MED-3554": 2, "MED-3555": 2, "MED-3834": 1, "MED-3853": 1, "MED-3845": 1, "MED-4089": 1, "MED-4090": 1, "MED-4220": 1, "MED-5337": 1, "MED-4612": 1, "MED-5341": 1, "MED-4222": 1, "MED-4223": 1, "MED-4224": 1, "MED-4225": 1, "MED-4226": 1, "MED-4227": 1, "MED-4228": 1, "MED-4652": 1, "MED-5072": 1, "MED-5069": 1, "MED-5070": 1, "MED-5071": 1}, "PLAIN-2740": {"MED-3623": 2, "MED-3624": 2, "MED-3625": 2, "MED-3626": 2, "MED-3627": 2, "MED-3609": 1, "MED-3610": 1, "MED-3617": 1, "MED-3620": 1, "MED-3622": 1, "MED-3618": 1, "MED-3621": 1, "MED-3628": 1, "MED-3629": 1, "MED-3630": 1, "MED-3631": 1, "MED-3632": 1, "MED-3633": 1, "MED-3634": 1, "MED-3635": 1, "MED-3636": 1, "MED-4560": 1, "MED-4609": 1, "MED-4613": 1, "MED-4617": 1, "MED-4612": 1, "MED-4615": 1, "MED-4616": 1, "MED-5036": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-2750": {"MED-3680": 2, "MED-3681": 2, "MED-3682": 2, "MED-3683": 2, "MED-3693": 2, "MED-3685": 2, "MED-3686": 2, "MED-3687": 2, "MED-3671": 1, "MED-3676": 1, "MED-3679": 1, "MED-3690": 1, "MED-3694": 1, "MED-3688": 1, "MED-3689": 1, "MED-3691": 1, "MED-3692": 1, "MED-3695": 1, "MED-3702": 1, "MED-3703": 1, "MED-3704": 1, "MED-3705": 1, "MED-3706": 1, "MED-3707": 1, "MED-3708": 1, "MED-3709": 1, "MED-3710": 1, "MED-3711": 1, "MED-3712": 1, "MED-3713": 1, "MED-3941": 1, "MED-3942": 1, "MED-3943": 1, "MED-3944": 1, "MED-3945": 1, "MED-3946": 1, "MED-3947": 1, "MED-4407": 1, "MED-3949": 1, "MED-3950": 1, "MED-3951": 1, "MED-3952": 1, "MED-3953": 1, "MED-3971": 1, "MED-3972": 1, "MED-3973": 1, "MED-3974": 1, "MED-3975": 1, "MED-3976": 1, "MED-3977": 1, "MED-3978": 1, "MED-3979": 1, "MED-3980": 1, "MED-3981": 1, "MED-4351": 1, "MED-4563": 1, "MED-4564": 1, "MED-4565": 1, "MED-5031": 1}, "PLAIN-2760": {"MED-3769": 2, "MED-3770": 2, "MED-4406": 1, "MED-4454": 1, "MED-4622": 1, "MED-4686": 1, "MED-4712": 1, "MED-4771": 1}, "PLAIN-2770": {"MED-3841": 2, "MED-3832": 2, "MED-3843": 2, "MED-3836": 2, "MED-3853": 2, "MED-3845": 2, "MED-3848": 2, "MED-3830": 2, "MED-3840": 2, "MED-3864": 1, "MED-3833": 1, "MED-3834": 1, "MED-3842": 1, "MED-3844": 1, "MED-3846": 1, "MED-3847": 1, "MED-3850": 1, "MED-4094": 1, "MED-3852": 1, "MED-3854": 1, "MED-3855": 1, "MED-3856": 1, "MED-3857": 1, "MED-3863": 1, "MED-3865": 1, "MED-3866": 1, "MED-3867": 1, "MED-3868": 1, "MED-3869": 1, "MED-3874": 1, "MED-3875": 1, "MED-3876": 1, "MED-3877": 1, "MED-3878": 1, "MED-4233": 1, "MED-4116": 1, "MED-4117": 1, "MED-4118": 1, "MED-4119": 1, "MED-4440": 1, "MED-4785": 1}, "PLAIN-2780": {"MED-3896": 2, "MED-3897": 2, "MED-3906": 2, "MED-3908": 2, "MED-3900": 2, "MED-4289": 2, "MED-3910": 2, "MED-4715": 2, "MED-3893": 1, "MED-3894": 1, "MED-3895": 1, "MED-3904": 1, "MED-5175": 1, "MED-3907": 1, "MED-4281": 1, "MED-4286": 1, "MED-4284": 1, "MED-4292": 1, "MED-4287": 1, "MED-4288": 1, "MED-4290": 1, "MED-4291": 1}, "PLAIN-2790": {"MED-3960": 2, "MED-3961": 2, "MED-3962": 2, "MED-3963": 2, "MED-3964": 2, "MED-3965": 2, "MED-3966": 2, "MED-3967": 2, "MED-3968": 2, "MED-3969": 2, "MED-3970": 2, "MED-4068": 1, "MED-4069": 1, "MED-4070": 1, "MED-4071": 1, "MED-4072": 1, "MED-4073": 1, "MED-5197": 1, "MED-4075": 1, "MED-4853": 1, "MED-4128": 1, "MED-4129": 1, "MED-4130": 1, "MED-4131": 1, "MED-4132": 1, "MED-4133": 1, "MED-4134": 1, "MED-4135": 1, "MED-4136": 1, "MED-4137": 1, "MED-4138": 1, "MED-4139": 1, "MED-4140": 1, "MED-4141": 1, "MED-4142": 1, "MED-4143": 1, "MED-4167": 1, "MED-4168": 1, "MED-4333": 1, "MED-4337": 1, "MED-4342": 1, "MED-4349": 1, "MED-5326": 1, "MED-4488": 1, "MED-4489": 1, "MED-4490": 1, "MED-4491": 1, "MED-4492": 1, "MED-4604": 1, "MED-4605": 1, "MED-4606": 1, "MED-4607": 1, "MED-4627": 1, "MED-4908": 1, "MED-4909": 1, "MED-4936": 1, "MED-4932": 1, "MED-5062": 1, "MED-5063": 1}, "PLAIN-2800": {"MED-4040": 2, "MED-4976": 2, "MED-5197": 2, "MED-4049": 1, "MED-4053": 1, "MED-4051": 1, "MED-5116": 1, "MED-4047": 1, "MED-4055": 1, "MED-4050": 1, "MED-4068": 1, "MED-4069": 1, "MED-4070": 1, "MED-4071": 1, "MED-4072": 1, "MED-4073": 1, "MED-4075": 1, "MED-4438": 1, "MED-4439": 1, "MED-4454": 1, "MED-4455": 1, "MED-4471": 1, "MED-4472": 1, "MED-4861": 1}, "PLAIN-2810": {"MED-4091": 2, "MED-4089": 2, "MED-5071": 2, "MED-4090": 1, "MED-4413": 1, "MED-4585": 1, "MED-4687": 1, "MED-4689": 1}, "PLAIN-2820": {"MED-4106": 2, "MED-4103": 1, "MED-4104": 1, "MED-4853": 1, "MED-4107": 1, "MED-4299": 1, "MED-4522": 1, "MED-4545": 1}, "PLAIN-2830": {"MED-4120": 2, "MED-4121": 1, "MED-4124": 1}, "PLAIN-2840": {"MED-3811": 1, "MED-5341": 1, "MED-5337": 1, "MED-4464": 1, "MED-4465": 1, "MED-4468": 1, "MED-4469": 1, "MED-4543": 1, "MED-4538": 1, "MED-4539": 1, "MED-4540": 1, "MED-4541": 1, "MED-4671": 1, "MED-5047": 1, "MED-5048": 1, "MED-5049": 1, "MED-5050": 1}, "PLAIN-2850": {"MED-4166": 2, "MED-3693": 1, "MED-3690": 1, "MED-3694": 1, "MED-4167": 1, "MED-4168": 1, "MED-4433": 1, "MED-4436": 1, "MED-4437": 1}, "PLAIN-2860": {"MED-4185": 2, "MED-4186": 2}, "PLAIN-2870": {"MED-4200": 2, "MED-4201": 1, "MED-4202": 1, "MED-4531": 1, "MED-4944": 1}, "PLAIN-2880": {"MED-4612": 2, "MED-4220": 2, "MED-5341": 2, "MED-3875": 1, "MED-3876": 1, "MED-3877": 1, "MED-4233": 1, "MED-4222": 1, "MED-4223": 1, "MED-4224": 1, "MED-4225": 1, "MED-4226": 1, "MED-4227": 1, "MED-4228": 1, "MED-4239": 1, "MED-4243": 1, "MED-4255": 1}, "PLAIN-2890": {"MED-4250": 2, "MED-4300": 1, "MED-4301": 1, "MED-4324": 1, "MED-4349": 1, "MED-4345": 1, "MED-4346": 1, "MED-4603": 1, "MED-5322": 1, "MED-5323": 1, "MED-5324": 1, "MED-5325": 1, "MED-5326": 1, "MED-5327": 1, "MED-5328": 1, "MED-5329": 1, "MED-5330": 1, "MED-5331": 1, "MED-5332": 1, "MED-5333": 1, "MED-5334": 1, "MED-5335": 1, "MED-5363": 1, "MED-5337": 1, "MED-5338": 1, "MED-5339": 1, "MED-5340": 1, "MED-5341": 1, "MED-5342": 1}, "PLAIN-2900": {"MED-4276": 2, "MED-4269": 2, "MED-5332": 2, "MED-4271": 2, "MED-3788": 1, "MED-4768": 1, "MED-4313": 1, "MED-4278": 1, "MED-4325": 1, "MED-4442": 1, "MED-4443": 1, "MED-4810": 1}, "PLAIN-2910": {"MED-4846": 2, "MED-4295": 2, "MED-4300": 1, "MED-4301": 1, "MED-4352": 1, "MED-4353": 1, "MED-4423": 1, "MED-4424": 1, "MED-4425": 1, "MED-4560": 1, "MED-4832": 1, "MED-4833": 1, "MED-4834": 1, "MED-4835": 1, "MED-4906": 1}, "PLAIN-2920": {"MED-4539": 1, "MED-4540": 1, "MED-4541": 1, "MED-4632": 1, "MED-4633": 1, "MED-4634": 1, "MED-4995": 1}, "PLAIN-2930": {"MED-4325": 2, "MED-4516": 1, "MED-4517": 1, "MED-4518": 1, "MED-4522": 1, "MED-4635": 1, "MED-4640": 1, "MED-4637": 1, "MED-4689": 1, "MED-4810": 1, "MED-4832": 1, "MED-4833": 1, "MED-4834": 1, "MED-4835": 1}, "PLAIN-2940": {"MED-4342": 2, "MED-4349": 2, "MED-4347": 1, "MED-4627": 1, "MED-4350": 1, "MED-4628": 1, "MED-4629": 1, "MED-4630": 1, "MED-4631": 1, "MED-4877": 1, "MED-4878": 1, "MED-4879": 1, "MED-4886": 1, "MED-4887": 1, "MED-4888": 1, "MED-5337": 1, "MED-4890": 1, "MED-4891": 1, "MED-4984": 1, "MED-4985": 1, "MED-4988": 1, "MED-4987": 1}, "PLAIN-2950": {"MED-4355": 2, "MED-4132": 1}, "PLAIN-2960": {"MED-4370": 2, "MED-4575": 1}, "PLAIN-2970": {"MED-4382": 2, "MED-4385": 1, "MED-4383": 1, "MED-4384": 1, "MED-4984": 1, "MED-4388": 1, "MED-4389": 1, "MED-4553": 1, "MED-4554": 1}, "PLAIN-2981": {"MED-4403": 2, "MED-4908": 1, "MED-4909": 1, "MED-4912": 1, "MED-4911": 1, "MED-4936": 1, "MED-4932": 1}, "PLAIN-2991": {"MED-4423": 2, "MED-4424": 2}, "PLAIN-3001": {"MED-4436": 2, "MED-4437": 2}, "PLAIN-3014": {"MED-4454": 2}, "PLAIN-3026": {"MED-4470": 2}, "PLAIN-3037": {"MED-4508": 2, "MED-4504": 2, "MED-4505": 2, "MED-4506": 2}, "PLAIN-3053": {"MED-4525": 2, "MED-4526": 2, "MED-4527": 2}, "PLAIN-3063": {"MED-3742": 1, "MED-3893": 1, "MED-3894": 1, "MED-4544": 1}, "PLAIN-3074": {"MED-4555": 2}, "PLAIN-3085": {"MED-4573": 2, "MED-4570": 2}, "PLAIN-3097": {"MED-4590": 2, "MED-4591": 2, "MED-2215": 1, "MED-726": 1}, "PLAIN-3116": {"MED-5331": 2}, "PLAIN-3131": {"MED-4619": 2, "MED-4620": 2, "MED-4621": 2, "MED-4514": 1, "MED-4515": 1, "MED-4618": 1}, "PLAIN-3141": {"MED-4642": 2, "MED-4643": 2, "MED-4644": 2, "MED-3762": 1, "MED-3763": 1, "MED-3764": 1, "MED-1375": 1, "MED-1362": 1, "MED-1363": 1, "MED-1383": 1, "MED-1365": 1, "MED-1366": 1, "MED-1643": 1, "MED-1374": 1, "MED-1380": 1, "MED-5301": 1, "MED-1371": 1, "MED-5271": 1, "MED-1373": 1, "MED-3841": 1, "MED-3843": 1, "MED-3849": 1, "MED-3830": 1, "MED-1825": 1, "MED-1826": 1, "MED-1827": 1, "MED-1828": 1, "MED-1829": 1, "MED-1830": 1, "MED-3858": 1, "MED-4094": 1, "MED-3860": 1, "MED-4313": 1, "MED-3862": 1}, "PLAIN-3151": {"MED-4660": 2, "MED-4666": 2, "MED-4662": 2, "MED-4663": 2, "MED-4664": 2, "MED-3441": 1, "MED-3442": 1, "MED-3443": 1, "MED-3444": 1, "MED-3445": 1, "MED-3446": 1, "MED-3447": 1, "MED-3448": 1, "MED-3971": 1, "MED-3972": 1, "MED-3973": 1, "MED-3974": 1, "MED-3975": 1}, "PLAIN-3161": {"MED-4677": 2, "MED-1223": 1, "MED-1224": 1, "MED-5239": 1, "MED-1226": 1, "MED-1227": 1, "MED-2126": 1, "MED-1229": 1, "MED-1230": 1, "MED-1337": 1, "MED-1338": 1, "MED-1339": 1, "MED-1340": 1, "MED-1341": 1, "MED-1763": 1, "MED-1584": 1, "MED-1585": 1, "MED-1586": 1, "MED-1587": 1, "MED-1588": 1, "MED-1595": 1, "MED-1596": 1, "MED-1762": 1, "MED-1764": 1, "MED-1765": 1, "MED-1766": 1, "MED-1778": 1, "MED-1768": 1, "MED-4951": 1, "MED-1770": 1, "MED-1771": 1, "MED-1781": 1, "MED-1773": 1, "MED-1774": 1, "MED-4726": 1, "MED-3935": 1, "MED-3936": 1, "MED-3937": 1, "MED-3938": 1, "MED-3939": 1, "MED-3940": 1, "MED-4398": 1, "MED-4396": 1}, "PLAIN-3171": {"MED-4696": 2, "MED-1486": 1, "MED-1487": 1, "MED-1488": 1, "MED-1489": 1, "MED-1490": 1, "MED-1496": 1, "MED-1497": 1, "MED-1498": 1, "MED-1499": 1, "MED-1500": 1, "MED-1501": 1, "MED-1502": 1, "MED-1503": 1, "MED-1504": 1, "MED-1505": 1, "MED-1506": 1, "MED-5328": 1, "MED-1540": 1, "MED-1541": 1, "MED-1998": 1, "MED-1985": 1, "MED-1986": 1, "MED-1987": 1, "MED-1988": 1, "MED-1999": 1, "MED-1990": 1, "MED-1991": 1, "MED-1992": 1, "MED-1993": 1, "MED-1994": 1, "MED-2221": 1, "MED-1996": 1, "MED-1997": 1}, "PLAIN-3181": {"MED-1574": 1, "MED-1575": 1, "MED-1576": 1, "MED-1577": 1, "MED-1578": 1, "MED-1579": 1, "MED-1580": 1, "MED-1581": 1, "MED-1582": 1, "MED-1789": 1, "MED-1790": 1, "MED-1791": 1, "MED-1792": 1, "MED-1793": 1, "MED-1794": 1, "MED-1795": 1, "MED-3780": 1, "MED-3781": 1, "MED-3782": 1, "MED-3783": 1, "MED-3784": 1, "MED-3785": 1, "MED-3786": 1, "MED-3787": 1, "MED-3788": 1, "MED-3789": 1, "MED-3790": 1}, "PLAIN-3191": {"MED-4729": 2, "MED-4730": 2, "MED-4731": 2, "MED-4732": 2, "MED-1831": 1, "MED-1832": 1, "MED-1833": 1, "MED-1834": 1, "MED-5092": 1, "MED-2750": 1, "MED-2751": 1, "MED-2752": 1, "MED-2753": 1, "MED-2754": 1, "MED-2755": 1, "MED-2756": 1}, "PLAIN-3201": {"MED-4752": 2, "MED-1223": 1, "MED-1224": 1, "MED-5239": 1, "MED-1226": 1, "MED-1227": 1, "MED-2126": 1, "MED-1229": 1, "MED-1230": 1, "MED-1337": 1, "MED-1338": 1, "MED-1339": 1, "MED-1340": 1, "MED-1341": 1, "MED-1762": 1, "MED-1763": 1, "MED-1764": 1, "MED-1765": 1, "MED-1766": 1, "MED-1778": 1, "MED-1768": 1, "MED-4951": 1, "MED-1770": 1, "MED-1771": 1, "MED-1781": 1, "MED-1773": 1, "MED-1774": 1, "MED-3597": 1, "MED-3598": 1, "MED-3599": 1, "MED-5106": 1, "MED-4398": 1}, "PLAIN-3211": {"MED-3810": 1, "MED-3109": 1, "MED-3098": 1, "MED-3099": 1, "MED-3100": 1, "MED-3112": 1, "MED-3102": 1, "MED-3103": 1, "MED-3709": 1, "MED-3105": 1, "MED-3106": 1, "MED-3107": 1, "MED-3108": 1, "MED-3110": 1, "MED-3111": 1, "MED-4040": 1, "MED-4976": 1, "MED-5197": 1}, "PLAIN-3221": {"MED-4790": 2, "MED-2215": 1, "MED-726": 1, "MED-4671": 1, "MED-743": 1, "MED-744": 1, "MED-745": 1, "MED-746": 1, "MED-4193": 1, "MED-3585": 1, "MED-3586": 1, "MED-3587": 1, "MED-3588": 1, "MED-3589": 1, "MED-3590": 1, "MED-3591": 1, "MED-3592": 1, "MED-3593": 1, "MED-4954": 1, "MED-3595": 1, "MED-3596": 1, "MED-4533": 1, "MED-4529": 1, "MED-4535": 1}, "PLAIN-3231": {"MED-4820": 2, "MED-4821": 2, "MED-4436": 1, "MED-4437": 1, "MED-5032": 1, "MED-5033": 1, "MED-5034": 1, "MED-5035": 1}, "PLAIN-3241": {"MED-4841": 2, "MED-1223": 1, "MED-1224": 1, "MED-5239": 1, "MED-1226": 1, "MED-1227": 1, "MED-2126": 1, "MED-1229": 1, "MED-1230": 1, "MED-2053": 1, "MED-2054": 1, "MED-2055": 1, "MED-2056": 1, "MED-2057": 1, "MED-2058": 1, "MED-2059": 1, "MED-2060": 1, "MED-2061": 1, "MED-2062": 1, "MED-2063": 1, "MED-2769": 1, "MED-2770": 1, "MED-2771": 1, "MED-2772": 1, "MED-2773": 1, "MED-2774": 1, "MED-2775": 1, "MED-4398": 1, "MED-4396": 1}, "PLAIN-3251": {"MED-4861": 2, "MED-4390": 1, "MED-1559": 1, "MED-1560": 1, "MED-3557": 1, "MED-3242": 1, "MED-1563": 1, "MED-1564": 1, "MED-1565": 1, "MED-3699": 1, "MED-1567": 1, "MED-3742": 1, "MED-3743": 1, "MED-3744": 1, "MED-3745": 1, "MED-3746": 1, "MED-3747": 1, "MED-3748": 1, "MED-4089": 1, "MED-4090": 1, "MED-4095": 1}, "PLAIN-3261": {"MED-4872": 2, "MED-4873": 2, "MED-4874": 2, "MED-3985": 1, "MED-3986": 1, "MED-3987": 1, "MED-3988": 1, "MED-3989": 1, "MED-3990": 1, "MED-3991": 1, "MED-4566": 1, "MED-4574": 1, "MED-4575": 1}, "PLAIN-3271": {"MED-1527": 1, "MED-1528": 1, "MED-1529": 1, "MED-1530": 1, "MED-1531": 1, "MED-1532": 1, "MED-1559": 1, "MED-1560": 1, "MED-3557": 1, "MED-3242": 1, "MED-1563": 1, "MED-1564": 1, "MED-1565": 1, "MED-3699": 1, "MED-1567": 1, "MED-1598": 1, "MED-1599": 1, "MED-1600": 1, "MED-1601": 1, "MED-1602": 1, "MED-1603": 1, "MED-1604": 1, "MED-1605": 1, "MED-1606": 1, "MED-1607": 1, "MED-4476": 1, "MED-1609": 1, "MED-1610": 1, "MED-1611": 1, "MED-1612": 1, "MED-1613": 1, "MED-1614": 1, "MED-1615": 1, "MED-1616": 1, "MED-1617": 1, "MED-1618": 1, "MED-1619": 1, "MED-1620": 1, "MED-3780": 1, "MED-3781": 1, "MED-3782": 1, "MED-3783": 1, "MED-3784": 1, "MED-3785": 1, "MED-3786": 1, "MED-3787": 1, "MED-3788": 1, "MED-3789": 1, "MED-3790": 1}, "PLAIN-3281": {"MED-4908": 2, "MED-4909": 2, "MED-2248": 1, "MED-2249": 1, "MED-2250": 1, "MED-2251": 1, "MED-2252": 1, "MED-2253": 1, "MED-2254": 1, "MED-2255": 1, "MED-2256": 1, "MED-2257": 1, "MED-2258": 1, "MED-2259": 1, "MED-2260": 1, "MED-2261": 1, "MED-2262": 1, "MED-2263": 1, "MED-2264": 1, "MED-4533": 1, "MED-4529": 1, "MED-4535": 1, "MED-4531": 1, "MED-4532": 1, "MED-4534": 1, "MED-4536": 1}, "PLAIN-3292": {"MED-4925": 2, "MED-2759": 1, "MED-2760": 1, "MED-2761": 1, "MED-4645": 1, "MED-4646": 1, "MED-4647": 1}, "PLAIN-3302": {"MED-4946": 2, "MED-2385": 1, "MED-2386": 1, "MED-2395": 1, "MED-2388": 1, "MED-2389": 1, "MED-2396": 1, "MED-2391": 1, "MED-2407": 1, "MED-3021": 1, "MED-3023": 1, "MED-2904": 1, "MED-2905": 1, "MED-2906": 1, "MED-2907": 1, "MED-3024": 1, "MED-3025": 1, "MED-2910": 1, "MED-3027": 1, "MED-3028": 1, "MED-2913": 1, "MED-3012": 1, "MED-3013": 1, "MED-3030": 1, "MED-2917": 1, "MED-3033": 1, "MED-3034": 1, "MED-3035": 1, "MED-2921": 1, "MED-3019": 1, "MED-3022": 1, "MED-3026": 1, "MED-3029": 1, "MED-3032": 1, "MED-3020": 1, "MED-3031": 1, "MED-4376": 1}, "PLAIN-3312": {"MED-4962": 2, "MED-4963": 2, "MED-4964": 2, "MED-4969": 2, "MED-4966": 2, "MED-4967": 2, "MED-4968": 2, "MED-1568": 1, "MED-1569": 1, "MED-1570": 1, "MED-1571": 1, "MED-1572": 1, "MED-1573": 1}, "PLAIN-3322": {"MED-2926": 1, "MED-2922": 1, "MED-3465": 1, "MED-2923": 1, "MED-2925": 1, "MED-1527": 1, "MED-1528": 1, "MED-1529": 1, "MED-1530": 1, "MED-1531": 1, "MED-1532": 1, "MED-1559": 1, "MED-1560": 1, "MED-3557": 1, "MED-3242": 1, "MED-1563": 1, "MED-1564": 1, "MED-1565": 1, "MED-3699": 1, "MED-1567": 1, "MED-1598": 1, "MED-1599": 1, "MED-1600": 1, "MED-1601": 1, "MED-1602": 1, "MED-1603": 1, "MED-1604": 1, "MED-1605": 1, "MED-1606": 1, "MED-1607": 1, "MED-4476": 1, "MED-3841": 1, "MED-3843": 1, "MED-3849": 1, "MED-3830": 1, "MED-1825": 1, "MED-1826": 1, "MED-1827": 1, "MED-1828": 1, "MED-1829": 1, "MED-1830": 1, "MED-2150": 1, "MED-4646": 1, "MED-2152": 1, "MED-2153": 1, "MED-2568": 1, "MED-4319": 1, "MED-2570": 1, "MED-2571": 1, "MED-2572": 1, "MED-2573": 1, "MED-2574": 1, "MED-2575": 1, "MED-2988": 1, "MED-2559": 1, "MED-2577": 1, "MED-2578": 1, "MED-2579": 1, "MED-2580": 1, "MED-2581": 1, "MED-2583": 1, "MED-2584": 1, "MED-2585": 1, "MED-3270": 1, "MED-3271": 1, "MED-3272": 1, "MED-3273": 1, "MED-3274": 1, "MED-3275": 1, "MED-3276": 1, "MED-3277": 1, "MED-3278": 1, "MED-3279": 1, "MED-3280": 1, "MED-3281": 1, "MED-3282": 1, "MED-3283": 1, "MED-5341": 1, "MED-5337": 1}, "PLAIN-3332": {"MED-4994": 2, "MED-3696": 1, "MED-3697": 1, "MED-3698": 1, "MED-3699": 1, "MED-3700": 1, "MED-3701": 1, "MED-3762": 1, "MED-3763": 1, "MED-3764": 1, "MED-3765": 1, "MED-3766": 1, "MED-3767": 1, "MED-3768": 1}, "PLAIN-3342": {"MED-5012": 2, "MED-5013": 2, "MED-5014": 2}, "PLAIN-3352": {"MED-5026": 2, "MED-5027": 2, "MED-2896": 1, "MED-4384": 1, "MED-2898": 1, "MED-2899": 1, "MED-2900": 1, "MED-2901": 1, "MED-3422": 1, "MED-3423": 1, "MED-3424": 1, "MED-3425": 1, "MED-3426": 1, "MED-3427": 1, "MED-3428": 1, "MED-3407": 1, "MED-3429": 1, "MED-3430": 1, "MED-3432": 1, "MED-3433": 1, "MED-3434": 1, "MED-3435": 1, "MED-3436": 1, "MED-3437": 1, "MED-3438": 1, "MED-3417": 1, "MED-3439": 1, "MED-3440": 1, "MED-3420": 1, "MED-3421": 1, "MED-3540": 1, "MED-3541": 1, "MED-3542": 1, "MED-3543": 1, "MED-3544": 1, "MED-3545": 1, "MED-3546": 1, "MED-3547": 1, "MED-4095": 1}, "PLAIN-3362": {"MED-5041": 2, "MED-5042": 2, "MED-2222": 1, "MED-2223": 1, "MED-2224": 1, "MED-2225": 1, "MED-2226": 1, "MED-2227": 1, "MED-2228": 1, "MED-4286": 1, "MED-4281": 1}, "PLAIN-3372": {"MED-5056": 2, "MED-3742": 1, "MED-2826": 1, "MED-2827": 1, "MED-2830": 1, "MED-2831": 1, "MED-4523": 1, "MED-2966": 1, "MED-2967": 1, "MED-2968": 1, "MED-2969": 1, "MED-2970": 1, "MED-2971": 1, "MED-2972": 1}, "PLAIN-3382": {"MED-5062": 2, "MED-5063": 2, "MED-3378": 1, "MED-3380": 1, "MED-3382": 1, "MED-2616": 1, "MED-2617": 1, "MED-2618": 1, "MED-2619": 1, "MED-2992": 1, "MED-2993": 1, "MED-2994": 1, "MED-3379": 1, "MED-3381": 1}, "PLAIN-3392": {"MED-1836": 1, "MED-1837": 1, "MED-1838": 1, "MED-1839": 1, "MED-1840": 1, "MED-1841": 1, "MED-1842": 1, "MED-1843": 1, "MED-1844": 1, "MED-1845": 1, "MED-1846": 1, "MED-3918": 1, "MED-3919": 1, "MED-3920": 1, "MED-3921": 1, "MED-3922": 1, "MED-3923": 1, "MED-3924": 1, "MED-4049": 1, "MED-4050": 1, "MED-4051": 1, "MED-4055": 1, "MED-4332": 1}, "PLAIN-3402": {"MED-4544": 1, "MED-2008": 1, "MED-2009": 1, "MED-2010": 1, "MED-2011": 1, "MED-2830": 1, "MED-2831": 1, "MED-3742": 1, "MED-3743": 1, "MED-3744": 1, "MED-3745": 1, "MED-3746": 1, "MED-3747": 1, "MED-3748": 1}, "PLAIN-3412": {"MED-5108": 2, "MED-5109": 2, "MED-1703": 1, "MED-1699": 1, "MED-1700": 1, "MED-1701": 1, "MED-1702": 1, "MED-1762": 1, "MED-1763": 1, "MED-1764": 1, "MED-1765": 1, "MED-1766": 1, "MED-1778": 1, "MED-1768": 1, "MED-4951": 1, "MED-1770": 1, "MED-1771": 1, "MED-1781": 1, "MED-1773": 1, "MED-1774": 1, "MED-3226": 1, "MED-3227": 1, "MED-3228": 1, "MED-3229": 1, "MED-3230": 1, "MED-3231": 1, "MED-3232": 1, "MED-3233": 1, "MED-4722": 1, "MED-3235": 1, "MED-3236": 1, "MED-3237": 1, "MED-3597": 1, "MED-3598": 1, "MED-3599": 1, "MED-3780": 1, "MED-3781": 1, "MED-3782": 1, "MED-3783": 1, "MED-3784": 1, "MED-3785": 1, "MED-3786": 1, "MED-3787": 1, "MED-3788": 1, "MED-3789": 1, "MED-3790": 1, "MED-4319": 1, "MED-4320": 1, "MED-4682": 1}, "PLAIN-3422": {"MED-5128": 2, "MED-5129": 2, "MED-5130": 2, "MED-5131": 2, "MED-5132": 2, "MED-5133": 2, "MED-5134": 2, "MED-5135": 2, "MED-2938": 1, "MED-2939": 1, "MED-2940": 1, "MED-2941": 1, "MED-2942": 1, "MED-2943": 1, "MED-2945": 1, "MED-2946": 1, "MED-2947": 1, "MED-3985": 1, "MED-3986": 1, "MED-3987": 1, "MED-3988": 1, "MED-3989": 1, "MED-3990": 1, "MED-3991": 1, "MED-4511": 1, "MED-4512": 1, "MED-4513": 1, "MED-4566": 1, "MED-4574": 1, "MED-4575": 1, "MED-4925": 1}, "PLAIN-3432": {"MED-2274": 1, "MED-3535": 1, "MED-2222": 1, "MED-2223": 1, "MED-2224": 1, "MED-2225": 1, "MED-2226": 1, "MED-2227": 1, "MED-2228": 1, "MED-2279": 1, "MED-2280": 1, "MED-2281": 1, "MED-2282": 1, "MED-3464": 1, "MED-2284": 1, "MED-2285": 1, "MED-2287": 1, "MED-3518": 1, "MED-3519": 1, "MED-3520": 1, "MED-3521": 1, "MED-3522": 1, "MED-3523": 1, "MED-3524": 1, "MED-3525": 1, "MED-3526": 1, "MED-3527": 1, "MED-3528": 1, "MED-3539": 1, "MED-3530": 1, "MED-3531": 1, "MED-3532": 1, "MED-3533": 1, "MED-3534": 1, "MED-3906": 1, "MED-3907": 1, "MED-3908": 1, "MED-4715": 1, "MED-3910": 1, "MED-4106": 1, "MED-4286": 1, "MED-4281": 1}, "PLAIN-3442": {"MED-5159": 2, "MED-5160": 2, "MED-5161": 2, "MED-5162": 2, "MED-4390": 1, "MED-1880": 1, "MED-1881": 1, "MED-2098": 1, "MED-3548": 1, "MED-3549": 1, "MED-3550": 1, "MED-3551": 1, "MED-3552": 1, "MED-3553": 1, "MED-3554": 1, "MED-3555": 1, "MED-4049": 1, "MED-4053": 1, "MED-4051": 1, "MED-5116": 1, "MED-4047": 1, "MED-4055": 1, "MED-4286": 1, "MED-4281": 1, "MED-4585": 1, "MED-4590": 1, "MED-4587": 1, "MED-4588": 1, "MED-4589": 1}, "PLAIN-3452": {"MED-5177": 2, "MED-5175": 2, "MED-2099": 1, "MED-2100": 1, "MED-2101": 1, "MED-2102": 1, "MED-2103": 1, "MED-2529": 1, "MED-2105": 1, "MED-2106": 1, "MED-2107": 1, "MED-2108": 1, "MED-4637": 1, "MED-4638": 1, "MED-4639": 1, "MED-4640": 1, "MED-4641": 1}, "PLAIN-3462": {"MED-5260": 2, "MED-5261": 2, "MED-5262": 2, "MED-5263": 2, "MED-5278": 2, "MED-5265": 2, "MED-5266": 2, "MED-5267": 2, "MED-5268": 2, "MED-5269": 2, "MED-5270": 2, "MED-5271": 2, "MED-5272": 2, "MED-5273": 2, "MED-5274": 2, "MED-1401": 1, "MED-1402": 1, "MED-1366": 1, "MED-1403": 1, "MED-1404": 1, "MED-1405": 1, "MED-1406": 1, "MED-1407": 1, "MED-1408": 1, "MED-1409": 1, "MED-1410": 1, "MED-1411": 1, "MED-1634": 1, "MED-1635": 1, "MED-1636": 1, "MED-1637": 1, "MED-1638": 1, "MED-1639": 1, "MED-1640": 1, "MED-1641": 1, "MED-1642": 1, "MED-1643": 1, "MED-5258": 1, "MED-1645": 1, "MED-1646": 1, "MED-1647": 1, "MED-1648": 1, "MED-1649": 1, "MED-2222": 1, "MED-2223": 1, "MED-2224": 1, "MED-2225": 1, "MED-2226": 1, "MED-2227": 1, "MED-2228": 1, "MED-2370": 1, "MED-2371": 1, "MED-2372": 1, "MED-4324": 1, "MED-2376": 1, "MED-2593": 1, "MED-2378": 1, "MED-2379": 1, "MED-2380": 1, "MED-2381": 1, "MED-2382": 1, "MED-2383": 1, "MED-2384": 1, "MED-4249": 1}, "PLAIN-3472": {"MED-5343": 2, "MED-5344": 2, "MED-5345": 2, "MED-5346": 2, "MED-5347": 2, "MED-1487": 1, "MED-1482": 1, "MED-1483": 1, "MED-1484": 1, "MED-1490": 1, "MED-3113": 1, "MED-4247": 1, "MED-3623": 1, "MED-3624": 1, "MED-3625": 1, "MED-3626": 1, "MED-3627": 1}} \ No newline at end of file diff --git a/evals/benchmark/results/nfcorpus/run-dense.trec.gz b/evals/benchmark/results/nfcorpus/run-dense.trec.gz new file mode 100644 index 000000000..9dfb71ac0 Binary files /dev/null and b/evals/benchmark/results/nfcorpus/run-dense.trec.gz differ diff --git a/evals/benchmark/results/nfcorpus/run-w0.25.trec.gz b/evals/benchmark/results/nfcorpus/run-w0.25.trec.gz new file mode 100644 index 000000000..8d6101f4f Binary files /dev/null and b/evals/benchmark/results/nfcorpus/run-w0.25.trec.gz differ diff --git a/evals/benchmark/results/nfcorpus/run-w0.5.trec.gz b/evals/benchmark/results/nfcorpus/run-w0.5.trec.gz new file mode 100644 index 000000000..691f6d3ac Binary files /dev/null and b/evals/benchmark/results/nfcorpus/run-w0.5.trec.gz differ diff --git a/evals/benchmark/results/nfcorpus/run-w0.75.trec.gz b/evals/benchmark/results/nfcorpus/run-w0.75.trec.gz new file mode 100644 index 000000000..bfb7a264f Binary files /dev/null and b/evals/benchmark/results/nfcorpus/run-w0.75.trec.gz differ diff --git a/evals/benchmark/results/nfcorpus/run-w1.0.trec.gz b/evals/benchmark/results/nfcorpus/run-w1.0.trec.gz new file mode 100644 index 000000000..c26a3cbf1 Binary files /dev/null and b/evals/benchmark/results/nfcorpus/run-w1.0.trec.gz differ diff --git a/evals/benchmark/results/nfcorpus/stats-w0.25-vs-dense.jsonl b/evals/benchmark/results/nfcorpus/stats-w0.25-vs-dense.jsonl new file mode 100644 index 000000000..b73253f08 --- /dev/null +++ b/evals/benchmark/results/nfcorpus/stats-w0.25-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w0.25", "arm_b": "dense"} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "nDCG@10", "n": 322, "mean_a": 0.3766503382385221, "mean_b": 0.36660768485593975, "mean_diff": -0.010042653382582369, "ci_low": -0.01796642508970083, "ci_high": -0.002241685002275251, "p_value": 0.012398760123987601, "significant": true} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "Recall@20", "n": 322, "mean_a": 0.2135261475395606, "mean_b": 0.2135261475395606, "mean_diff": 0.0, "ci_low": 0.0, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "MRR@10", "n": 322, "mean_a": 0.6015926275243045, "mean_b": 0.5918453065831581, "mean_diff": -0.009747320941146297, "ci_low": -0.03331158475602666, "ci_high": 0.013405486396443645, "p_value": 0.4206579342065793, "significant": false} diff --git a/evals/benchmark/results/nfcorpus/stats-w0.5-vs-dense.jsonl b/evals/benchmark/results/nfcorpus/stats-w0.5-vs-dense.jsonl new file mode 100644 index 000000000..3e7cbc7c0 --- /dev/null +++ b/evals/benchmark/results/nfcorpus/stats-w0.5-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w0.5", "arm_b": "dense"} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "nDCG@10", "n": 322, "mean_a": 0.3756124710889875, "mean_b": 0.36660768485593975, "mean_diff": -0.009004786233047775, "ci_low": -0.01739290052306032, "ci_high": -0.0006621888962514344, "p_value": 0.0377962203779622, "significant": true} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "Recall@20", "n": 322, "mean_a": 0.2135261475395606, "mean_b": 0.2135261475395606, "mean_diff": 0.0, "ci_low": 0.0, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "MRR@10", "n": 322, "mean_a": 0.591085381147493, "mean_b": 0.5918453065831581, "mean_diff": 0.0007599254356652975, "ci_low": -0.026191611348603493, "ci_high": 0.026991407805140897, "p_value": 0.9528047195280472, "significant": false} diff --git a/evals/benchmark/results/nfcorpus/stats-w0.75-vs-dense.jsonl b/evals/benchmark/results/nfcorpus/stats-w0.75-vs-dense.jsonl new file mode 100644 index 000000000..6211157e6 --- /dev/null +++ b/evals/benchmark/results/nfcorpus/stats-w0.75-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w0.75", "arm_b": "dense"} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "nDCG@10", "n": 322, "mean_a": 0.3742649985734925, "mean_b": 0.36660768485593975, "mean_diff": -0.007657313717552728, "ci_low": -0.01618285260141904, "ci_high": 0.0009490136713570532, "p_value": 0.0851914808519148, "significant": false} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "Recall@20", "n": 322, "mean_a": 0.2135261475395606, "mean_b": 0.2135261475395606, "mean_diff": 0.0, "ci_low": 0.0, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "MRR@10", "n": 322, "mean_a": 0.5887007316355143, "mean_b": 0.5918453065831581, "mean_diff": 0.003144574947644002, "ci_low": -0.023807734277454775, "ci_high": 0.029881292932192643, "p_value": 0.8169183081691831, "significant": false} diff --git a/evals/benchmark/results/nfcorpus/stats-w1.0-vs-dense.jsonl b/evals/benchmark/results/nfcorpus/stats-w1.0-vs-dense.jsonl new file mode 100644 index 000000000..1ab2e9744 --- /dev/null +++ b/evals/benchmark/results/nfcorpus/stats-w1.0-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w1.0", "arm_b": "dense"} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "nDCG@10", "n": 322, "mean_a": 0.3733682232834116, "mean_b": 0.36660768485593975, "mean_diff": -0.006760538427471916, "ci_low": -0.01694383424960257, "ci_high": 0.0036739432646231596, "p_value": 0.21177882211778823, "significant": false} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "Recall@20", "n": 322, "mean_a": 0.21594796166795874, "mean_b": 0.2135261475395606, "mean_diff": -0.002421814128398175, "ci_low": -0.008410688244491336, "ci_high": 0.0035884774847761993, "p_value": 0.45565443455654436, "significant": false} +{"row_type": "ir", "dataset": "nfcorpus", "metric": "MRR@10", "n": 322, "mean_a": 0.5885255393676783, "mean_b": 0.5918453065831581, "mean_diff": 0.0033197672154799632, "ci_low": -0.026048960073860084, "ci_high": 0.03189871075028241, "p_value": 0.8223177682231777, "significant": false} diff --git a/evals/benchmark/results/scifact/ir-dense.jsonl b/evals/benchmark/results/scifact/ir-dense.jsonl new file mode 100644 index 000000000..e368e208d --- /dev/null +++ b/evals/benchmark/results/scifact/ir-dense.jsonl @@ -0,0 +1 @@ +{"dataset": "scifact", "run_tag": "dense", "aggregated": {"nDCG@10": 0.6981, "Recall@20": 0.8633, "MRR@10": 0.6648}, "per_query": {"nDCG@10": {"1": 0.0, "3": 0.5, "5": 1.0, "13": 0.0, "36": 0.21840743681816419, "42": 1.0, "48": 0.5, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.0, "72": 0.3562071871080222, "75": 1.0, "94": 0.0, "99": 0.3333333333333333, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 0.5, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 0.9560290452802053, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 0.6309297535714575, "163": 1.0, "171": 1.0, "179": 0.9267582364714126, "180": 1.0, "183": 1.0, "185": 0.6309297535714575, "198": 0.0, "208": 1.0, "212": 0.5, "213": 0.3333333333333333, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.6309297535714575, "233": 1.0, "236": 1.0, "237": 0.6309297535714575, "238": 0.2890648263178879, "239": 0.3333333333333333, "248": 1.0, "249": 1.0, "261": 0.6052602440527057, "268": 0.5, "269": 1.0, "274": 1.0, "275": 0.7039180890341347, "279": 1.0, "294": 0.0, "295": 0.6309297535714575, "298": 1.0, "300": 0.0, "303": 0.0, "312": 0.0, "314": 0.3562071871080222, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.6509209298071326, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.5, "385": 0.6509209298071326, "386": 1.0, "388": 0.6309297535714575, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.6309297535714575, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.6131471927654584, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.6309297535714575, "508": 0.3333333333333333, "513": 0.6309297535714575, "514": 1.0, "516": 1.0, "517": 0.5, "521": 1.0, "525": 1.0, "527": 0.5, "528": 1.0, "532": 1.0, "533": 0.5, "535": 0.5, "536": 1.0, "539": 1.0, "540": 0.6131471927654584, "544": 0.5, "549": 0.3562071871080222, "551": 0.3333333333333333, "552": 1.0, "554": 0.0, "560": 0.0, "569": 0.38685280723454163, "575": 1.0, "577": 0.6309297535714575, "578": 1.0, "587": 0.6309297535714575, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.6309297535714575, "613": 1.0, "619": 0.0, "623": 0.6309297535714575, "628": 0.6309297535714575, "636": 1.0, "637": 0.5, "641": 0.8315546295836225, "644": 0.6309297535714575, "649": 1.0, "659": 0.0, "660": 0.0, "674": 0.5, "684": 0.6309297535714575, "690": 0.0, "691": 0.6309297535714575, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.38685280723454163, "728": 0.8772153153380493, "729": 1.0, "742": 0.6309297535714575, "743": 0.6309297535714575, "744": 0.5, "756": 0.6309297535714575, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.0, "784": 1.0, "785": 1.0, "793": 0.43067655807339306, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.2890648263178879, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.5, "847": 1.0, "852": 0.6309297535714575, "859": 0.6309297535714575, "870": 0.0, "873": 0.7860137352654724, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 0.9828920819566879, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.6309297535714575, "1012": 1.0, "1014": 1.0, "1019": 0.6309297535714575, "1020": 0.6309297535714575, "1021": 0.6309297535714575, "1024": 0.38685280723454163, "1029": 0.9217868789962071, "1041": 0.430624116386567, "1049": 0.3333333333333333, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 0.38685280723454163, "1100": 0.2890648263178879, "1104": 1.0, "1107": 0.6309297535714575, "1110": 0.2890648263178879, "1121": 1.0, "1130": 1.0, "1132": 0.37800210039683896, "1137": 0.6309297535714575, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.3333333333333333, "1185": 1.0, "1187": 0.6309297535714575, "1191": 0.43067655807339306, "1194": 1.0, "1196": 0.43067655807339306, "1197": 0.38685280723454163, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 0.3333333333333333, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 0.6309297535714575, "1273": 1.0, "1274": 0.8278123145308894, "1278": 0.0, "1279": 0.0, "1280": 0.3333333333333333, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 0.0, "1316": 0.0, "1319": 0.5, "1320": 0.6309297535714575, "1332": 0.0, "1335": 0.6309297535714575, "1336": 0.6309297535714575, "1337": 1.0, "1339": 1.0, "1344": 0.6309297535714575, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 1.0, "1370": 1.0, "1379": 0.9267582364714126, "1382": 0.6309297535714575, "1385": 1.0, "1389": 1.0, "1395": 1.0}, "Recall@20": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 1.0, "42": 1.0, "48": 1.0, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.5, "72": 1.0, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 1.0, "213": 1.0, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 1.0, "233": 1.0, "236": 1.0, "237": 1.0, "238": 1.0, "239": 1.0, "248": 1.0, "249": 1.0, "261": 1.0, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 1.0, "303": 0.0, "312": 0.0, "314": 1.0, "324": 0.0, "327": 1.0, "338": 1.0, "343": 1.0, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 1.0, "385": 1.0, "386": 1.0, "388": 1.0, "399": 1.0, "410": 1.0, "411": 1.0, "415": 1.0, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.5, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 1.0, "508": 1.0, "513": 1.0, "514": 1.0, "516": 1.0, "517": 1.0, "521": 1.0, "525": 1.0, "527": 1.0, "528": 1.0, "532": 1.0, "533": 1.0, "535": 1.0, "536": 1.0, "539": 1.0, "540": 1.0, "544": 1.0, "549": 1.0, "551": 1.0, "552": 1.0, "554": 1.0, "560": 0.0, "569": 1.0, "575": 1.0, "577": 1.0, "578": 1.0, "587": 1.0, "589": 1.0, "593": 1.0, "597": 1.0, "598": 1.0, "613": 1.0, "619": 0.0, "623": 1.0, "628": 1.0, "636": 1.0, "637": 1.0, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.0, "660": 0.0, "674": 1.0, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 1.0, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 1.0, "756": 1.0, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 1.0, "784": 1.0, "785": 1.0, "793": 1.0, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 1.0, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 1.0, "847": 1.0, "852": 1.0, "859": 1.0, "870": 1.0, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 1.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 1.0, "1012": 1.0, "1014": 1.0, "1019": 1.0, "1020": 1.0, "1021": 1.0, "1024": 1.0, "1029": 1.0, "1041": 1.0, "1049": 1.0, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 1.0, "1100": 1.0, "1104": 1.0, "1107": 1.0, "1110": 1.0, "1121": 1.0, "1130": 1.0, "1132": 1.0, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 1.0, "1185": 1.0, "1187": 1.0, "1191": 1.0, "1194": 1.0, "1196": 1.0, "1197": 1.0, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 1.0, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 1.0, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 1.0, "1279": 0.0, "1280": 1.0, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 1.0, "1320": 1.0, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 1.0, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 1.0, "1370": 1.0, "1379": 1.0, "1382": 1.0, "1385": 1.0, "1389": 1.0, "1395": 1.0}, "MRR@10": {"1": 0.0, "3": 0.3333333333333333, "5": 1.0, "13": 0.0, "36": 0.16666666666666666, "42": 1.0, "48": 0.3333333333333333, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.07692307692307693, "72": 0.16666666666666666, "75": 1.0, "94": 0.0, "99": 0.14285714285714285, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 0.3333333333333333, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 0.5, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 0.5, "198": 0.0, "208": 1.0, "212": 0.3333333333333333, "213": 0.14285714285714285, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.5, "233": 1.0, "236": 1.0, "237": 0.5, "238": 0.1, "239": 0.14285714285714285, "248": 1.0, "249": 1.0, "261": 0.5, "268": 0.3333333333333333, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 0.5, "298": 1.0, "300": 0.05, "303": 0.0, "312": 0.0, "314": 0.16666666666666666, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.5, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.3333333333333333, "385": 0.5, "386": 1.0, "388": 0.5, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.5, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 1.0, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.5, "508": 0.14285714285714285, "513": 0.5, "514": 1.0, "516": 1.0, "517": 0.3333333333333333, "521": 1.0, "525": 1.0, "527": 0.3333333333333333, "528": 1.0, "532": 1.0, "533": 0.3333333333333333, "535": 0.3333333333333333, "536": 1.0, "539": 1.0, "540": 1.0, "544": 0.3333333333333333, "549": 0.16666666666666666, "551": 0.14285714285714285, "552": 1.0, "554": 0.08333333333333333, "560": 0.0, "569": 0.2, "575": 1.0, "577": 0.5, "578": 1.0, "587": 0.5, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.5, "613": 1.0, "619": 0.0, "623": 0.5, "628": 0.5, "636": 1.0, "637": 0.3333333333333333, "641": 1.0, "644": 0.5, "649": 1.0, "659": 0.0, "660": 0.0, "674": 0.3333333333333333, "684": 0.5, "690": 0.0, "691": 0.5, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.2, "728": 1.0, "729": 1.0, "742": 0.5, "743": 0.5, "744": 0.3333333333333333, "756": 0.5, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.07142857142857142, "784": 1.0, "785": 1.0, "793": 0.25, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.1, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.3333333333333333, "847": 1.0, "852": 0.5, "859": 0.5, "870": 0.09090909090909091, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0625, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.5, "1012": 1.0, "1014": 1.0, "1019": 0.5, "1020": 0.5, "1021": 0.5, "1024": 0.2, "1029": 1.0, "1041": 0.2, "1049": 0.14285714285714285, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 0.2, "1100": 0.1, "1104": 1.0, "1107": 0.5, "1110": 0.1, "1121": 1.0, "1130": 1.0, "1132": 0.125, "1137": 0.5, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.14285714285714285, "1185": 1.0, "1187": 0.5, "1191": 0.25, "1194": 1.0, "1196": 0.25, "1197": 0.2, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.08333333333333333, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 0.14285714285714285, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 0.5, "1273": 1.0, "1274": 1.0, "1278": 0.06666666666666667, "1279": 0.0, "1280": 0.14285714285714285, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 0.09090909090909091, "1316": 0.0, "1319": 0.3333333333333333, "1320": 0.5, "1332": 0.0, "1335": 0.5, "1336": 0.5, "1337": 1.0, "1339": 1.0, "1344": 0.5, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 1.0, "1370": 1.0, "1379": 1.0, "1382": 0.5, "1385": 1.0, "1389": 1.0, "1395": 1.0}}} diff --git a/evals/benchmark/results/scifact/ir-w0.25.jsonl b/evals/benchmark/results/scifact/ir-w0.25.jsonl new file mode 100644 index 000000000..7d7269eb8 --- /dev/null +++ b/evals/benchmark/results/scifact/ir-w0.25.jsonl @@ -0,0 +1 @@ +{"dataset": "scifact", "run_tag": "w0.25", "aggregated": {"nDCG@10": 0.7155, "Recall@20": 0.8633, "MRR@10": 0.6838}, "per_query": {"nDCG@10": {"1": 0.0, "3": 0.6309297535714575, "5": 1.0, "13": 0.0, "36": 0.2640681225725909, "42": 1.0, "48": 0.5, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.20438239758848611, "72": 0.6309297535714575, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 0.8712322899646365, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 0.6309297535714575, "163": 1.0, "171": 1.0, "179": 0.7095272044910244, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.5, "213": 0.38685280723454163, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.6309297535714575, "233": 1.0, "236": 1.0, "237": 0.6309297535714575, "238": 0.43067655807339306, "239": 0.5, "248": 1.0, "249": 1.0, "261": 0.5912352048230277, "268": 0.6309297535714575, "269": 1.0, "274": 1.0, "275": 0.7039180890341347, "279": 1.0, "294": 0.0, "295": 0.6309297535714575, "298": 1.0, "300": 0.43067655807339306, "303": 0.0, "312": 0.0, "314": 0.5, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.6934264036172708, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.3562071871080222, "385": 0.6934264036172708, "386": 1.0, "388": 0.6309297535714575, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.6309297535714575, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3065735963827292, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.6309297535714575, "508": 0.3010299956639812, "513": 0.6309297535714575, "514": 1.0, "516": 1.0, "517": 0.3333333333333333, "521": 1.0, "525": 1.0, "527": 0.6309297535714575, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.3333333333333333, "536": 1.0, "539": 1.0, "540": 0.38685280723454163, "544": 0.5, "549": 1.0, "551": 0.3562071871080222, "552": 1.0, "554": 0.3010299956639812, "560": 0.0, "569": 0.38685280723454163, "575": 1.0, "577": 0.6309297535714575, "578": 1.0, "587": 0.6309297535714575, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.6309297535714575, "613": 1.0, "619": 0.0, "623": 0.0, "628": 1.0, "636": 1.0, "637": 0.5, "641": 0.8065735963827293, "644": 1.0, "649": 1.0, "659": 0.0, "660": 0.0, "674": 0.5, "684": 0.6309297535714575, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.43067655807339306, "728": 0.8772153153380493, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.6309297535714575, "756": 0.5, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.3562071871080222, "784": 1.0, "785": 1.0, "793": 0.6309297535714575, "800": 0.6309297535714575, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.43067655807339306, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.5, "847": 1.0, "852": 1.0, "859": 0.6309297535714575, "870": 0.0, "873": 0.7870111744655099, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.6309297535714575, "1012": 1.0, "1014": 1.0, "1019": 0.6309297535714575, "1020": 0.6309297535714575, "1021": 0.6309297535714575, "1024": 0.5, "1029": 0.7653606369886217, "1041": 0.4415801103577823, "1049": 0.3333333333333333, "1062": 1.0, "1086": 1.0, "1088": 0.43067655807339306, "1089": 1.0, "1099": 0.43067655807339306, "1100": 0.3333333333333333, "1104": 1.0, "1107": 0.6309297535714575, "1110": 0.2890648263178879, "1121": 1.0, "1130": 1.0, "1132": 0.21840743681816419, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.38685280723454163, "1185": 1.0, "1187": 1.0, "1191": 0.38685280723454163, "1194": 1.0, "1196": 0.43067655807339306, "1197": 0.38685280723454163, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 0.38685280723454163, "1225": 1.0, "1226": 0.3010299956639812, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 0.6309297535714575, "1273": 1.0, "1274": 0.8278123145308894, "1278": 0.5, "1279": 0.0, "1280": 0.31546487678572877, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.6309297535714575, "1320": 0.6309297535714575, "1332": 0.0, "1335": 0.6309297535714575, "1336": 0.6309297535714575, "1337": 1.0, "1339": 1.0, "1344": 0.6309297535714575, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 0.5, "1370": 1.0, "1379": 0.7095272044910244, "1382": 0.5, "1385": 1.0, "1389": 1.0, "1395": 0.5}, "Recall@20": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 1.0, "42": 1.0, "48": 1.0, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.5, "72": 1.0, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 1.0, "213": 1.0, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 1.0, "233": 1.0, "236": 1.0, "237": 1.0, "238": 1.0, "239": 1.0, "248": 1.0, "249": 1.0, "261": 1.0, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 1.0, "303": 0.0, "312": 0.0, "314": 1.0, "324": 0.0, "327": 1.0, "338": 1.0, "343": 1.0, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 1.0, "385": 1.0, "386": 1.0, "388": 1.0, "399": 1.0, "410": 1.0, "411": 1.0, "415": 1.0, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.5, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 1.0, "508": 1.0, "513": 1.0, "514": 1.0, "516": 1.0, "517": 1.0, "521": 1.0, "525": 1.0, "527": 1.0, "528": 1.0, "532": 1.0, "533": 1.0, "535": 1.0, "536": 1.0, "539": 1.0, "540": 1.0, "544": 1.0, "549": 1.0, "551": 1.0, "552": 1.0, "554": 1.0, "560": 0.0, "569": 1.0, "575": 1.0, "577": 1.0, "578": 1.0, "587": 1.0, "589": 1.0, "593": 1.0, "597": 1.0, "598": 1.0, "613": 1.0, "619": 0.0, "623": 1.0, "628": 1.0, "636": 1.0, "637": 1.0, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.0, "660": 0.0, "674": 1.0, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 1.0, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 1.0, "756": 1.0, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 1.0, "784": 1.0, "785": 1.0, "793": 1.0, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 1.0, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 1.0, "847": 1.0, "852": 1.0, "859": 1.0, "870": 1.0, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 1.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 1.0, "1012": 1.0, "1014": 1.0, "1019": 1.0, "1020": 1.0, "1021": 1.0, "1024": 1.0, "1029": 1.0, "1041": 1.0, "1049": 1.0, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 1.0, "1100": 1.0, "1104": 1.0, "1107": 1.0, "1110": 1.0, "1121": 1.0, "1130": 1.0, "1132": 1.0, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 1.0, "1185": 1.0, "1187": 1.0, "1191": 1.0, "1194": 1.0, "1196": 1.0, "1197": 1.0, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 1.0, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 1.0, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 1.0, "1279": 0.0, "1280": 1.0, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 1.0, "1320": 1.0, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 1.0, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 1.0, "1370": 1.0, "1379": 1.0, "1382": 1.0, "1385": 1.0, "1389": 1.0, "1395": 1.0}, "MRR@10": {"1": 0.0, "3": 0.5, "5": 1.0, "13": 0.0, "36": 0.25, "42": 1.0, "48": 0.3333333333333333, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.14285714285714285, "72": 0.5, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 0.5, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.3333333333333333, "213": 0.2, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.5, "233": 1.0, "236": 1.0, "237": 0.5, "238": 0.25, "239": 0.3333333333333333, "248": 1.0, "249": 1.0, "261": 0.5, "268": 0.5, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 0.5, "298": 1.0, "300": 0.25, "303": 0.0, "312": 0.0, "314": 0.3333333333333333, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.5, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.16666666666666666, "385": 0.5, "386": 1.0, "388": 0.5, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.5, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3333333333333333, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.5, "508": 0.1111111111111111, "513": 0.5, "514": 1.0, "516": 1.0, "517": 0.14285714285714285, "521": 1.0, "525": 1.0, "527": 0.5, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.14285714285714285, "536": 1.0, "539": 1.0, "540": 0.5, "544": 0.3333333333333333, "549": 1.0, "551": 0.16666666666666666, "552": 1.0, "554": 0.1111111111111111, "560": 0.0, "569": 0.2, "575": 1.0, "577": 0.5, "578": 1.0, "587": 0.5, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.5, "613": 1.0, "619": 0.0, "623": 0.08333333333333333, "628": 1.0, "636": 1.0, "637": 0.3333333333333333, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.0, "660": 0.0, "674": 0.3333333333333333, "684": 0.5, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.25, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.5, "756": 0.3333333333333333, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.16666666666666666, "784": 1.0, "785": 1.0, "793": 0.5, "800": 0.5, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.25, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.3333333333333333, "847": 1.0, "852": 1.0, "859": 0.5, "870": 0.07142857142857142, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0625, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.5, "1012": 1.0, "1014": 1.0, "1019": 0.5, "1020": 0.5, "1021": 0.5, "1024": 0.3333333333333333, "1029": 1.0, "1041": 0.2, "1049": 0.14285714285714285, "1062": 1.0, "1086": 1.0, "1088": 0.25, "1089": 1.0, "1099": 0.25, "1100": 0.14285714285714285, "1104": 1.0, "1107": 0.5, "1110": 0.1, "1121": 1.0, "1130": 1.0, "1132": 0.16666666666666666, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.2, "1185": 1.0, "1187": 1.0, "1191": 0.2, "1194": 1.0, "1196": 0.25, "1197": 0.2, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.07692307692307693, "1216": 1.0, "1221": 0.2, "1225": 1.0, "1226": 0.1111111111111111, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 0.5, "1273": 1.0, "1274": 1.0, "1278": 0.3333333333333333, "1279": 0.0, "1280": 0.125, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.5, "1320": 0.5, "1332": 0.0, "1335": 0.5, "1336": 0.5, "1337": 1.0, "1339": 1.0, "1344": 0.5, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 0.3333333333333333, "1370": 1.0, "1379": 1.0, "1382": 0.3333333333333333, "1385": 1.0, "1389": 1.0, "1395": 0.3333333333333333}}} diff --git a/evals/benchmark/results/scifact/ir-w0.5.jsonl b/evals/benchmark/results/scifact/ir-w0.5.jsonl new file mode 100644 index 000000000..307d5c3da --- /dev/null +++ b/evals/benchmark/results/scifact/ir-w0.5.jsonl @@ -0,0 +1 @@ +{"dataset": "scifact", "run_tag": "w0.5", "aggregated": {"nDCG@10": 0.7167, "Recall@20": 0.8633, "MRR@10": 0.6846}, "per_query": {"nDCG@10": {"1": 0.0, "3": 0.6309297535714575, "5": 1.0, "13": 0.0, "36": 0.2640681225725909, "42": 1.0, "48": 0.6309297535714575, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.23719771276929622, "72": 0.6309297535714575, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 0.8712322899646365, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 0.6309297535714575, "163": 1.0, "171": 1.0, "179": 0.7095272044910244, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.5, "213": 0.43067655807339306, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.6309297535714575, "233": 1.0, "236": 1.0, "237": 0.6309297535714575, "238": 0.5, "239": 0.43067655807339306, "248": 1.0, "249": 1.0, "261": 0.46845052016107697, "268": 1.0, "269": 1.0, "274": 1.0, "275": 0.6508205185601091, "279": 1.0, "294": 0.0, "295": 0.6309297535714575, "298": 1.0, "300": 0.43067655807339306, "303": 0.0, "312": 0.0, "314": 0.5, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.6509209298071326, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.3562071871080222, "385": 0.6934264036172708, "386": 1.0, "388": 0.6309297535714575, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.5, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3065735963827292, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.6309297535714575, "508": 0.3010299956639812, "513": 0.6309297535714575, "514": 1.0, "516": 1.0, "517": 0.3333333333333333, "521": 1.0, "525": 1.0, "527": 0.6309297535714575, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.3333333333333333, "536": 1.0, "539": 1.0, "540": 0.38685280723454163, "544": 0.38685280723454163, "549": 1.0, "551": 0.38685280723454163, "552": 1.0, "554": 0.3010299956639812, "560": 0.0, "569": 0.6309297535714575, "575": 1.0, "577": 0.6309297535714575, "578": 1.0, "587": 0.6309297535714575, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.5, "613": 1.0, "619": 0.0, "623": 0.0, "628": 1.0, "636": 1.0, "637": 0.6309297535714575, "641": 0.8065735963827293, "644": 1.0, "649": 0.6309297535714575, "659": 0.0, "660": 0.0, "674": 0.6309297535714575, "684": 0.6309297535714575, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.43067655807339306, "728": 0.6934264036172708, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.6309297535714575, "756": 0.5, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.3333333333333333, "784": 1.0, "785": 1.0, "793": 0.6309297535714575, "800": 0.38685280723454163, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.43067655807339306, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.6309297535714575, "847": 1.0, "852": 1.0, "859": 0.6309297535714575, "870": 0.0, "873": 0.7870111744655099, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.6309297535714575, "1012": 1.0, "1014": 1.0, "1019": 0.6309297535714575, "1020": 0.6309297535714575, "1021": 0.6309297535714575, "1024": 0.5, "1029": 0.7653606369886217, "1041": 0.39780880120575696, "1049": 0.3333333333333333, "1062": 1.0, "1086": 1.0, "1088": 0.38685280723454163, "1089": 1.0, "1099": 0.5, "1100": 0.3562071871080222, "1104": 1.0, "1107": 0.6309297535714575, "1110": 0.2890648263178879, "1121": 1.0, "1130": 1.0, "1132": 0.21840743681816419, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.3562071871080222, "1185": 1.0, "1187": 1.0, "1191": 0.38685280723454163, "1194": 1.0, "1196": 0.43067655807339306, "1197": 0.43067655807339306, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 0.38685280723454163, "1225": 1.0, "1226": 0.3010299956639812, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 0.8278123145308894, "1278": 0.6309297535714575, "1279": 0.0, "1280": 0.31546487678572877, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.6309297535714575, "1320": 0.6309297535714575, "1332": 0.0, "1335": 0.6309297535714575, "1336": 0.6309297535714575, "1337": 1.0, "1339": 1.0, "1344": 0.6309297535714575, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 0.5, "1370": 1.0, "1379": 0.7095272044910244, "1382": 0.5, "1385": 1.0, "1389": 1.0, "1395": 0.5}, "Recall@20": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 1.0, "42": 1.0, "48": 1.0, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.5, "72": 1.0, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 1.0, "213": 1.0, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 1.0, "233": 1.0, "236": 1.0, "237": 1.0, "238": 1.0, "239": 1.0, "248": 1.0, "249": 1.0, "261": 1.0, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 1.0, "303": 0.0, "312": 0.0, "314": 1.0, "324": 0.0, "327": 1.0, "338": 1.0, "343": 1.0, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 1.0, "385": 1.0, "386": 1.0, "388": 1.0, "399": 1.0, "410": 1.0, "411": 1.0, "415": 1.0, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.5, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 1.0, "508": 1.0, "513": 1.0, "514": 1.0, "516": 1.0, "517": 1.0, "521": 1.0, "525": 1.0, "527": 1.0, "528": 1.0, "532": 1.0, "533": 1.0, "535": 1.0, "536": 1.0, "539": 1.0, "540": 1.0, "544": 1.0, "549": 1.0, "551": 1.0, "552": 1.0, "554": 1.0, "560": 0.0, "569": 1.0, "575": 1.0, "577": 1.0, "578": 1.0, "587": 1.0, "589": 1.0, "593": 1.0, "597": 1.0, "598": 1.0, "613": 1.0, "619": 0.0, "623": 1.0, "628": 1.0, "636": 1.0, "637": 1.0, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.0, "660": 0.0, "674": 1.0, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 1.0, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 1.0, "756": 1.0, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 1.0, "784": 1.0, "785": 1.0, "793": 1.0, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 1.0, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 1.0, "847": 1.0, "852": 1.0, "859": 1.0, "870": 1.0, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 1.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 1.0, "1012": 1.0, "1014": 1.0, "1019": 1.0, "1020": 1.0, "1021": 1.0, "1024": 1.0, "1029": 1.0, "1041": 1.0, "1049": 1.0, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 1.0, "1100": 1.0, "1104": 1.0, "1107": 1.0, "1110": 1.0, "1121": 1.0, "1130": 1.0, "1132": 1.0, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 1.0, "1185": 1.0, "1187": 1.0, "1191": 1.0, "1194": 1.0, "1196": 1.0, "1197": 1.0, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 1.0, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 1.0, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 1.0, "1279": 0.0, "1280": 1.0, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 1.0, "1320": 1.0, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 1.0, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 1.0, "1370": 1.0, "1379": 1.0, "1382": 1.0, "1385": 1.0, "1389": 1.0, "1395": 1.0}, "MRR@10": {"1": 0.0, "3": 0.5, "5": 1.0, "13": 0.0, "36": 0.25, "42": 1.0, "48": 0.5, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.2, "72": 0.5, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 0.5, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.3333333333333333, "213": 0.25, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.5, "233": 1.0, "236": 1.0, "237": 0.5, "238": 0.3333333333333333, "239": 0.25, "248": 1.0, "249": 1.0, "261": 0.25, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 0.5, "298": 1.0, "300": 0.25, "303": 0.0, "312": 0.0, "314": 0.3333333333333333, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.5, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.16666666666666666, "385": 0.5, "386": 1.0, "388": 0.5, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.3333333333333333, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3333333333333333, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.5, "508": 0.1111111111111111, "513": 0.5, "514": 1.0, "516": 1.0, "517": 0.14285714285714285, "521": 1.0, "525": 1.0, "527": 0.5, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.14285714285714285, "536": 1.0, "539": 1.0, "540": 0.5, "544": 0.2, "549": 1.0, "551": 0.2, "552": 1.0, "554": 0.1111111111111111, "560": 0.0, "569": 0.5, "575": 1.0, "577": 0.5, "578": 1.0, "587": 0.5, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.3333333333333333, "613": 1.0, "619": 0.0, "623": 0.08333333333333333, "628": 1.0, "636": 1.0, "637": 0.5, "641": 1.0, "644": 1.0, "649": 0.5, "659": 0.0, "660": 0.0, "674": 0.5, "684": 0.5, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.25, "728": 0.5, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.5, "756": 0.3333333333333333, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.14285714285714285, "784": 1.0, "785": 1.0, "793": 0.5, "800": 0.2, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.25, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.5, "847": 1.0, "852": 1.0, "859": 0.5, "870": 0.07142857142857142, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0625, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.5, "1012": 1.0, "1014": 1.0, "1019": 0.5, "1020": 0.5, "1021": 0.5, "1024": 0.3333333333333333, "1029": 1.0, "1041": 0.14285714285714285, "1049": 0.14285714285714285, "1062": 1.0, "1086": 1.0, "1088": 0.2, "1089": 1.0, "1099": 0.3333333333333333, "1100": 0.16666666666666666, "1104": 1.0, "1107": 0.5, "1110": 0.1, "1121": 1.0, "1130": 1.0, "1132": 0.16666666666666666, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.16666666666666666, "1185": 1.0, "1187": 1.0, "1191": 0.2, "1194": 1.0, "1196": 0.25, "1197": 0.25, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.07692307692307693, "1216": 1.0, "1221": 0.2, "1225": 1.0, "1226": 0.1111111111111111, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 0.5, "1279": 0.0, "1280": 0.125, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.5, "1320": 0.5, "1332": 0.0, "1335": 0.5, "1336": 0.5, "1337": 1.0, "1339": 1.0, "1344": 0.5, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 0.3333333333333333, "1370": 1.0, "1379": 1.0, "1382": 0.3333333333333333, "1385": 1.0, "1389": 1.0, "1395": 0.3333333333333333}}} diff --git a/evals/benchmark/results/scifact/ir-w0.75.jsonl b/evals/benchmark/results/scifact/ir-w0.75.jsonl new file mode 100644 index 000000000..da9663c25 --- /dev/null +++ b/evals/benchmark/results/scifact/ir-w0.75.jsonl @@ -0,0 +1 @@ +{"dataset": "scifact", "run_tag": "w0.75", "aggregated": {"nDCG@10": 0.7226, "Recall@20": 0.8633, "MRR@10": 0.693}, "per_query": {"nDCG@10": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 0.3065735963827292, "42": 1.0, "48": 0.6309297535714575, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.2640681225725909, "72": 0.6309297535714575, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 0.8712322899646365, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 0.6975637657739948, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.5, "213": 0.43067655807339306, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.6309297535714575, "233": 1.0, "236": 1.0, "237": 1.0, "238": 0.5, "239": 0.38685280723454163, "248": 1.0, "249": 1.0, "261": 0.46845052016107697, "268": 1.0, "269": 1.0, "274": 1.0, "275": 0.6364391809889587, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 0.5, "303": 0.0, "312": 0.0, "314": 0.5, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.6509209298071326, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.3562071871080222, "385": 0.6934264036172708, "386": 1.0, "388": 0.6309297535714575, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.5, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3065735963827292, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.6309297535714575, "508": 0.3010299956639812, "513": 0.6309297535714575, "514": 1.0, "516": 1.0, "517": 0.3333333333333333, "521": 1.0, "525": 1.0, "527": 0.6309297535714575, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.3333333333333333, "536": 1.0, "539": 1.0, "540": 0.38685280723454163, "544": 0.38685280723454163, "549": 1.0, "551": 0.38685280723454163, "552": 1.0, "554": 0.31546487678572877, "560": 0.0, "569": 0.6309297535714575, "575": 1.0, "577": 0.6309297535714575, "578": 1.0, "587": 0.6309297535714575, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.5, "613": 1.0, "619": 0.0, "623": 0.0, "628": 1.0, "636": 1.0, "637": 0.6309297535714575, "641": 0.8065735963827293, "644": 1.0, "649": 1.0, "659": 0.0, "660": 0.0, "674": 0.6309297535714575, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.38685280723454163, "728": 0.6934264036172708, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.6309297535714575, "756": 0.5, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.3333333333333333, "784": 1.0, "785": 1.0, "793": 0.6309297535714575, "800": 0.38685280723454163, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.43067655807339306, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.6309297535714575, "847": 1.0, "852": 1.0, "859": 0.6309297535714575, "870": 0.0, "873": 0.7870111744655099, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.6309297535714575, "1012": 1.0, "1014": 1.0, "1019": 0.6309297535714575, "1020": 0.6309297535714575, "1021": 0.6309297535714575, "1024": 0.5, "1029": 0.7653606369886217, "1041": 0.41183384043543503, "1049": 0.3333333333333333, "1062": 1.0, "1086": 1.0, "1088": 0.38685280723454163, "1089": 1.0, "1099": 0.5, "1100": 0.3562071871080222, "1104": 1.0, "1107": 0.6309297535714575, "1110": 0.2890648263178879, "1121": 1.0, "1130": 1.0, "1132": 0.21840743681816419, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.3562071871080222, "1185": 1.0, "1187": 1.0, "1191": 0.38685280723454163, "1194": 1.0, "1196": 0.43067655807339306, "1197": 0.43067655807339306, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 0.38685280723454163, "1225": 1.0, "1226": 0.3010299956639812, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 0.8278123145308894, "1278": 0.6309297535714575, "1279": 0.0, "1280": 0.31546487678572877, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.6309297535714575, "1320": 0.6309297535714575, "1332": 0.0, "1335": 0.6309297535714575, "1336": 0.6309297535714575, "1337": 1.0, "1339": 1.0, "1344": 0.6309297535714575, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 0.38685280723454163, "1370": 0.6309297535714575, "1379": 0.6886342695939197, "1382": 0.5, "1385": 1.0, "1389": 1.0, "1395": 0.5}, "Recall@20": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 1.0, "42": 1.0, "48": 1.0, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.5, "72": 1.0, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 1.0, "213": 1.0, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 1.0, "233": 1.0, "236": 1.0, "237": 1.0, "238": 1.0, "239": 1.0, "248": 1.0, "249": 1.0, "261": 1.0, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 1.0, "303": 0.0, "312": 0.0, "314": 1.0, "324": 0.0, "327": 1.0, "338": 1.0, "343": 1.0, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 1.0, "385": 1.0, "386": 1.0, "388": 1.0, "399": 1.0, "410": 1.0, "411": 1.0, "415": 1.0, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.5, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 1.0, "508": 1.0, "513": 1.0, "514": 1.0, "516": 1.0, "517": 1.0, "521": 1.0, "525": 1.0, "527": 1.0, "528": 1.0, "532": 1.0, "533": 1.0, "535": 1.0, "536": 1.0, "539": 1.0, "540": 1.0, "544": 1.0, "549": 1.0, "551": 1.0, "552": 1.0, "554": 1.0, "560": 0.0, "569": 1.0, "575": 1.0, "577": 1.0, "578": 1.0, "587": 1.0, "589": 1.0, "593": 1.0, "597": 1.0, "598": 1.0, "613": 1.0, "619": 0.0, "623": 1.0, "628": 1.0, "636": 1.0, "637": 1.0, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.0, "660": 0.0, "674": 1.0, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 1.0, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 1.0, "756": 1.0, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 1.0, "784": 1.0, "785": 1.0, "793": 1.0, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 1.0, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 1.0, "847": 1.0, "852": 1.0, "859": 1.0, "870": 1.0, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 1.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 1.0, "1012": 1.0, "1014": 1.0, "1019": 1.0, "1020": 1.0, "1021": 1.0, "1024": 1.0, "1029": 1.0, "1041": 1.0, "1049": 1.0, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 1.0, "1100": 1.0, "1104": 1.0, "1107": 1.0, "1110": 1.0, "1121": 1.0, "1130": 1.0, "1132": 1.0, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 1.0, "1185": 1.0, "1187": 1.0, "1191": 1.0, "1194": 1.0, "1196": 1.0, "1197": 1.0, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 1.0, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 1.0, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 1.0, "1279": 0.0, "1280": 1.0, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 1.0, "1320": 1.0, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 1.0, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 1.0, "1370": 1.0, "1379": 1.0, "1382": 1.0, "1385": 1.0, "1389": 1.0, "1395": 1.0}, "MRR@10": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 0.3333333333333333, "42": 1.0, "48": 0.5, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.25, "72": 0.5, "75": 1.0, "94": 0.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.0, "208": 1.0, "212": 0.3333333333333333, "213": 0.25, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.5, "233": 1.0, "236": 1.0, "237": 1.0, "238": 0.3333333333333333, "239": 0.2, "248": 1.0, "249": 1.0, "261": 0.25, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.0, "295": 1.0, "298": 1.0, "300": 0.3333333333333333, "303": 0.0, "312": 0.0, "314": 0.3333333333333333, "324": 0.0, "327": 1.0, "338": 1.0, "343": 0.5, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.16666666666666666, "385": 0.5, "386": 1.0, "388": 0.5, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.3333333333333333, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3333333333333333, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.5, "508": 0.1111111111111111, "513": 0.5, "514": 1.0, "516": 1.0, "517": 0.14285714285714285, "521": 1.0, "525": 1.0, "527": 0.5, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.14285714285714285, "536": 1.0, "539": 1.0, "540": 0.5, "544": 0.2, "549": 1.0, "551": 0.2, "552": 1.0, "554": 0.125, "560": 0.0, "569": 0.5, "575": 1.0, "577": 0.5, "578": 1.0, "587": 0.5, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.3333333333333333, "613": 1.0, "619": 0.0, "623": 0.08333333333333333, "628": 1.0, "636": 1.0, "637": 0.5, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.0, "660": 0.0, "674": 0.5, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.2, "728": 0.5, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.5, "756": 0.3333333333333333, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 0.14285714285714285, "784": 1.0, "785": 1.0, "793": 0.5, "800": 0.2, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.25, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.5, "847": 1.0, "852": 1.0, "859": 0.5, "870": 0.07142857142857142, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0625, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.5, "1012": 1.0, "1014": 1.0, "1019": 0.5, "1020": 0.5, "1021": 0.5, "1024": 0.3333333333333333, "1029": 1.0, "1041": 0.16666666666666666, "1049": 0.14285714285714285, "1062": 1.0, "1086": 1.0, "1088": 0.2, "1089": 1.0, "1099": 0.3333333333333333, "1100": 0.16666666666666666, "1104": 1.0, "1107": 0.5, "1110": 0.1, "1121": 1.0, "1130": 1.0, "1132": 0.16666666666666666, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.16666666666666666, "1185": 1.0, "1187": 1.0, "1191": 0.2, "1194": 1.0, "1196": 0.25, "1197": 0.25, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.07692307692307693, "1216": 1.0, "1221": 0.2, "1225": 1.0, "1226": 0.1111111111111111, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 0.5, "1279": 0.0, "1280": 0.125, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.5, "1320": 0.5, "1332": 0.0, "1335": 0.5, "1336": 0.5, "1337": 1.0, "1339": 1.0, "1344": 0.5, "1352": 1.0, "1359": 1.0, "1362": 0.0, "1363": 0.0, "1368": 0.2, "1370": 0.5, "1379": 1.0, "1382": 0.3333333333333333, "1385": 1.0, "1389": 1.0, "1395": 0.3333333333333333}}} diff --git a/evals/benchmark/results/scifact/ir-w1.0.jsonl b/evals/benchmark/results/scifact/ir-w1.0.jsonl new file mode 100644 index 000000000..419bb87c4 --- /dev/null +++ b/evals/benchmark/results/scifact/ir-w1.0.jsonl @@ -0,0 +1 @@ +{"dataset": "scifact", "run_tag": "w1.0", "aggregated": {"nDCG@10": 0.728, "Recall@20": 0.8893, "MRR@10": 0.6946}, "per_query": {"nDCG@10": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 0.2640681225725909, "42": 1.0, "48": 0.6309297535714575, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.2640681225725909, "72": 0.6309297535714575, "75": 1.0, "94": 0.6309297535714575, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.3333333333333333, "129": 1.0, "130": 1.0, "132": 0.0, "133": 0.6661998717660157, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 0.715696758318853, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.43067655807339306, "208": 1.0, "212": 0.6309297535714575, "213": 0.5, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.5, "233": 1.0, "236": 1.0, "237": 1.0, "238": 0.5, "239": 0.38685280723454163, "248": 1.0, "249": 1.0, "261": 0.46845052016107697, "268": 1.0, "269": 1.0, "274": 1.0, "275": 0.6364391809889587, "279": 1.0, "294": 0.3333333333333333, "295": 1.0, "298": 1.0, "300": 0.5, "303": 0.0, "312": 0.2890648263178879, "314": 0.5, "324": 0.3562071871080222, "327": 1.0, "338": 1.0, "343": 0.5706417189553201, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.2890648263178879, "385": 0.6934264036172708, "386": 1.0, "388": 0.6309297535714575, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.5, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3065735963827292, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.6309297535714575, "508": 0.0, "513": 0.6309297535714575, "514": 0.6309297535714575, "516": 1.0, "517": 0.3010299956639812, "521": 1.0, "525": 1.0, "527": 0.6309297535714575, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.3333333333333333, "536": 1.0, "539": 1.0, "540": 0.38685280723454163, "544": 0.38685280723454163, "549": 1.0, "551": 0.43067655807339306, "552": 1.0, "554": 0.31546487678572877, "560": 0.0, "569": 0.6309297535714575, "575": 1.0, "577": 0.6309297535714575, "578": 1.0, "587": 0.6309297535714575, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.38685280723454163, "613": 1.0, "619": 0.0, "623": 0.0, "628": 1.0, "636": 0.6309297535714575, "637": 0.6309297535714575, "641": 0.6131471927654584, "644": 1.0, "649": 1.0, "659": 0.6309297535714575, "660": 0.6309297535714575, "674": 0.6309297535714575, "684": 1.0, "690": 0.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.43067655807339306, "728": 0.5706417189553201, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.6309297535714575, "756": 0.5, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 0.6309297535714575, "783": 0.3333333333333333, "784": 1.0, "785": 1.0, "793": 0.6309297535714575, "800": 0.38685280723454163, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.5, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.6309297535714575, "847": 1.0, "852": 1.0, "859": 0.6309297535714575, "870": 0.0, "873": 0.6552438651310554, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.6309297535714575, "1012": 1.0, "1014": 1.0, "1019": 0.6309297535714575, "1020": 0.6309297535714575, "1021": 0.6309297535714575, "1024": 1.0, "1029": 0.7653606369886217, "1041": 0.430624116386567, "1049": 0.0, "1062": 1.0, "1086": 1.0, "1088": 0.38685280723454163, "1089": 1.0, "1099": 0.5, "1100": 0.38685280723454163, "1104": 1.0, "1107": 0.6309297535714575, "1110": 0.0, "1121": 1.0, "1130": 1.0, "1132": 0.23719771276929622, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.38685280723454163, "1185": 1.0, "1187": 1.0, "1191": 0.31546487678572877, "1194": 1.0, "1196": 0.43067655807339306, "1197": 0.43067655807339306, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 0.3562071871080222, "1225": 1.0, "1226": 0.0, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 0.6309297535714575, "1273": 1.0, "1274": 0.6871475159848289, "1278": 0.6309297535714575, "1279": 0.0, "1280": 0.3010299956639812, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.5, "1320": 0.6309297535714575, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 0.6309297535714575, "1352": 1.0, "1359": 1.0, "1362": 0.3562071871080222, "1363": 0.3562071871080222, "1368": 0.38685280723454163, "1370": 0.6309297535714575, "1379": 0.544556608342548, "1382": 0.5, "1385": 1.0, "1389": 1.0, "1395": 0.43067655807339306}, "Recall@20": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 0.5, "42": 1.0, "48": 1.0, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 1.0, "72": 1.0, "75": 1.0, "94": 1.0, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 1.0, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 1.0, "208": 1.0, "212": 1.0, "213": 1.0, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 1.0, "233": 1.0, "236": 1.0, "237": 1.0, "238": 1.0, "239": 1.0, "248": 1.0, "249": 1.0, "261": 1.0, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 1.0, "295": 1.0, "298": 1.0, "300": 1.0, "303": 0.0, "312": 1.0, "314": 1.0, "324": 1.0, "327": 1.0, "338": 1.0, "343": 1.0, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 1.0, "385": 1.0, "386": 1.0, "388": 1.0, "399": 1.0, "410": 1.0, "411": 1.0, "415": 1.0, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.5, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 1.0, "508": 1.0, "513": 1.0, "514": 1.0, "516": 1.0, "517": 1.0, "521": 1.0, "525": 1.0, "527": 1.0, "528": 1.0, "532": 1.0, "533": 1.0, "535": 1.0, "536": 1.0, "539": 1.0, "540": 0.5, "544": 1.0, "549": 1.0, "551": 1.0, "552": 1.0, "554": 1.0, "560": 0.0, "569": 1.0, "575": 1.0, "577": 1.0, "578": 1.0, "587": 1.0, "589": 1.0, "593": 1.0, "597": 1.0, "598": 1.0, "613": 1.0, "619": 0.5, "623": 1.0, "628": 1.0, "636": 1.0, "637": 1.0, "641": 1.0, "644": 1.0, "649": 1.0, "659": 1.0, "660": 1.0, "674": 1.0, "684": 1.0, "690": 1.0, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 1.0, "728": 1.0, "729": 1.0, "742": 1.0, "743": 1.0, "744": 1.0, "756": 1.0, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 1.0, "783": 1.0, "784": 1.0, "785": 1.0, "793": 1.0, "800": 1.0, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 1.0, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 1.0, "847": 1.0, "852": 1.0, "859": 1.0, "870": 0.0, "873": 0.8, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 1.0, "1012": 1.0, "1014": 1.0, "1019": 1.0, "1020": 1.0, "1021": 1.0, "1024": 1.0, "1029": 1.0, "1041": 1.0, "1049": 1.0, "1062": 1.0, "1086": 1.0, "1088": 1.0, "1089": 1.0, "1099": 1.0, "1100": 1.0, "1104": 1.0, "1107": 1.0, "1110": 0.0, "1121": 1.0, "1130": 1.0, "1132": 1.0, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 1.0, "1185": 1.0, "1187": 1.0, "1191": 1.0, "1194": 1.0, "1196": 1.0, "1197": 1.0, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 1.0, "1225": 1.0, "1226": 1.0, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 1.0, "1273": 1.0, "1274": 1.0, "1278": 1.0, "1279": 1.0, "1280": 1.0, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 1.0, "1320": 1.0, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 1.0, "1352": 1.0, "1359": 1.0, "1362": 1.0, "1363": 1.0, "1368": 1.0, "1370": 1.0, "1379": 1.0, "1382": 1.0, "1385": 1.0, "1389": 1.0, "1395": 1.0}, "MRR@10": {"1": 0.0, "3": 1.0, "5": 1.0, "13": 0.0, "36": 0.25, "42": 1.0, "48": 0.5, "49": 1.0, "50": 1.0, "51": 1.0, "53": 1.0, "54": 1.0, "56": 1.0, "57": 1.0, "70": 0.25, "72": 0.5, "75": 1.0, "94": 0.5, "99": 1.0, "100": 1.0, "113": 1.0, "115": 1.0, "118": 1.0, "124": 1.0, "127": 1.0, "128": 0.14285714285714285, "129": 1.0, "130": 1.0, "132": 0.0, "133": 1.0, "137": 1.0, "141": 1.0, "142": 1.0, "143": 1.0, "146": 1.0, "148": 1.0, "163": 1.0, "171": 1.0, "179": 1.0, "180": 1.0, "183": 1.0, "185": 1.0, "198": 0.25, "208": 1.0, "212": 0.5, "213": 0.3333333333333333, "216": 1.0, "217": 1.0, "218": 1.0, "219": 1.0, "230": 1.0, "232": 0.3333333333333333, "233": 1.0, "236": 1.0, "237": 1.0, "238": 0.3333333333333333, "239": 0.2, "248": 1.0, "249": 1.0, "261": 0.25, "268": 1.0, "269": 1.0, "274": 1.0, "275": 1.0, "279": 1.0, "294": 0.14285714285714285, "295": 1.0, "298": 1.0, "300": 0.3333333333333333, "303": 0.0, "312": 0.1, "314": 0.3333333333333333, "324": 0.16666666666666666, "327": 1.0, "338": 1.0, "343": 0.3333333333333333, "350": 1.0, "354": 1.0, "362": 1.0, "380": 1.0, "384": 0.1, "385": 0.5, "386": 1.0, "388": 0.5, "399": 1.0, "410": 1.0, "411": 1.0, "415": 0.3333333333333333, "421": 0.0, "431": 0.0, "436": 1.0, "437": 0.0, "439": 1.0, "440": 1.0, "443": 1.0, "452": 0.3333333333333333, "475": 0.0, "478": 1.0, "491": 1.0, "501": 1.0, "502": 0.0, "507": 0.5, "508": 0.07692307692307693, "513": 0.5, "514": 0.5, "516": 1.0, "517": 0.1111111111111111, "521": 1.0, "525": 1.0, "527": 0.5, "528": 1.0, "532": 1.0, "533": 1.0, "535": 0.14285714285714285, "536": 1.0, "539": 1.0, "540": 0.5, "544": 0.2, "549": 1.0, "551": 0.25, "552": 1.0, "554": 0.125, "560": 0.0, "569": 0.5, "575": 1.0, "577": 0.5, "578": 1.0, "587": 0.5, "589": 1.0, "593": 1.0, "597": 1.0, "598": 0.2, "613": 1.0, "619": 0.07692307692307693, "623": 0.08333333333333333, "628": 1.0, "636": 0.5, "637": 0.5, "641": 1.0, "644": 1.0, "649": 1.0, "659": 0.5, "660": 0.5, "674": 0.5, "684": 1.0, "690": 0.06666666666666667, "691": 1.0, "692": 1.0, "693": 1.0, "700": 1.0, "702": 1.0, "715": 0.0, "716": 0.0, "718": 1.0, "721": 1.0, "723": 1.0, "727": 0.25, "728": 0.3333333333333333, "729": 1.0, "742": 1.0, "743": 1.0, "744": 0.5, "756": 0.3333333333333333, "759": 1.0, "768": 0.0, "770": 1.0, "775": 0.0, "781": 0.5, "783": 0.14285714285714285, "784": 1.0, "785": 1.0, "793": 0.5, "800": 0.2, "805": 1.0, "808": 1.0, "811": 1.0, "814": 1.0, "820": 0.0, "821": 0.0, "823": 1.0, "830": 0.3333333333333333, "831": 0.0, "832": 1.0, "834": 0.0, "837": 1.0, "839": 1.0, "845": 0.5, "847": 1.0, "852": 1.0, "859": 0.5, "870": 0.0, "873": 1.0, "879": 1.0, "880": 1.0, "882": 1.0, "887": 0.0, "903": 1.0, "904": 1.0, "907": 1.0, "911": 1.0, "913": 0.0, "914": 0.0, "921": 1.0, "922": 1.0, "936": 1.0, "956": 1.0, "957": 1.0, "960": 1.0, "967": 1.0, "971": 1.0, "975": 0.0, "982": 1.0, "985": 1.0, "993": 0.5, "1012": 1.0, "1014": 1.0, "1019": 0.5, "1020": 0.5, "1021": 0.5, "1024": 1.0, "1029": 1.0, "1041": 0.2, "1049": 0.08333333333333333, "1062": 1.0, "1086": 1.0, "1088": 0.2, "1089": 1.0, "1099": 0.3333333333333333, "1100": 0.2, "1104": 1.0, "1107": 0.5, "1110": 0.0, "1121": 1.0, "1130": 1.0, "1132": 0.2, "1137": 1.0, "1140": 1.0, "1144": 1.0, "1146": 1.0, "1150": 1.0, "1163": 1.0, "1175": 0.0, "1179": 0.0, "1180": 0.2, "1185": 1.0, "1187": 1.0, "1191": 0.125, "1194": 1.0, "1196": 0.25, "1197": 0.25, "1199": 0.0, "1200": 1.0, "1202": 1.0, "1204": 1.0, "1207": 1.0, "1213": 0.0, "1216": 1.0, "1221": 0.16666666666666666, "1225": 1.0, "1226": 0.07692307692307693, "1232": 1.0, "1241": 1.0, "1245": 1.0, "1259": 1.0, "1262": 1.0, "1266": 1.0, "1270": 1.0, "1271": 1.0, "1272": 0.5, "1273": 1.0, "1274": 0.5, "1278": 0.5, "1279": 0.08333333333333333, "1280": 0.1111111111111111, "1281": 0.0, "1282": 1.0, "1290": 1.0, "1292": 1.0, "1298": 1.0, "1303": 1.0, "1316": 0.0, "1319": 0.3333333333333333, "1320": 0.5, "1332": 0.0, "1335": 1.0, "1336": 1.0, "1337": 1.0, "1339": 1.0, "1344": 0.5, "1352": 1.0, "1359": 1.0, "1362": 0.16666666666666666, "1363": 0.16666666666666666, "1368": 0.2, "1370": 0.5, "1379": 0.5, "1382": 0.3333333333333333, "1385": 1.0, "1389": 1.0, "1395": 0.25}}} diff --git a/evals/benchmark/results/scifact/qrels.json b/evals/benchmark/results/scifact/qrels.json new file mode 100644 index 000000000..659a594b2 --- /dev/null +++ b/evals/benchmark/results/scifact/qrels.json @@ -0,0 +1 @@ +{"1": {"31715818": 1}, "3": {"14717500": 1}, "5": {"13734012": 1}, "13": {"1606628": 1}, "36": {"5152028": 1, "11705328": 1}, "42": {"18174210": 1}, "48": {"13734012": 1}, "49": {"5953485": 1}, "50": {"12580014": 1}, "51": {"45638119": 1}, "53": {"45638119": 1}, "54": {"49556906": 1}, "56": {"4709641": 1}, "57": {"4709641": 1}, "70": {"5956380": 1, "4414547": 1}, "72": {"6076903": 1}, "75": {"4387784": 1}, "94": {"1215116": 1}, "99": {"18810195": 1}, "100": {"4381486": 1}, "113": {"6157837": 1}, "115": {"33872649": 1}, "118": {"6372244": 1}, "124": {"4883040": 1}, "127": {"21598000": 1}, "128": {"8290953": 1}, "129": {"27768226": 1}, "130": {"27768226": 1}, "132": {"7975937": 1}, "133": {"38485364": 1, "6969753": 1, "17934082": 1, "16280642": 1, "12640810": 1}, "137": {"26016929": 1}, "141": {"6955746": 1, "14437255": 1}, "142": {"10582939": 1}, "143": {"10582939": 1}, "146": {"10582939": 1}, "148": {"1084345": 1}, "163": {"18872233": 1}, "171": {"12670680": 1}, "179": {"16322674": 1, "27123743": 1, "23557241": 1, "17450673": 1}, "180": {"16966326": 1}, "183": {"12827098": 1}, "185": {"18340282": 1}, "198": {"2177022": 1}, "208": {"13519661": 1}, "212": {"22038539": 1}, "213": {"13625993": 1}, "216": {"21366394": 1}, "217": {"21366394": 1}, "218": {"21366394": 1}, "219": {"21366394": 1}, "230": {"3067015": 1}, "232": {"10536636": 1}, "233": {"4388470": 1}, "236": {"4388470": 1}, "237": {"4942718": 1}, "238": {"2251426": 1}, "239": {"14079881": 1}, "248": {"1568684": 1}, "249": {"1568684": 1}, "261": {"1122279": 1, "10697096": 1}, "268": {"970012": 1}, "269": {"970012": 1}, "274": {"11614737": 1}, "275": {"4961038": 1, "14241418": 1, "14819804": 1}, "279": {"14376683": 1}, "294": {"10874408": 1}, "295": {"20310709": 1}, "298": {"39381118": 1}, "300": {"3553087": 1}, "303": {"4388470": 1}, "312": {"6173523": 1}, "314": {"4347374": 1}, "324": {"2014909": 1}, "327": {"17997584": 1}, "338": {"23349986": 1}, "343": {"7873737": 1, "5884524": 1}, "350": {"16927286": 1}, "354": {"8774475": 1}, "362": {"38587347": 1}, "380": {"19005293": 1}, "384": {"13770184": 1}, "385": {"9955779": 1, "9767444": 1}, "386": {"16495649": 1}, "388": {"1148122": 1}, "399": {"791050": 1}, "410": {"14924526": 1}, "411": {"14924526": 1}, "415": {"6309659": 1}, "421": {"11172205": 1}, "431": {"28937856": 1}, "436": {"14637235": 1}, "437": {"18399038": 1}, "439": {"4423559": 1}, "440": {"4423559": 1}, "443": {"10165258": 1}, "452": {"12804937": 1, "464511": 1}, "475": {"18678095": 1}, "478": {"14767844": 1}, "491": {"56893404": 1}, "501": {"17930286": 1}, "502": {"13071728": 1}, "507": {"30774694": 1}, "508": {"13980338": 1}, "513": {"13230773": 1}, "514": {"16256507": 1}, "516": {"29564505": 1}, "517": {"15663829": 1}, "521": {"34873974": 1}, "525": {"13639330": 1}, "527": {"3863543": 1}, "528": {"5476778": 1}, "532": {"12991445": 1}, "533": {"12991445": 1}, "535": {"39368721": 1}, "536": {"16056514": 1}, "539": {"13282296": 1}, "540": {"11886686": 1, "25007443": 1}, "544": {"24221369": 1}, "549": {"9433958": 1}, "551": {"33499189": 1}, "552": {"1471041": 1}, "554": {"1049501": 1}, "560": {"40096222": 1}, "569": {"23460562": 1}, "575": {"10300888": 1}, "577": {"5289038": 1}, "578": {"8764879": 1}, "587": {"16999023": 1}, "589": {"10984005": 1}, "593": {"19675911": 1}, "597": {"12779444": 1, "36355784": 1, "25742130": 1}, "598": {"25742130": 1}, "613": {"9638032": 1}, "619": {"20888849": 1, "2565138": 1}, "623": {"17000834": 1}, "628": {"24512064": 1}, "636": {"24294572": 1}, "637": {"25649714": 1}, "641": {"5912283": 1, "31554917": 1}, "644": {"13619127": 1}, "649": {"12789595": 1}, "659": {"1215116": 1}, "660": {"1215116": 1}, "674": {"2095573": 1}, "684": {"4942718": 1}, "690": {"18750453": 1}, "691": {"10991183": 1}, "692": {"24088502": 1}, "693": {"24088502": 1}, "700": {"4350400": 1}, "702": {"4350400": 1}, "715": {"18421962": 1}, "716": {"18421962": 1}, "718": {"17587795": 1}, "721": {"1834762": 1}, "723": {"5531479": 1}, "727": {"7521113": 1}, "728": {"7521113": 1, "36444198": 1}, "729": {"26851674": 1}, "742": {"32159283": 1}, "743": {"32159283": 1}, "744": {"8460275": 1}, "756": {"2831620": 1}, "759": {"1805641": 1}, "768": {"6421792": 1}, "770": {"15476777": 1}, "775": {"32275758": 1}, "781": {"24338780": 1}, "783": {"40632104": 1}, "784": {"2356950": 1}, "785": {"12471115": 1}, "793": {"8551160": 1}, "800": {"22543403": 1}, "805": {"22180793": 1}, "808": {"36606083": 1}, "811": {"19799455": 1}, "814": {"33387953": 1}, "820": {"8646760": 1}, "821": {"8646760": 1}, "823": {"15319019": 1}, "830": {"1897324": 1}, "831": {"1897324": 1}, "832": {"30303335": 1}, "834": {"5483793": 1}, "837": {"15928989": 1}, "839": {"1469751": 1}, "845": {"17741440": 1}, "847": {"16787954": 1}, "852": {"13843341": 1}, "859": {"1982286": 1}, "870": {"195689316": 1}, "873": {"1180972": 1, "19307912": 1, "27393799": 1, "29025270": 1, "3315558": 1}, "879": {"8426046": 1}, "880": {"8426046": 1}, "882": {"14803797": 1}, "887": {"18855191": 1}, "903": {"10648422": 1}, "904": {"7370282": 1}, "907": {"6923961": 1}, "911": {"11254556": 1}, "913": {"3203590": 1}, "914": {"3203590": 1}, "921": {"1642727": 1}, "922": {"17077004": 1}, "936": {"5483793": 1}, "956": {"12956194": 1}, "957": {"123859": 1}, "960": {"8780599": 1}, "967": {"2119889": 1, "8997410": 1}, "971": {"46695481": 1, "27873158": 1, "28617573": 1, "9764256": 1}, "975": {"5304891": 1}, "982": {"2988714": 1}, "985": {"6828370": 1}, "993": {"16472469": 1}, "1012": {"9745001": 1}, "1014": {"6277638": 1}, "1019": {"11603066": 1}, "1020": {"9433958": 1}, "1021": {"9433958": 1}, "1024": {"5373138": 1}, "1029": {"13923140": 1, "13940200": 1, "11899391": 1}, "1041": {"25254425": 1, "16626264": 1}, "1049": {"12486491": 1}, "1062": {"20381484": 1}, "1086": {"39281140": 1}, "1088": {"37549932": 1}, "1089": {"17628888": 1}, "1099": {"7662206": 1}, "1100": {"7662206": 1}, "1104": {"3898784": 1}, "1107": {"20532591": 1}, "1110": {"13770184": 1}, "1121": {"4456756": 1}, "1130": {"17997584": 1}, "1132": {"33499189": 1, "9283422": 1}, "1137": {"33370": 1}, "1140": {"12009265": 1}, "1144": {"10071552": 1}, "1146": {"13906581": 1}, "1150": {"11369420": 1}, "1163": {"15305881": 1}, "1175": {"31272411": 1}, "1179": {"31272411": 1}, "1180": {"31272411": 1}, "1185": {"16737210": 1}, "1187": {"52873726": 1}, "1191": {"30655442": 1}, "1194": {"11419230": 1}, "1196": {"25649714": 1}, "1197": {"25649714": 1}, "1199": {"16760369": 1}, "1200": {"3441524": 1}, "1202": {"3475317": 1}, "1204": {"31141365": 1}, "1207": {"18909530": 1}, "1213": {"14407673": 1}, "1216": {"24142891": 1}, "1221": {"19736671": 1}, "1225": {"9650982": 1}, "1226": {"13777138": 1}, "1232": {"13905670": 1}, "1241": {"4427392": 1}, "1245": {"7662395": 1}, "1259": {"24341590": 1}, "1262": {"44172171": 1}, "1266": {"37480103": 1}, "1270": {"13900610": 1}, "1271": {"13768432": 1}, "1272": {"17081238": 1}, "1273": {"11041152": 1}, "1274": {"12428814": 1, "27731651": 1, "4406819": 1}, "1278": {"11335781": 1}, "1279": {"11335781": 1}, "1280": {"4387784": 1}, "1281": {"4387784": 1}, "1282": {"23649163": 1}, "1290": {"4687948": 1}, "1292": {"56893404": 1}, "1298": {"11718220": 1}, "1303": {"12631697": 1}, "1316": {"27910499": 1}, "1319": {"16284655": 1}, "1320": {"16284655": 1}, "1332": {"5304891": 1}, "1335": {"27910499": 1}, "1336": {"27910499": 1}, "1337": {"20231138": 1}, "1339": {"15482274": 1}, "1344": {"9559146": 1}, "1352": {"12885341": 1}, "1359": {"11614737": 1}, "1362": {"8290953": 1}, "1363": {"8290953": 1}, "1368": {"2425364": 1}, "1370": {"2425364": 1}, "1379": {"16322674": 1, "27123743": 1, "23557241": 1, "17450673": 1}, "1382": {"17755060": 1}, "1385": {"306006": 1}, "1389": {"23895668": 1}, "1395": {"17717391": 1}} \ No newline at end of file diff --git a/evals/benchmark/results/scifact/run-dense.trec.gz b/evals/benchmark/results/scifact/run-dense.trec.gz new file mode 100644 index 000000000..3b1c5d2c1 Binary files /dev/null and b/evals/benchmark/results/scifact/run-dense.trec.gz differ diff --git a/evals/benchmark/results/scifact/run-w0.25.trec.gz b/evals/benchmark/results/scifact/run-w0.25.trec.gz new file mode 100644 index 000000000..8507baa8b Binary files /dev/null and b/evals/benchmark/results/scifact/run-w0.25.trec.gz differ diff --git a/evals/benchmark/results/scifact/run-w0.5.trec.gz b/evals/benchmark/results/scifact/run-w0.5.trec.gz new file mode 100644 index 000000000..a8a911677 Binary files /dev/null and b/evals/benchmark/results/scifact/run-w0.5.trec.gz differ diff --git a/evals/benchmark/results/scifact/run-w0.75.trec.gz b/evals/benchmark/results/scifact/run-w0.75.trec.gz new file mode 100644 index 000000000..58709ab4f Binary files /dev/null and b/evals/benchmark/results/scifact/run-w0.75.trec.gz differ diff --git a/evals/benchmark/results/scifact/run-w1.0.trec.gz b/evals/benchmark/results/scifact/run-w1.0.trec.gz new file mode 100644 index 000000000..54115b8d6 Binary files /dev/null and b/evals/benchmark/results/scifact/run-w1.0.trec.gz differ diff --git a/evals/benchmark/results/scifact/stats-w0.25-vs-dense.jsonl b/evals/benchmark/results/scifact/stats-w0.25-vs-dense.jsonl new file mode 100644 index 000000000..c39d72137 --- /dev/null +++ b/evals/benchmark/results/scifact/stats-w0.25-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w0.25", "arm_b": "dense"} +{"row_type": "ir", "dataset": "scifact", "metric": "nDCG@10", "n": 300, "mean_a": 0.715541266680292, "mean_b": 0.6981162821923903, "mean_diff": -0.01742498448790186, "ci_low": -0.03424797587259849, "ci_high": -0.0011774831418385093, "p_value": 0.04229577042295771, "significant": true} +{"row_type": "ir", "dataset": "scifact", "metric": "Recall@20", "n": 300, "mean_a": 0.8633333333333333, "mean_b": 0.8633333333333333, "mean_diff": 0.0, "ci_low": 0.0, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "scifact", "metric": "MRR@10", "n": 300, "mean_a": 0.6837782356532356, "mean_b": 0.6647890581640581, "mean_diff": -0.01898917748917749, "ci_low": -0.03896908184408184, "ci_high": 0.00012348392348392277, "p_value": 0.059994000599940006, "significant": false} diff --git a/evals/benchmark/results/scifact/stats-w0.5-vs-dense.jsonl b/evals/benchmark/results/scifact/stats-w0.5-vs-dense.jsonl new file mode 100644 index 000000000..084423b49 --- /dev/null +++ b/evals/benchmark/results/scifact/stats-w0.5-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w0.5", "arm_b": "dense"} +{"row_type": "ir", "dataset": "scifact", "metric": "nDCG@10", "n": 300, "mean_a": 0.7167036895886415, "mean_b": 0.6981162821923903, "mean_diff": -0.018587407396251216, "ci_low": -0.03742859962988448, "ci_high": -0.0011315378911106106, "p_value": 0.045795420457954206, "significant": true} +{"row_type": "ir", "dataset": "scifact", "metric": "Recall@20", "n": 300, "mean_a": 0.8633333333333333, "mean_b": 0.8633333333333333, "mean_diff": 0.0, "ci_low": 0.0, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "scifact", "metric": "MRR@10", "n": 300, "mean_a": 0.6846115689865689, "mean_b": 0.6647890581640581, "mean_diff": -0.01982251082251082, "ci_low": -0.04220122747622748, "ci_high": 0.0014833492895992817, "p_value": 0.0751924807519248, "significant": false} diff --git a/evals/benchmark/results/scifact/stats-w0.75-vs-dense.jsonl b/evals/benchmark/results/scifact/stats-w0.75-vs-dense.jsonl new file mode 100644 index 000000000..ce33c2ca6 --- /dev/null +++ b/evals/benchmark/results/scifact/stats-w0.75-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w0.75", "arm_b": "dense"} +{"row_type": "ir", "dataset": "scifact", "metric": "nDCG@10", "n": 300, "mean_a": 0.7225852831633272, "mean_b": 0.6981162821923903, "mean_diff": -0.024469000970936988, "ci_low": -0.04406164668163103, "ci_high": -0.005730083692031733, "p_value": 0.012798720127987202, "significant": true} +{"row_type": "ir", "dataset": "scifact", "metric": "Recall@20", "n": 300, "mean_a": 0.8633333333333333, "mean_b": 0.8633333333333333, "mean_diff": 0.0, "ci_low": 0.0, "ci_high": 0.0, "p_value": 1.0, "significant": false} +{"row_type": "ir", "dataset": "scifact", "metric": "MRR@10", "n": 300, "mean_a": 0.6930150081400082, "mean_b": 0.6647890581640581, "mean_diff": -0.028225949975949974, "ci_low": -0.05189266381766382, "ci_high": -0.005090256502756504, "p_value": 0.018698130186981302, "significant": true} diff --git a/evals/benchmark/results/scifact/stats-w1.0-vs-dense.jsonl b/evals/benchmark/results/scifact/stats-w1.0-vs-dense.jsonl new file mode 100644 index 000000000..54f43a1eb --- /dev/null +++ b/evals/benchmark/results/scifact/stats-w1.0-vs-dense.jsonl @@ -0,0 +1,4 @@ +{"row_type": "meta", "run_id": "parity-2026-07", "fingerprint": "8abf1d39d4a2bfb065f6b7fedf9a7b7f1bf22d4a5f7ebb3e4f665c1899c131d7", "arm_a": "w1.0", "arm_b": "dense"} +{"row_type": "ir", "dataset": "scifact", "metric": "nDCG@10", "n": 300, "mean_a": 0.7280475062326587, "mean_b": 0.6981162821923903, "mean_diff": -0.029931224040268413, "ci_low": -0.052810587918646644, "ci_high": -0.007280555960053212, "p_value": 0.0112988701129887, "significant": true} +{"row_type": "ir", "dataset": "scifact", "metric": "Recall@20", "n": 300, "mean_a": 0.8893333333333334, "mean_b": 0.8633333333333333, "mean_diff": -0.026, "ci_low": -0.053, "ci_high": 0.0006666666666666665, "p_value": 0.0704929507049295, "significant": false} +{"row_type": "ir", "dataset": "scifact", "metric": "MRR@10", "n": 300, "mean_a": 0.6945814000814001, "mean_b": 0.6647890581640581, "mean_diff": -0.029792341917341918, "ci_low": -0.05662218163780664, "ci_high": -0.004029409595034603, "p_value": 0.027297270272972702, "significant": true} diff --git a/evals/benchmark/runfile.py b/evals/benchmark/runfile.py new file mode 100644 index 000000000..6e42a67de --- /dev/null +++ b/evals/benchmark/runfile.py @@ -0,0 +1,93 @@ +"""TREC run files and the chunk-to-document collapse both arms share. + +A run file is the standard six-column TREC format:: + + query_id Q0 doc_id rank score run_tag + +Retrieval returns chunks; scoring happens at the document level, so chunks are +collapsed to their parent document (best score wins) and re-ranked before the +run file is written. Everything here is pure so it can be fully unit-tested. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +TREC_UNUSED_COLUMN = "Q0" + + +@dataclass(frozen=True) +class ChunkHit: + """One retrieved chunk, tagged with the document it belongs to.""" + + query_id: str + doc_id: str + score: float + + +@dataclass(frozen=True) +class RunEntry: + """One ranked document for one query, as written to a TREC run file.""" + + query_id: str + doc_id: str + rank: int + score: float + run_tag: str + + def to_line(self) -> str: + return ( + f"{self.query_id} {TREC_UNUSED_COLUMN} {self.doc_id} " + f"{self.rank} {self.score:.6f} {self.run_tag}" + ) + + @classmethod + def from_line(cls, line: str) -> RunEntry: + query_id, _unused, doc_id, rank, score, run_tag = line.split() + return cls( + query_id=query_id, + doc_id=doc_id, + rank=int(rank), + score=float(score), + run_tag=run_tag, + ) + + +def collapse_hits(hits: list[ChunkHit], run_tag: str) -> list[RunEntry]: + """Collapse chunk hits to documents (best score wins) and re-rank per query. + + Multiple chunks from one document keep only that document's best score. + Within each query, documents are ranked by descending score; ties break on + doc_id so the ordering is deterministic and reproducible. + """ + best: dict[str, dict[str, float]] = {} + for hit in hits: + per_query = best.setdefault(hit.query_id, {}) + if hit.doc_id not in per_query or hit.score > per_query[hit.doc_id]: + per_query[hit.doc_id] = hit.score + entries: list[RunEntry] = [] + for query_id in sorted(best): + ranked = sorted(best[query_id].items(), key=lambda item: (-item[1], item[0])) + for rank, (doc_id, score) in enumerate(ranked, start=1): + entries.append(RunEntry(query_id, doc_id, rank, score, run_tag)) + return entries + + +def write_run(path: Path, entries: list[RunEntry]) -> None: + """Serialize run entries to a TREC run file.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(entry.to_line() + "\n" for entry in entries)) + + +def read_run(path: Path) -> list[RunEntry]: + """Parse a TREC run file; blank lines are skipped.""" + return [RunEntry.from_line(line) for line in path.read_text().splitlines() if line.strip()] + + +def run_to_pytrec(entries: list[RunEntry]) -> dict[str, dict[str, float]]: + """Shape run entries as pytrec_eval expects: query_id -> doc_id -> score.""" + run: dict[str, dict[str, float]] = {} + for entry in entries: + run.setdefault(entry.query_id, {})[entry.doc_id] = entry.score + return run diff --git a/evals/benchmark/stats.py b/evals/benchmark/stats.py new file mode 100644 index 000000000..2af1d2164 --- /dev/null +++ b/evals/benchmark/stats.py @@ -0,0 +1,120 @@ +"""Paired statistics on per-query metric vectors. + +Two arms are scored on the same queries, so every metric difference is paired. +A paired bootstrap gives a 95% confidence interval for the mean difference and +a sign-flip randomization test gives a p-value. Both are seeded and pure, so a +result is bit-for-bit reproducible. A difference whose CI crosses zero is +flagged not significant. +""" + +from __future__ import annotations + +import random +import statistics +from dataclasses import asdict, dataclass +from typing import Any + +DEFAULT_RESAMPLES = 10000 +DEFAULT_SEED = 20260714 +DEFAULT_ALPHA = 0.05 +# Probability of flipping a paired difference's sign in the randomization test. +SIGN_FLIP_PROBABILITY = 0.5 + + +@dataclass(frozen=True) +class PairedResult: + """The paired comparison of arm B minus arm A on one metric.""" + + metric: str + n: int + mean_a: float + mean_b: float + mean_diff: float + ci_low: float + ci_high: float + p_value: float + significant: bool + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _percentile(sorted_values: list[float], fraction: float) -> float: + """Linear-interpolated percentile of an already-sorted list.""" + if not sorted_values: + return 0.0 + if len(sorted_values) == 1: + return sorted_values[0] + position = fraction * (len(sorted_values) - 1) + lower = int(position) + upper = min(lower + 1, len(sorted_values) - 1) + weight = position - lower + return sorted_values[lower] * (1 - weight) + sorted_values[upper] * weight + + +def paired_bootstrap_ci( + diffs: list[float], resamples: int, seed: int, alpha: float +) -> tuple[float, float]: + """Percentile CI of the mean paired difference from a seeded bootstrap.""" + if not diffs: + return 0.0, 0.0 + rng = random.Random(seed) + count = len(diffs) + means: list[float] = [] + for _ in range(resamples): + resample = [diffs[rng.randrange(count)] for _ in range(count)] + means.append(statistics.fmean(resample)) + means.sort() + return _percentile(means, alpha / 2), _percentile(means, 1 - alpha / 2) + + +def permutation_test(diffs: list[float], resamples: int, seed: int) -> float: + """Two-sided sign-flip randomization p-value for a nonzero mean difference.""" + if not diffs: + return 1.0 + rng = random.Random(seed) + observed = abs(statistics.fmean(diffs)) + at_least_as_extreme = 0 + for _ in range(resamples): + flipped = [d if rng.random() < SIGN_FLIP_PROBABILITY else -d for d in diffs] + if abs(statistics.fmean(flipped)) >= observed: + at_least_as_extreme += 1 + return (at_least_as_extreme + 1) / (resamples + 1) + + +def _paired_diffs( + a_scores: dict[str, float], b_scores: dict[str, float] +) -> tuple[list[float], list[float], list[float]]: + """Align two per-query score maps on shared query ids, sorted for determinism.""" + shared = sorted(qid for qid in a_scores if qid in b_scores) + a_vec = [a_scores[qid] for qid in shared] + b_vec = [b_scores[qid] for qid in shared] + diffs = [b - a for a, b in zip(a_vec, b_vec, strict=True)] + return a_vec, b_vec, diffs + + +def compare( + metric: str, + a_scores: dict[str, float], + b_scores: dict[str, float], + *, + resamples: int = DEFAULT_RESAMPLES, + seed: int = DEFAULT_SEED, + alpha: float = DEFAULT_ALPHA, +) -> PairedResult: + """Full paired comparison of arm B minus arm A on one metric's per-query scores.""" + a_vec, b_vec, diffs = _paired_diffs(a_scores, b_scores) + ci_low, ci_high = paired_bootstrap_ci(diffs, resamples, seed, alpha) + p_value = permutation_test(diffs, resamples, seed) + mean_diff = statistics.fmean(diffs) if diffs else 0.0 + return PairedResult( + metric=metric, + n=len(diffs), + mean_a=statistics.fmean(a_vec) if a_vec else 0.0, + mean_b=statistics.fmean(b_vec) if b_vec else 0.0, + mean_diff=mean_diff, + ci_low=ci_low, + ci_high=ci_high, + p_value=p_value, + significant=not (ci_low <= 0.0 <= ci_high), + ) diff --git a/evals/retrieval/README.md b/evals/retrieval/README.md new file mode 100644 index 000000000..0e0159a25 --- /dev/null +++ b/evals/retrieval/README.md @@ -0,0 +1,107 @@ +# Retrieval eval harness + +Blind, noise-calibrated A/B evaluation of lilbee's end-to-end retrieval +quality. Two lilbee servers (arm A and arm B: old vs new version, or two +configs) answer the same question battery over the SAME index; a judge model +grades answers blind; exact-truth questions are checked mechanically against +the store. Lives outside `src/` on purpose: it never ships in the package. + +## How it works + +1. **questions** generates the battery from an existing indexed library: + - *topical*: the configured chat model writes one question from each + sampled stored passage, so the passage that must support the answer is + known ground truth. Passages are reservoir-sampled from a streaming scan + (one per source); nothing materializes the index, so it scales to very + large libraries. + - *known_item*: asks what a sampled document is about; ground truth is the + document's head chunks, captured in the same scan. + - *count*: asks how many chunks and documents mention a mid-frequency + term. Ground truth is an exact streaming scan of the LanceDB store; no + judge involved. +2. **answer** runs the battery against one server (`/api/ask`), one run per + arm. It waits for the server's health route, retries each question three + times, and checkpoints every row to JSONL, so a killed pod run resumes + without redoing completed questions. +3. **judge** shuffles every gradable answer under an opaque id and grades + them one at a time: the judge sees only question + ground truth + one + answer, never arm labels, and never knows a comparison is happening. Arm + B's answers are judged twice under different ids; the disagreement between + those two passes is the judge's noise floor. Grades are checkpointed too. +4. **score** unblinds mechanically: per-dimension means (faithfulness, + relevance, citation, each 0-2) with the noise floor as the error bar, plus + exact pass/fail for count and known-item questions. Writes machine-readable + `results.jsonl`. +5. **report** renders `results.jsonl` as markdown; any cross-arm delta at or + below the noise floor is labeled within noise. + +## Running on a pod + +Typical detached run against a large private corpus: one shared index, two +installed lilbee versions serving on different ports. + +```bash +# One venv per arm, both pointed at the same data root. +OLD_PORT=8081 NEW_PORT=8082 +"$OLD_VENV/bin/lilbee" --data-dir "$DATA_ROOT" serve --port "$OLD_PORT" & +"$NEW_VENV/bin/lilbee" --data-dir "$DATA_ROOT" serve --port "$NEW_PORT" & + +cd ~/lilbee # the repo checkout; run everything from the repo root + +# 1. Questions (uses the configured chat model; scans the index directly). +uv run python -m evals.retrieval questions \ + --data-root "$DATA_ROOT" --out /tmp/eval/questions.jsonl + +# 2. Answers, one run per arm. Interrupted runs resume from the checkpoint. +uv run python -m evals.retrieval answer \ + --questions /tmp/eval/questions.jsonl --base-url "http://127.0.0.1:$OLD_PORT" \ + --arm old --out /tmp/eval/answers-old.jsonl +uv run python -m evals.retrieval answer \ + --questions /tmp/eval/questions.jsonl --base-url "http://127.0.0.1:$NEW_PORT" \ + --arm new --out /tmp/eval/answers-new.jsonl + +# 3. Blind judging. --answers-b is the arm judged twice for the noise floor. +# Grades are checkpointed; re-running the same command resumes. +uv run python -m evals.retrieval judge \ + --questions /tmp/eval/questions.jsonl \ + --answers-a /tmp/eval/answers-old.jsonl --answers-b /tmp/eval/answers-new.jsonl \ + --work-dir /tmp/eval/judge + +# 4 + 5. Score and render. +uv run python -m evals.retrieval score \ + --questions /tmp/eval/questions.jsonl \ + --answers-a /tmp/eval/answers-old.jsonl --answers-b /tmp/eval/answers-new.jsonl \ + --work-dir /tmp/eval/judge --out /tmp/eval/results.jsonl +uv run python -m evals.retrieval report \ + --results /tmp/eval/results.jsonl --out /tmp/eval/report.md +``` + +Question generation and judging talk to the configured lilbee chat model by +default (set `LILBEE_CHAT_MODEL` before step 1, and switch it to a stronger +judge model before step 3 if both run locally). The judge can instead be any +OpenAI-compatible endpoint: + +```bash +export LILBEE_EVAL_JUDGE_BASE_URL="https://api.example.com/v1" +export LILBEE_EVAL_JUDGE_MODEL="judge-model-name" +export LILBEE_EVAL_JUDGE_API_KEY="..." # optional +``` + +## Determinism and resume + +- `--seed` (questions, judge) makes sampling, blinding ids, and shuffles + reproducible. Re-running `judge` with the same seed and inputs regenerates + the identical blind set, so the grades checkpoint stays valid. +- `answer` and `judge` both checkpoint per row. Kill and re-run freely; only + unfinished work repeats. +- Answers that hard-fail after all retries are recorded as failures and score + zero everywhere (prefailed: they never reach the judge). + +## Files in a judge work dir + +| file | contents | +| --- | --- | +| `blind_rows.jsonl` | what judges see: gid, question, ground truth, answer | +| `gid_map.json` | secret gid to (qid, arm, replicate) mapping | +| `prefailed.json` | gids scored zero without judging | +| `grades.jsonl` | checkpointed judge output, one row per gid | diff --git a/evals/retrieval/__init__.py b/evals/retrieval/__init__.py new file mode 100644 index 000000000..778cae64f --- /dev/null +++ b/evals/retrieval/__init__.py @@ -0,0 +1 @@ +"""Blind, noise-calibrated A/B evaluation of lilbee retrieval quality.""" diff --git a/evals/retrieval/__main__.py b/evals/retrieval/__main__.py new file mode 100644 index 000000000..0f9016f24 --- /dev/null +++ b/evals/retrieval/__main__.py @@ -0,0 +1,6 @@ +"""Runs the retrieval eval CLI: python -m evals.retrieval .""" + +from evals.retrieval.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/retrieval/answers.py b/evals/retrieval/answers.py new file mode 100644 index 000000000..d3db37578 --- /dev/null +++ b/evals/retrieval/answers.py @@ -0,0 +1,133 @@ +"""Answer collection from a running lilbee server, checkpointed for pod restarts.""" + +from __future__ import annotations + +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +import httpx + +from evals.retrieval.checkpoint import JsonlCheckpoint +from evals.retrieval.questions import Question + +HEALTH_ROUTE = "/api/health" +ASK_ROUTE = "/api/ask" +ASK_TIMEOUT_SECONDS = 600.0 +ANSWER_ATTEMPTS = 3 +ANSWER_RETRY_DELAY_SECONDS = 5.0 +WARMUP_ATTEMPTS = 180 +WARMUP_POLL_SECONDS = 5.0 + + +@dataclass +class AnswerRow: + """One arm's answer to one question, or its hard failure.""" + + qid: str + arm: str + answer: str + sources: list[str] + cited_sources: list[str] + seconds: float + error: str | None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> AnswerRow: + return cls( + qid=data["qid"], + arm=data["arm"], + answer=data["answer"], + sources=list(data["sources"]), + cited_sources=list(data["cited_sources"]), + seconds=data["seconds"], + error=data["error"], + ) + + +def make_http_client() -> httpx.Client: + return httpx.Client(timeout=ASK_TIMEOUT_SECONDS) + + +def wait_for_server( + base_url: str, + client: httpx.Client, + *, + attempts: int = WARMUP_ATTEMPTS, + poll: float = WARMUP_POLL_SECONDS, +) -> None: + """Block until the server's health route answers 200.""" + for _ in range(attempts): + try: + if client.get(f"{base_url.rstrip('/')}{HEALTH_ROUTE}").status_code == httpx.codes.OK: + return + except httpx.HTTPError: + pass + time.sleep(poll) + raise RuntimeError(f"server at {base_url} never became healthy") + + +def _ask(client: httpx.Client, base_url: str, question: str, top_k: int) -> AnswerRow: + response = client.post( + f"{base_url.rstrip('/')}{ASK_ROUTE}", json={"question": question, "top_k": top_k} + ) + response.raise_for_status() + body = response.json() + return AnswerRow( + qid="", + arm="", + answer=body["answer"], + sources=[chunk["source"] for chunk in body["sources"]], + cited_sources=[chunk["source"] for chunk in body.get("cited_sources", [])], + seconds=0.0, + error=None, + ) + + +def answer_questions( + questions: list[Question], + base_url: str, + arm: str, + out_path: Path, + *, + top_k: int = 0, + attempts: int = ANSWER_ATTEMPTS, + retry_delay: float = ANSWER_RETRY_DELAY_SECONDS, + client: httpx.Client | None = None, +) -> list[AnswerRow]: + """Answer every question not already checkpointed; return this run's rows.""" + http = client or make_http_client() + checkpoint = JsonlCheckpoint(out_path, "qid") + wait_for_server(base_url, http) + rows: list[AnswerRow] = [] + for question in questions: + if question.qid in checkpoint: + continue + started = time.monotonic() + row: AnswerRow | None = None + error = "" + for attempt in range(attempts): + try: + row = _ask(http, base_url, question.question, top_k) + break + except (httpx.HTTPError, KeyError, ValueError) as exc: + error = f"{type(exc).__name__}: {exc}" + if attempt + 1 < attempts: + time.sleep(retry_delay) + if row is None: + # A question that fails all attempts is itself a result. + row = AnswerRow( + qid="", arm="", answer="", sources=[], cited_sources=[], seconds=0.0, error=error + ) + row.qid = question.qid + row.arm = arm + row.seconds = round(time.monotonic() - started, 2) + checkpoint.append(row.to_dict()) + rows.append(row) + status = "ERROR" if row.error else "ok" + print(f"[{arm}] {question.qid} {status}", flush=True) + return rows diff --git a/evals/retrieval/blinding.py b/evals/retrieval/blinding.py new file mode 100644 index 000000000..ed4023cc3 --- /dev/null +++ b/evals/retrieval/blinding.py @@ -0,0 +1,119 @@ +"""Blind row construction and mechanical unblinding. + +Every gradable (question, answer) pair becomes a row under an opaque gid. +The noise arm's answers appear twice (replicates 0 and 1) so the judge's +per-question disagreement with itself is measurable; all rows shuffle +together, so a judge cannot tell arms, replicates, or that a comparison is +happening at all. +""" + +from __future__ import annotations + +import random +from dataclasses import asdict, dataclass +from typing import Any, NamedTuple + +from evals.retrieval.answers import AnswerRow +from evals.retrieval.questions import Question, QuestionKind + +GROUND_CHARS = 2400 +ANSWER_CHARS = 2400 +GID_SPACE = 10**9 +NOISE_REPLICATES = 2 +JUDGED_KINDS = (QuestionKind.TOPICAL, QuestionKind.KNOWN_ITEM) + + +@dataclass(frozen=True) +class BlindRow: + """What a judge sees: no qid, no arm, no replicate.""" + + gid: str + question: str + source: str + ground: str + answer: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class BlindAssignment: + """The secret side of a gid; never shown to a judge.""" + + qid: str + arm: str + replicate: int + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> BlindAssignment: + return cls(qid=data["qid"], arm=data["arm"], replicate=data["replicate"]) + + +class BlindSet(NamedTuple): + rows: list[BlindRow] + assignments: dict[str, BlindAssignment] + prefailed: list[str] + + +def _new_gid(rng: random.Random, taken: set[str]) -> str: + gid = f"g{rng.randrange(GID_SPACE):09d}" + while gid in taken: + gid = f"g{rng.randrange(GID_SPACE):09d}" + return gid + + +def build_blind_rows( + questions: list[Question], + answers_by_arm: dict[str, dict[str, AnswerRow]], + noise_arm: str, + rng: random.Random, +) -> BlindSet: + """Blind rows for every judged question, with the noise arm duplicated. + + Missing, errored, and empty answers are prefailed: they score zero without + wasting a judge call, and their gids never reach a judge. + """ + rows: list[BlindRow] = [] + assignments: dict[str, BlindAssignment] = {} + prefailed: list[str] = [] + for question in questions: + if question.kind not in JUDGED_KINDS: + continue + for arm, answers in answers_by_arm.items(): + replicates = NOISE_REPLICATES if arm == noise_arm else 1 + for replicate in range(replicates): + gid = _new_gid(rng, set(assignments)) + assignments[gid] = BlindAssignment(qid=question.qid, arm=arm, replicate=replicate) + answer = answers.get(question.qid) + if answer is None or answer.error or not answer.answer.strip(): + prefailed.append(gid) + continue + rows.append( + BlindRow( + gid=gid, + question=question.question, + source=question.source, + ground=question.ground_passage[:GROUND_CHARS], + answer=answer.answer[:ANSWER_CHARS], + ) + ) + rng.shuffle(rows) + return BlindSet(rows=rows, assignments=assignments, prefailed=prefailed) + + +def unblind( + assignments: dict[str, BlindAssignment], grades: dict[str, dict[str, int]] +) -> dict[str, dict[int, dict[str, dict[str, int]]]]: + """Regroup blind grades as arm -> replicate -> qid -> scores.""" + regrouped: dict[str, dict[int, dict[str, dict[str, int]]]] = {} + for gid, assignment in assignments.items(): + by_replicate = regrouped.setdefault(assignment.arm, {}) + by_qid = by_replicate.setdefault(assignment.replicate, {}) + scores = grades.get(gid) + if scores is not None: + by_qid[assignment.qid] = scores + return regrouped diff --git a/evals/retrieval/checkpoint.py b/evals/retrieval/checkpoint.py new file mode 100644 index 000000000..9769c766c --- /dev/null +++ b/evals/retrieval/checkpoint.py @@ -0,0 +1,36 @@ +"""Append-only JSONL checkpointing so interrupted runs resume without rework.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def load_jsonl(path: Path) -> list[dict[str, Any]]: + """Parse one JSON object per line, skipping blanks; missing file is empty.""" + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +class JsonlCheckpoint: + """Appends keyed records to a JSONL file and remembers which keys are done.""" + + def __init__(self, path: Path, key_field: str) -> None: + self._path = path + self._key_field = key_field + self._done = {str(record[key_field]) for record in load_jsonl(path)} + path.parent.mkdir(parents=True, exist_ok=True) + + @property + def done(self) -> set[str]: + return set(self._done) + + def __contains__(self, key: str) -> bool: + return key in self._done + + def append(self, record: dict[str, Any]) -> None: + with self._path.open("a") as fh: + fh.write(json.dumps(record) + "\n") + self._done.add(str(record[self._key_field])) diff --git a/evals/retrieval/cli.py b/evals/retrieval/cli.py new file mode 100644 index 000000000..1a80a3f30 --- /dev/null +++ b/evals/retrieval/cli.py @@ -0,0 +1,200 @@ +"""Command-line entry point: questions, answer, judge, score, report.""" + +from __future__ import annotations + +import argparse +import json +import random +import sys +from pathlib import Path +from typing import Any + +from evals.retrieval.answers import AnswerRow, answer_questions, make_http_client +from evals.retrieval.blinding import BlindAssignment, build_blind_rows, unblind +from evals.retrieval.checkpoint import load_jsonl +from evals.retrieval.judging import DIMENSIONS, judge_rows +from evals.retrieval.llm import judge_chat_fn, lilbee_chat_fn, warm_chat +from evals.retrieval.questions import ( + COUNT_QUESTIONS, + DEFAULT_SEED, + KNOWN_ITEM_QUESTIONS, + TOPICAL_QUESTIONS, + Question, + build_questions, +) +from evals.retrieval.report import render_report +from evals.retrieval.scoring import build_results + +GID_MAP_FILE = "gid_map.json" +PREFAILED_FILE = "prefailed.json" +BLIND_ROWS_FILE = "blind_rows.jsonl" +GRADES_FILE = "grades.jsonl" + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + +def _load_questions(path: Path) -> list[Question]: + return [Question.from_dict(row) for row in load_jsonl(path)] + + +def _load_answer_arm(path: Path) -> tuple[str, dict[str, AnswerRow]]: + rows = [AnswerRow.from_dict(row) for row in load_jsonl(path)] + if not rows: + raise ValueError(f"no answer rows in {path}") + return rows[0].arm, {row.qid: row for row in rows} + + +def _load_arms(args: argparse.Namespace) -> dict[str, dict[str, AnswerRow]]: + arm_a, answers_a = _load_answer_arm(args.answers_a) + arm_b, answers_b = _load_answer_arm(args.answers_b) + if arm_a == arm_b: + raise ValueError(f"both answer files carry the arm label '{arm_a}'") + return {arm_a: answers_a, arm_b: answers_b} + + +def _cmd_questions(args: argparse.Namespace) -> int: + lancedb_dir = args.lancedb_dir or args.data_root / "data" / "lancedb" + chat = lilbee_chat_fn() + warm_chat(chat) + questions = build_questions( + lancedb_dir, + chat, + topical=args.topical, + known_item=args.known_item, + count=args.count, + seed=args.seed, + ) + _write_jsonl(args.out, [question.to_dict() for question in questions]) + print(f"wrote {len(questions)} questions -> {args.out}") + return 0 + + +def _cmd_answer(args: argparse.Namespace) -> int: + questions = _load_questions(args.questions) + rows = answer_questions( + questions, + args.base_url, + args.arm, + args.out, + top_k=args.top_k, + client=make_http_client(), + ) + failed = sum(1 for row in rows if row.error) + print(f"answered {len(rows)} new questions, {failed} errors -> {args.out}") + return 0 + + +def _cmd_judge(args: argparse.Namespace) -> int: + questions = _load_questions(args.questions) + answers_by_arm = _load_arms(args) + noise_arm = next(reversed(answers_by_arm)) + blind = build_blind_rows(questions, answers_by_arm, noise_arm, random.Random(args.seed)) + args.work_dir.mkdir(parents=True, exist_ok=True) + (args.work_dir / GID_MAP_FILE).write_text( + json.dumps({gid: a.to_dict() for gid, a in blind.assignments.items()}, indent=1) + ) + (args.work_dir / PREFAILED_FILE).write_text(json.dumps(blind.prefailed, indent=1)) + _write_jsonl(args.work_dir / BLIND_ROWS_FILE, [row.to_dict() for row in blind.rows]) + chat = judge_chat_fn() + warm_chat(chat) + grades = judge_rows(blind.rows, chat, args.work_dir / GRADES_FILE) + unreturned = len(blind.rows) - sum(1 for row in blind.rows if row.gid in grades) + print( + f"{len(blind.rows)} blind rows judged ({unreturned} unreturned), " + f"{len(blind.prefailed)} prefailed -> {args.work_dir}" + ) + return 0 + + +def _cmd_score(args: argparse.Namespace) -> int: + questions = _load_questions(args.questions) + answers_by_arm = _load_arms(args) + noise_arm = next(reversed(answers_by_arm)) + assignments = { + gid: BlindAssignment.from_dict(data) + for gid, data in json.loads((args.work_dir / GID_MAP_FILE).read_text()).items() + } + prefailed = json.loads((args.work_dir / PREFAILED_FILE).read_text()) + grades = { + record["gid"]: {k: v for k, v in record.items() if k != "gid"} + for record in load_jsonl(args.work_dir / GRADES_FILE) + } + for gid in prefailed: + grades[gid] = dict.fromkeys(DIMENSIONS, 0) + results = build_results(questions, answers_by_arm, unblind(assignments, grades), noise_arm) + _write_jsonl(args.out, results) + print(f"wrote {len(results)} result rows -> {args.out}") + return 0 + + +def _cmd_report(args: argparse.Namespace) -> int: + report = render_report(load_jsonl(args.results)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(report) + print(f"wrote {args.out}") + return 0 + + +def _add_arm_io_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--questions", type=Path, required=True) + parser.add_argument("--answers-a", type=Path, required=True) + parser.add_argument( + "--answers-b", type=Path, required=True, help="the arm judged twice for the noise floor" + ) + parser.add_argument("--work-dir", type=Path, required=True) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="evals.retrieval", description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + questions = subparsers.add_parser("questions", help="generate the question battery") + questions.add_argument("--data-root", type=Path, required=True) + questions.add_argument( + "--lancedb-dir", + type=Path, + default=None, + help="override the default /data/lancedb", + ) + questions.add_argument("--out", type=Path, required=True) + questions.add_argument("--topical", type=int, default=TOPICAL_QUESTIONS) + questions.add_argument("--known-item", type=int, default=KNOWN_ITEM_QUESTIONS) + questions.add_argument("--count", type=int, default=COUNT_QUESTIONS) + questions.add_argument("--seed", type=int, default=DEFAULT_SEED) + questions.set_defaults(handler=_cmd_questions) + + answer = subparsers.add_parser("answer", help="answer the battery against one server") + answer.add_argument("--questions", type=Path, required=True) + answer.add_argument("--base-url", required=True) + answer.add_argument("--arm", required=True, help="label recorded on every row") + answer.add_argument("--out", type=Path, required=True) + answer.add_argument("--top-k", type=int, default=0) + answer.set_defaults(handler=_cmd_answer) + + judge = subparsers.add_parser("judge", help="blind-judge both arms' answers") + _add_arm_io_arguments(judge) + judge.add_argument("--seed", type=int, default=DEFAULT_SEED) + judge.set_defaults(handler=_cmd_judge) + + score = subparsers.add_parser("score", help="unblind grades and write results.jsonl") + _add_arm_io_arguments(score) + score.add_argument("--out", type=Path, required=True) + score.set_defaults(handler=_cmd_score) + + report = subparsers.add_parser("report", help="render results.jsonl as markdown") + report.add_argument("--results", type=Path, required=True) + report.add_argument("--out", type=Path, required=True) + report.set_defaults(handler=_cmd_report) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return int(args.handler(args)) + except (ValueError, FileNotFoundError, RuntimeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 diff --git a/evals/retrieval/judging.py b/evals/retrieval/judging.py new file mode 100644 index 000000000..777ceb7db --- /dev/null +++ b/evals/retrieval/judging.py @@ -0,0 +1,97 @@ +"""Blind judging: one answer at a time against its ground truth.""" + +from __future__ import annotations + +import json +import re +import sys +import time +from enum import StrEnum +from pathlib import Path + +from evals.retrieval.blinding import BlindRow +from evals.retrieval.checkpoint import JsonlCheckpoint, load_jsonl +from evals.retrieval.llm import ChatFn + +SCORE_MIN = 0 +SCORE_MAX = 2 +JUDGE_ATTEMPTS = 3 +JUDGE_RETRY_DELAY_SECONDS = 5.0 + +_JSON_RE = re.compile(r"\{[^{}]*\}") + + +class Dimension(StrEnum): + FAITHFULNESS = "faithfulness" + RELEVANCE = "relevance" + CITATION = "citation" + + +DIMENSIONS = tuple(str(dimension) for dimension in Dimension) + +JUDGE_PROMPT = ( + "You are grading an answer produced by a document-search assistant.\n" + "Question: {question}\n\n" + "Ground-truth material the answer should be based on (from document " + "{source}):\n{ground}\n\n" + "Answer to grade:\n{answer}\n\n" + "Score STRICTLY as JSON with integer fields 0-2 each: " + '{{"faithfulness": _, "relevance": _, "citation": _}}. ' + "faithfulness: 2 = every claim supported by the ground material, " + "1 = minor unsupported detail, 0 = contradicts or invents. " + "relevance: 2 = directly answers the question, 1 = partial, 0 = misses. " + "citation: 2 = names or cites the correct document, 1 = cites nothing, " + "0 = cites a wrong document. Return ONLY the JSON." +) + + +def parse_grade(text: str) -> dict[str, int] | None: + """Scores clamped to the 0-2 scale, or None when the judge returned junk.""" + match = _JSON_RE.search(text) + if not match: + return None + try: + scores = json.loads(match.group(0)) + return { + dimension: max(SCORE_MIN, min(SCORE_MAX, int(scores[dimension]))) + for dimension in DIMENSIONS + } + except (KeyError, TypeError, ValueError): + return None + + +def judge_rows( + rows: list[BlindRow], + chat: ChatFn, + out_path: Path, + *, + attempts: int = JUDGE_ATTEMPTS, + retry_delay: float = JUDGE_RETRY_DELAY_SECONDS, +) -> dict[str, dict[str, int]]: + """Grade every row not already checkpointed; return all grades on disk.""" + checkpoint = JsonlCheckpoint(out_path, "gid") + for row in rows: + if row.gid in checkpoint: + continue + prompt = JUDGE_PROMPT.format( + question=row.question, source=row.source, ground=row.ground, answer=row.answer + ) + grade: dict[str, int] | None = None + for attempt in range(attempts): + try: + grade = parse_grade(chat(prompt)) + except Exception as exc: # a failed grade is skipped, not fatal + print(f"judge call failed for {row.gid}: {exc}", file=sys.stderr) + grade = None + if grade is not None: + break + if attempt + 1 < attempts: + time.sleep(retry_delay) + if grade is None: + print(f"no parseable grade for {row.gid}; leaving unreturned", file=sys.stderr) + continue + checkpoint.append({"gid": row.gid, **grade}) + return { + record["gid"]: {dimension: record[dimension] for dimension in DIMENSIONS} + for record in load_jsonl(out_path) + } diff --git a/evals/retrieval/llm.py b/evals/retrieval/llm.py new file mode 100644 index 000000000..de120d66b --- /dev/null +++ b/evals/retrieval/llm.py @@ -0,0 +1,82 @@ +"""Chat backends for question authoring and judging.""" + +from __future__ import annotations + +import os +import time +from collections.abc import Callable + +import httpx + +ChatFn = Callable[[str], str] + +JUDGE_BASE_URL_ENV = "LILBEE_EVAL_JUDGE_BASE_URL" +JUDGE_MODEL_ENV = "LILBEE_EVAL_JUDGE_MODEL" +JUDGE_API_KEY_ENV = "LILBEE_EVAL_JUDGE_API_KEY" + +CHAT_MAX_TOKENS = 256 +CHAT_TIMEOUT_SECONDS = 300.0 +WARM_ATTEMPTS = 30 +WARM_DELAY_SECONDS = 10.0 + + +def openai_chat_fn( + base_url: str, model: str, api_key: str | None = None, *, client: httpx.Client | None = None +) -> ChatFn: + """Prompt-to-text over an OpenAI-compatible /chat/completions endpoint.""" + http = client or httpx.Client(timeout=CHAT_TIMEOUT_SECONDS) + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + url = f"{base_url.rstrip('/')}/chat/completions" + + def chat(prompt: str) -> str: + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0, + "max_tokens": CHAT_MAX_TOKENS, + } + response = http.post(url, json=payload, headers=headers) + response.raise_for_status() + return str(response.json()["choices"][0]["message"]["content"]) + + return chat + + +def lilbee_chat_fn() -> ChatFn: + """Prompt-to-text on the configured lilbee chat model, reasoning stripped.""" + from lilbee.app.services import get_services # heavy: builds the service container + from lilbee.retrieval.reasoning import strip_reasoning + + provider = get_services().provider + options = {"temperature": 0, "num_predict": CHAT_MAX_TOKENS, "think": False} + + def chat(prompt: str) -> str: + result = provider.chat([{"role": "user", "content": prompt}], options=options) + return strip_reasoning(result.text) + + return chat + + +def judge_chat_fn() -> ChatFn: + """The judge backend: env-configured OpenAI-compatible endpoint, else lilbee.""" + base_url = os.environ.get(JUDGE_BASE_URL_ENV) + if base_url: + model = os.environ.get(JUDGE_MODEL_ENV, "") + return openai_chat_fn(base_url, model, os.environ.get(JUDGE_API_KEY_ENV)) + return lilbee_chat_fn() + + +def warm_chat( + chat: ChatFn, attempts: int = WARM_ATTEMPTS, delay: float = WARM_DELAY_SECONDS +) -> None: + """Block until the chat backend answers; first calls race model load.""" + last: Exception | None = None + for _ in range(attempts): + try: + chat("ok") + except Exception as exc: # retried until the budget runs out + last = exc + time.sleep(delay) + else: + return + raise RuntimeError(f"chat backend never came up: {last}") diff --git a/evals/retrieval/questions.py b/evals/retrieval/questions.py new file mode 100644 index 000000000..c980b43e0 --- /dev/null +++ b/evals/retrieval/questions.py @@ -0,0 +1,211 @@ +"""Corpus-agnostic question generation from an existing lilbee index. + +Three kinds, each with ground truth attached at authoring time so judging +never needs the index again: + +- topical: written by the chat model FROM a sampled stored passage, so the + passage that must support the answer is known. +- known_item: asks what a sampled document is about; ground truth is the + document's head chunks. +- count: asks how many chunks and documents mention a term; ground truth is + an exact streaming scan of the store, no judge involved. +""" + +from __future__ import annotations + +import collections +import random +import re +import sys +import time +from dataclasses import asdict, dataclass +from enum import StrEnum +from pathlib import Path +from typing import Any + +from evals.retrieval.llm import ChatFn +from evals.retrieval.store_scan import ( + count_term_hits, + iter_chunks, + iter_source_names, + reservoir_sample, + scan_passages_and_heads, +) + +TOPICAL_QUESTIONS = 60 +KNOWN_ITEM_QUESTIONS = 20 +COUNT_QUESTIONS = 8 +MIN_PASSAGE_CHARS = 400 +PASSAGE_PROMPT_CHARS = 1800 +MIN_QUESTION_CHARS = 16 +TERM_DF_LOW = 0.05 +TERM_DF_HIGH = 0.4 +AUTHOR_ATTEMPTS = 3 +AUTHOR_RETRY_DELAY_SECONDS = 5.0 +DEFAULT_SEED = 20260714 + +_WORD_RE = re.compile(r"[a-z]{5,16}") + +QUESTION_PROMPT = ( + "Below is a passage from a document. Write ONE specific question that this " + "passage answers. The question must be answerable from the passage alone, " + "must not mention 'the passage' or 'the text', and must not quote more " + "than three consecutive words from it. Return ONLY the question.\n\n" + "Passage:\n{passage}" +) + + +class QuestionKind(StrEnum): + TOPICAL = "topical" + KNOWN_ITEM = "known_item" + COUNT = "count" + + +@dataclass +class CountOracle: + """Exact scan result a count answer is checked against.""" + + term: str + chunks: int + sources: int + + +@dataclass +class Question: + """One eval question with its ground truth attached.""" + + qid: str + kind: QuestionKind + question: str + source: str = "" + ground_passage: str = "" + oracle: CountOracle | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Question: + oracle = data.get("oracle") + return cls( + qid=data["qid"], + kind=QuestionKind(data["kind"]), + question=data["question"], + source=data.get("source", ""), + ground_passage=data.get("ground_passage", ""), + oracle=CountOracle(**oracle) if oracle else None, + ) + + +def parse_question(text: str) -> str | None: + """The first line as a question, or None when the model returned junk.""" + lines = text.strip().splitlines() + if not lines: + return None + candidate = lines[0].strip().strip('"') + if candidate.endswith("?") and len(candidate) >= MIN_QUESTION_CHARS: + return candidate + return None + + +def author_topical( + passages: list[tuple[str, str]], + chat: ChatFn, + *, + attempts: int = AUTHOR_ATTEMPTS, + retry_delay: float = AUTHOR_RETRY_DELAY_SECONDS, +) -> list[Question]: + """One question per sampled passage; a bad passage never kills the set.""" + questions: list[Question] = [] + for index, (source, passage) in enumerate(passages): + prompt = QUESTION_PROMPT.format(passage=passage[:PASSAGE_PROMPT_CHARS]) + response = None + for attempt in range(attempts): + try: + response = chat(prompt) + break + except Exception as exc: # a failed question is skipped, not fatal + print( + f"authoring attempt {attempt + 1} failed for {source}: {exc}", file=sys.stderr + ) + time.sleep(retry_delay) + if response is None: + continue + question = parse_question(response) + if question is None: + continue + questions.append( + Question( + qid=f"tq{index:03d}", + kind=QuestionKind.TOPICAL, + question=question, + source=source, + ground_passage=passage, + ) + ) + return questions + + +def sample_terms(passages: list[tuple[str, str]], count: int, rng: random.Random) -> list[str]: + """Mid-document-frequency terms drawn from the sampled passages. + + Document frequency is estimated over the sample, never the full index; + the exact oracle counts come from a full streaming scan afterwards. + """ + per_source_terms: dict[str, set[str]] = collections.defaultdict(set) + for source, passage in passages: + per_source_terms[source].update(_WORD_RE.findall(passage.lower())) + doc_freq: collections.Counter[str] = collections.Counter() + for terms in per_source_terms.values(): + doc_freq.update(terms) + total = len(per_source_terms) + if not total: + return [] + band = sorted(t for t, df in doc_freq.items() if TERM_DF_LOW <= df / total <= TERM_DF_HIGH) + rng.shuffle(band) + return band[:count] + + +def build_questions( + lancedb_dir: Path, + chat: ChatFn, + *, + topical: int = TOPICAL_QUESTIONS, + known_item: int = KNOWN_ITEM_QUESTIONS, + count: int = COUNT_QUESTIONS, + seed: int = DEFAULT_SEED, + min_passage_chars: int = MIN_PASSAGE_CHARS, +) -> list[Question]: + """The full question battery: sampled sources, two streaming chunk scans.""" + rng = random.Random(seed) + known_sources = sorted(reservoir_sample(iter_source_names(lancedb_dir), known_item, rng)) + scan = scan_passages_and_heads( + iter_chunks(lancedb_dir), + passage_count=topical, + min_passage_chars=min_passage_chars, + head_sources=set(known_sources), + rng=rng, + ) + questions = author_topical(scan.passages, chat) + for index, name in enumerate(known_sources): + questions.append( + Question( + qid=f"ki{index:03d}", + kind=QuestionKind.KNOWN_ITEM, + question=f"What is {name} about? Summarize it briefly.", + source=name, + ground_passage=scan.doc_heads.get(name, ""), + ) + ) + terms = sample_terms(scan.passages, count, rng) + hits = count_term_hits(iter_chunks(lancedb_dir), terms) if terms else {} + for index, term in enumerate(terms): + questions.append( + Question( + qid=f"ct{index:03d}", + kind=QuestionKind.COUNT, + question=f"How many documents mention {term}?", + oracle=CountOracle(term=term, chunks=hits[term].chunks, sources=hits[term].sources), + ) + ) + return questions diff --git a/evals/retrieval/report.py b/evals/retrieval/report.py new file mode 100644 index 000000000..603968c00 --- /dev/null +++ b/evals/retrieval/report.py @@ -0,0 +1,57 @@ +"""Markdown rendering of a scored eval run.""" + +from __future__ import annotations + +from typing import Any + +from evals.retrieval.judging import DIMENSIONS +from evals.retrieval.scoring import ResultRowType + + +def _delta_cell(first: float, second: float, noise: float) -> str: + delta = round(second - first, 3) + flag = "" if abs(delta) > noise else " (within noise)" + return f"{delta:+g}{flag}" + + +def render_report(rows: list[dict[str, Any]]) -> str: + """The human-readable report for a results.jsonl produced by score.""" + summary = next((row for row in rows if row["row_type"] == ResultRowType.SUMMARY), None) + if summary is None: + raise ValueError("results contain no summary row; run score first") + arms: dict[str, dict[str, Any]] = summary["arms"] + first, second = list(arms) + noise = summary["noise_floor"] + + lines = [ + "# Retrieval eval report", + "", + f"{summary['judged']} judge-graded questions per arm; the second arm was " + "judged twice under fresh opaque ids to measure the judge's noise floor. " + "Judges saw only question + ground truth + one answer; no arm labels.", + f"Judge noise floor (same answers re-graded blind): plus or minus {noise} " + "per dimension. Deltas at or below it are labeled within noise.", + "", + f"| dimension | {first} | {second} | delta |", + "| --- | --- | --- | --- |", + ] + for dimension in DIMENSIONS: + first_mean = arms[first]["means"][dimension] + second_mean = arms[second]["means"][dimension] + lines.append( + f"| {dimension} (0-2) | {first_mean} | {second_mean} " + f"| {_delta_cell(first_mean, second_mean, noise)} |" + ) + lines += [ + "", + f"| exact-truth check | {first} | {second} |", + "| --- | --- | --- |", + ] + for label, key in ( + ("count questions", "count_pass"), + ("known-item citation", "known_item_pass"), + ): + cells = [f"{arms[arm][key][0]}/{arms[arm][key][1]}" for arm in (first, second)] + lines.append(f"| {label} | {cells[0]} | {cells[1]} |") + lines.append(f"| hard failures | {arms[first]['errors']} | {arms[second]['errors']} |") + return "\n".join(lines) + "\n" diff --git a/evals/retrieval/scoring.py b/evals/retrieval/scoring.py new file mode 100644 index 000000000..1181be7bf --- /dev/null +++ b/evals/retrieval/scoring.py @@ -0,0 +1,130 @@ +"""Scoring: noise-floor math, per-dimension means, and exact-truth checks.""" + +from __future__ import annotations + +import re +import statistics +from enum import StrEnum +from typing import Any + +from evals.retrieval.answers import AnswerRow +from evals.retrieval.judging import DIMENSIONS +from evals.retrieval.questions import CountOracle, Question, QuestionKind + +_NUMBER_RE = re.compile(r"\d+") + +Grades = dict[str, dict[str, int]] +Unblinded = dict[str, dict[int, Grades]] + +ARM_REPLICATE = 0 +NOISE_REPLICATE = 1 + + +class ResultRowType(StrEnum): + QUESTION = "question" + SUMMARY = "summary" + + +def noise_floor(rep0: Grades, rep1: Grades) -> float: + """Mean absolute per-question, per-dimension disagreement between replicates.""" + shared = [qid for qid in rep0 if qid in rep1] + if not shared: + return 0.0 + per_question = [ + sum(abs(rep0[qid][d] - rep1[qid][d]) for d in DIMENSIONS) / len(DIMENSIONS) + for qid in shared + ] + return round(statistics.mean(per_question), 3) + + +def dimension_means(grades: Grades) -> dict[str, float]: + """Per-dimension mean scores over every graded question.""" + if not grades: + return dict.fromkeys(DIMENSIONS, 0.0) + return { + d: round(statistics.mean(scores[d] for scores in grades.values()), 3) for d in DIMENSIONS + } + + +def count_question_pass(oracle: CountOracle, answer: str) -> bool: + """Both oracle numbers must appear in the answer.""" + numbers = set(_NUMBER_RE.findall(answer)) + return {str(oracle.chunks), str(oracle.sources)} <= numbers + + +def known_item_pass(source: str, row: AnswerRow) -> bool: + """The expected document must be among the answer's cited sources.""" + return source in row.cited_sources + + +def _question_row( + question: Question, arm: str, answer: AnswerRow | None, grades: Grades +) -> dict[str, Any]: + row: dict[str, Any] = { + "row_type": ResultRowType.QUESTION, + "qid": question.qid, + "kind": question.kind, + "arm": arm, + "error": answer.error if answer is not None else "missing", + } + if question.kind in (QuestionKind.TOPICAL, QuestionKind.KNOWN_ITEM): + row["grades"] = grades.get(question.qid) + answered = answer is not None and not answer.error + if question.kind is QuestionKind.COUNT and question.oracle is not None: + row["exact_pass"] = answered and count_question_pass(question.oracle, answer.answer) + elif question.kind is QuestionKind.KNOWN_ITEM: + row["exact_pass"] = answered and known_item_pass(question.source, answer) + return row + + +def _arm_summary( + questions: list[Question], answers: dict[str, AnswerRow], grades: Grades +) -> dict[str, Any]: + count_hits = count_total = known_hits = known_total = errors = 0 + for question in questions: + answer = answers.get(question.qid) + if answer is None or answer.error: + errors += 1 + answer = None + if question.kind is QuestionKind.COUNT and question.oracle is not None: + count_total += 1 + count_hits += answer is not None and count_question_pass(question.oracle, answer.answer) + elif question.kind is QuestionKind.KNOWN_ITEM: + known_total += 1 + known_hits += answer is not None and known_item_pass(question.source, answer) + return { + "means": dimension_means(grades), + "count_pass": [count_hits, count_total], + "known_item_pass": [known_hits, known_total], + "errors": errors, + } + + +def build_results( + questions: list[Question], + answers_by_arm: dict[str, dict[str, AnswerRow]], + unblinded: Unblinded, + noise_arm: str, +) -> list[dict[str, Any]]: + """Per-question rows for every arm, then one summary row, results.jsonl shaped.""" + rows: list[dict[str, Any]] = [] + for question in questions: + for arm, answers in answers_by_arm.items(): + arm_grades = unblinded.get(arm, {}).get(ARM_REPLICATE, {}) + rows.append(_question_row(question, arm, answers.get(question.qid), arm_grades)) + noise_replicates = unblinded.get(noise_arm, {}) + summary: dict[str, Any] = { + "row_type": ResultRowType.SUMMARY, + "noise_floor": noise_floor( + noise_replicates.get(ARM_REPLICATE, {}), noise_replicates.get(NOISE_REPLICATE, {}) + ), + "judged": sum( + 1 for q in questions if q.kind in (QuestionKind.TOPICAL, QuestionKind.KNOWN_ITEM) + ), + "arms": { + arm: _arm_summary(questions, answers, unblinded.get(arm, {}).get(ARM_REPLICATE, {})) + for arm, answers in answers_by_arm.items() + }, + } + rows.append(summary) + return rows diff --git a/evals/retrieval/store_scan.py b/evals/retrieval/store_scan.py new file mode 100644 index 000000000..80d960742 --- /dev/null +++ b/evals/retrieval/store_scan.py @@ -0,0 +1,137 @@ +"""Streaming access to a lilbee LanceDB index; nothing materializes the table.""" + +from __future__ import annotations + +import random +from collections.abc import Iterable, Iterator, Sequence +from pathlib import Path +from typing import NamedTuple, TypeVar + +SCAN_BATCH_ROWS = 8192 +HEAD_CHUNK_COUNT = 3 +PASSAGE_OVERSAMPLE = 4 + +T = TypeVar("T") + + +class ChunkRow(NamedTuple): + source: str + chunk: str + chunk_index: int + + +class TermCounts(NamedTuple): + chunks: int + sources: int + + +class PassageScan(NamedTuple): + passages: list[tuple[str, str]] + doc_heads: dict[str, str] + + +def iter_chunks(lancedb_dir: Path) -> Iterator[ChunkRow]: + """Stream (source, chunk, chunk_index) rows batch by batch.""" + import lancedb # heavy: arrow + datafusion + + from lilbee.core.config.defaults import CHUNKS_TABLE + + table = lancedb.connect(str(lancedb_dir)).open_table(CHUNKS_TABLE) + columns = ["source", "chunk", "chunk_index"] + for batch in table.search().select(columns).to_batches(batch_size=SCAN_BATCH_ROWS): + yield from ( + ChunkRow(source, chunk, index) + for source, chunk, index in zip( + batch.column("source").to_pylist(), + batch.column("chunk").to_pylist(), + batch.column("chunk_index").to_pylist(), + strict=True, + ) + ) + + +def iter_source_names(lancedb_dir: Path) -> Iterator[str]: + """Stream indexed source filenames batch by batch.""" + import lancedb # heavy: arrow + datafusion + + from lilbee.core.config.defaults import SOURCES_TABLE + + table = lancedb.connect(str(lancedb_dir)).open_table(SOURCES_TABLE) + for batch in table.search().select(["filename"]).to_batches(batch_size=SCAN_BATCH_ROWS): + yield from batch.column("filename").to_pylist() + + +def reservoir_sample(items: Iterable[T], k: int, rng: random.Random) -> list[T]: + """Uniform sample of up to k items in one streaming pass (Algorithm R).""" + reservoir: list[T] = [] + for seen, item in enumerate(items): + if seen < k: + reservoir.append(item) + continue + slot = rng.randrange(seen + 1) + if slot < k: + reservoir[slot] = item + return reservoir + + +def count_term_hits(rows: Iterable[ChunkRow], terms: Sequence[str]) -> dict[str, TermCounts]: + """Exact chunk and distinct-source hit counts for every term in one pass.""" + needles = {term: term.lower() for term in terms} + chunk_hits = dict.fromkeys(terms, 0) + source_hits: dict[str, set[str]] = {term: set() for term in terms} + for row in rows: + if not row.chunk: + continue + text = row.chunk.lower() + for term, needle in needles.items(): + if needle in text: + chunk_hits[term] += 1 + source_hits[term].add(row.source) + return {term: TermCounts(chunk_hits[term], len(source_hits[term])) for term in terms} + + +def scan_passages_and_heads( + rows: Iterable[ChunkRow], + *, + passage_count: int, + min_passage_chars: int, + head_sources: set[str], + rng: random.Random, +) -> PassageScan: + """One streaming pass: sample candidate passages and collect document heads. + + Passages are reservoir-sampled with oversampling, then deduplicated to at + most one per source, so the pass never holds more than a few hundred rows. + """ + reservoir: list[ChunkRow] = [] + candidate_cap = passage_count * PASSAGE_OVERSAMPLE + heads: dict[str, dict[int, str]] = {source: {} for source in head_sources} + seen = 0 + for row in rows: + if row.source in heads and row.chunk_index < HEAD_CHUNK_COUNT: + heads[row.source][row.chunk_index] = row.chunk + if not row.chunk or len(row.chunk) < min_passage_chars: + continue + if seen < candidate_cap: + reservoir.append(row) + else: + slot = rng.randrange(seen + 1) + if slot < candidate_cap: + reservoir[slot] = row + seen += 1 + + passages: list[tuple[str, str]] = [] + picked_sources: set[str] = set() + for row in reservoir: + if row.source in picked_sources: + continue + picked_sources.add(row.source) + passages.append((row.source, row.chunk)) + if len(passages) == passage_count: + break + doc_heads = { + source: "\n".join(chunk for _, chunk in sorted(indexed.items())) + for source, indexed in heads.items() + if indexed + } + return PassageScan(passages=passages, doc_heads=doc_heads) diff --git a/tests/evals/__init__.py b/tests/evals/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/evals/test_eval_answers.py b/tests/evals/test_eval_answers.py new file mode 100644 index 000000000..a9e9f6a16 --- /dev/null +++ b/tests/evals/test_eval_answers.py @@ -0,0 +1,120 @@ +"""Answer collection over HTTP: warm-up, retries, and checkpointed resume.""" + +import json + +import httpx +import pytest + +from evals.retrieval.answers import AnswerRow, answer_questions, wait_for_server +from evals.retrieval.checkpoint import load_jsonl +from evals.retrieval.questions import Question, QuestionKind + + +def _question(qid: str = "tq000") -> Question: + return Question(qid=qid, kind=QuestionKind.TOPICAL, question="Where?", source="a.txt") + + +def _client(handler) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler), base_url="http://test") + + +def test_wait_for_server_returns_once_healthy(): + states = iter([503, 503, 200]) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/health" + return httpx.Response(next(states)) + + wait_for_server("http://test", _client(handler), attempts=5, poll=0) + + +def test_wait_for_server_raises_after_budget(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("refused") + + with pytest.raises(RuntimeError, match="never became healthy"): + wait_for_server("http://test", _client(handler), attempts=2, poll=0) + + +def _ask_response(answer: str = "By the pier.") -> httpx.Response: + return httpx.Response( + 200, + json={ + "answer": answer, + "sources": [{"source": "a.txt", "content_type": "text", "chunk": "c"}], + "cited_sources": [{"source": "a.txt", "content_type": "text", "chunk": "c"}], + }, + ) + + +def test_answer_questions_records_answer_sources_and_citations(tmp_path): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/health": + return httpx.Response(200) + assert json.loads(request.content)["question"] == "Where?" + return _ask_response() + + out = tmp_path / "answers.jsonl" + rows = answer_questions( + [_question()], "http://test", "armA", out, retry_delay=0, client=_client(handler) + ) + assert len(rows) == 1 + row = AnswerRow.from_dict(load_jsonl(out)[0]) + assert row.arm == "armA" + assert row.answer == "By the pier." + assert row.sources == ["a.txt"] + assert row.cited_sources == ["a.txt"] + assert row.error is None + + +def test_answer_questions_retries_then_records_the_failure(tmp_path): + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/health": + return httpx.Response(200) + seen.append(request.url.path) + return httpx.Response(500) + + out = tmp_path / "answers.jsonl" + rows = answer_questions( + [_question()], + "http://test", + "armA", + out, + attempts=3, + retry_delay=0, + client=_client(handler), + ) + assert len(seen) == 3 + assert rows[0].error is not None + assert rows[0].answer == "" + + +def test_answer_questions_resumes_past_checkpointed_rows(tmp_path): + out = tmp_path / "answers.jsonl" + done = AnswerRow( + qid="tq000", + arm="armA", + answer="cached", + sources=[], + cited_sources=[], + seconds=0.1, + error=None, + ) + out.write_text(json.dumps(done.to_dict()) + "\n") + asked: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/health": + return httpx.Response(200) + asked.append(json.loads(request.content)["question"]) + return _ask_response("fresh") + + questions = [_question("tq000"), _question("tq001")] + rows = answer_questions( + questions, "http://test", "armA", out, retry_delay=0, client=_client(handler) + ) + assert [row.qid for row in rows] == ["tq001"] + assert asked == ["Where?"] + assert len(load_jsonl(out)) == 2 diff --git a/tests/evals/test_eval_blinding.py b/tests/evals/test_eval_blinding.py new file mode 100644 index 000000000..12e3a1152 --- /dev/null +++ b/tests/evals/test_eval_blinding.py @@ -0,0 +1,121 @@ +"""Blinding: opaque ids, a duplicated noise arm, and mechanical unblinding.""" + +import random + +from evals.retrieval.answers import AnswerRow +from evals.retrieval.blinding import build_blind_rows, unblind +from evals.retrieval.questions import Question, QuestionKind + + +def _question(qid: str, kind: QuestionKind = QuestionKind.TOPICAL) -> Question: + return Question( + qid=qid, + kind=kind, + question="Where did it happen?", + source="a.txt", + ground_passage="ground text", + ) + + +def _answer(qid: str, arm: str, answer: str = "an answer", error: str | None = None) -> AnswerRow: + return AnswerRow( + qid=qid, arm=arm, answer=answer, sources=[], cited_sources=[], seconds=0.1, error=error + ) + + +def _answers(questions, arm): + return {q.qid: _answer(q.qid, arm) for q in questions} + + +def test_noise_arm_is_judged_twice_and_other_arm_once(): + questions = [_question("tq000"), _question("tq001")] + blind = build_blind_rows( + questions, + {"A": _answers(questions, "A"), "B": _answers(questions, "B")}, + noise_arm="B", + rng=random.Random(5), + ) + assert len(blind.rows) == 6 + replicates = [(a.arm, a.replicate) for a in blind.assignments.values()] + assert replicates.count(("A", 0)) == 2 + assert replicates.count(("B", 0)) == 2 + assert replicates.count(("B", 1)) == 2 + + +def test_blind_rows_never_leak_arm_or_qid(): + questions = [_question("tq000")] + blind = build_blind_rows( + questions, + {"A": _answers(questions, "A"), "B": _answers(questions, "B")}, + noise_arm="B", + rng=random.Random(5), + ) + for row in blind.rows: + payload = f"{row.gid}{row.question}{row.ground}{row.answer}" + assert "tq000" not in payload + assert row.gid in blind.assignments + + +def test_count_questions_are_not_blind_judged(): + questions = [_question("ct000", QuestionKind.COUNT)] + blind = build_blind_rows( + questions, + {"A": _answers(questions, "A"), "B": _answers(questions, "B")}, + noise_arm="B", + rng=random.Random(5), + ) + assert blind.rows == [] + assert blind.assignments == {} + + +def test_failed_and_missing_answers_prefail_instead_of_judging(): + questions = [_question("tq000"), _question("tq001")] + answers_a = { + "tq000": _answer("tq000", "A", answer="", error="boom"), + } + blind = build_blind_rows( + questions, + {"A": answers_a, "B": _answers(questions, "B")}, + noise_arm="B", + rng=random.Random(5), + ) + assert len(blind.prefailed) == 2 + assert len(blind.rows) == 4 + for gid in blind.prefailed: + assert blind.assignments[gid].arm == "A" + + +def test_build_blind_rows_is_deterministic_for_a_seed(): + questions = [_question("tq000"), _question("tq001")] + arms = {"A": _answers(questions, "A"), "B": _answers(questions, "B")} + first = build_blind_rows(questions, arms, noise_arm="B", rng=random.Random(9)) + second = build_blind_rows(questions, arms, noise_arm="B", rng=random.Random(9)) + assert [row.gid for row in first.rows] == [row.gid for row in second.rows] + assert first.assignments == second.assignments + + +def test_unblind_regroups_by_arm_replicate_and_qid(): + questions = [_question("tq000")] + blind = build_blind_rows( + questions, + {"A": _answers(questions, "A"), "B": _answers(questions, "B")}, + noise_arm="B", + rng=random.Random(5), + ) + grades = {gid: {"faithfulness": 2, "relevance": 1, "citation": 0} for gid in blind.assignments} + regrouped = unblind(blind.assignments, grades) + assert regrouped["A"][0]["tq000"]["faithfulness"] == 2 + assert regrouped["B"][0]["tq000"]["relevance"] == 1 + assert regrouped["B"][1]["tq000"]["citation"] == 0 + + +def test_unblind_skips_unreturned_grades(): + questions = [_question("tq000")] + blind = build_blind_rows( + questions, + {"A": _answers(questions, "A"), "B": _answers(questions, "B")}, + noise_arm="B", + rng=random.Random(5), + ) + regrouped = unblind(blind.assignments, {}) + assert regrouped["A"][0] == {} diff --git a/tests/evals/test_eval_checkpoint.py b/tests/evals/test_eval_checkpoint.py new file mode 100644 index 000000000..54e437b5b --- /dev/null +++ b/tests/evals/test_eval_checkpoint.py @@ -0,0 +1,40 @@ +"""Checkpointed JSONL writing: interrupted runs resume without redoing work.""" + +import json + +from evals.retrieval.checkpoint import JsonlCheckpoint, load_jsonl + + +def test_load_jsonl_missing_file_returns_empty(tmp_path): + assert load_jsonl(tmp_path / "absent.jsonl") == [] + + +def test_load_jsonl_skips_blank_lines(tmp_path): + path = tmp_path / "rows.jsonl" + path.write_text('{"qid": "a"}\n\n{"qid": "b"}\n') + assert load_jsonl(path) == [{"qid": "a"}, {"qid": "b"}] + + +def test_append_records_and_marks_done(tmp_path): + path = tmp_path / "out.jsonl" + checkpoint = JsonlCheckpoint(path, "qid") + assert "q1" not in checkpoint + checkpoint.append({"qid": "q1", "answer": "x"}) + assert "q1" in checkpoint + assert load_jsonl(path) == [{"qid": "q1", "answer": "x"}] + + +def test_resume_skips_previously_written_keys(tmp_path): + path = tmp_path / "out.jsonl" + path.write_text(json.dumps({"qid": "q1"}) + "\n") + checkpoint = JsonlCheckpoint(path, "qid") + assert "q1" in checkpoint + assert "q2" not in checkpoint + checkpoint.append({"qid": "q2"}) + assert {row["qid"] for row in load_jsonl(path)} == {"q1", "q2"} + + +def test_done_returns_a_copy(tmp_path): + checkpoint = JsonlCheckpoint(tmp_path / "out.jsonl", "qid") + checkpoint.done.add("phantom") + assert "phantom" not in checkpoint diff --git a/tests/evals/test_eval_cli.py b/tests/evals/test_eval_cli.py new file mode 100644 index 000000000..8e1504821 --- /dev/null +++ b/tests/evals/test_eval_cli.py @@ -0,0 +1,173 @@ +"""End-to-end pipeline on synthetic fixtures via the CLI subcommands.""" + +import json + +from evals.retrieval import cli +from evals.retrieval.answers import AnswerRow +from evals.retrieval.checkpoint import load_jsonl +from evals.retrieval.questions import CountOracle, Question, QuestionKind +from evals.retrieval.scoring import ResultRowType + + +def _write_jsonl(path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + +def _fixture_files(tmp_path): + questions = [ + Question( + qid="tq000", + kind=QuestionKind.TOPICAL, + question="Where was the light?", + source="a.txt", + ground_passage="ground passage", + ), + Question( + qid="ct000", + kind=QuestionKind.COUNT, + question="How many documents mention 'lantern'?", + oracle=CountOracle(term="lantern", chunks=2, sources=1), + ), + ] + questions_path = tmp_path / "questions.jsonl" + _write_jsonl(questions_path, [q.to_dict() for q in questions]) + + def _answer(qid, arm, answer): + return AnswerRow( + qid=qid, + arm=arm, + answer=answer, + sources=["a.txt"], + cited_sources=["a.txt"], + seconds=0.2, + error=None, + ) + + answers_a = tmp_path / "answers-a.jsonl" + _write_jsonl( + answers_a, + [ + _answer("tq000", "old", "On the hill.").to_dict(), + _answer("ct000", "old", "2 chunks in 1 document.").to_dict(), + ], + ) + answers_b = tmp_path / "answers-b.jsonl" + _write_jsonl( + answers_b, + [ + _answer("tq000", "new", "On the headland.").to_dict(), + _answer("ct000", "new", "2 chunks in 1 document.").to_dict(), + ], + ) + return questions_path, answers_a, answers_b + + +def test_judge_score_report_pipeline(tmp_path, monkeypatch): + questions_path, answers_a, answers_b = _fixture_files(tmp_path) + work_dir = tmp_path / "work" + + fixed_grade = '{"faithfulness": 2, "relevance": 2, "citation": 1}' + monkeypatch.setattr(cli, "judge_chat_fn", lambda: lambda _prompt: fixed_grade) + monkeypatch.setattr(cli, "warm_chat", lambda chat: None) + + exit_code = cli.main( + [ + "judge", + "--questions", + str(questions_path), + "--answers-a", + str(answers_a), + "--answers-b", + str(answers_b), + "--work-dir", + str(work_dir), + "--seed", + "3", + ] + ) + assert exit_code == 0 + assert len(load_jsonl(work_dir / "grades.jsonl")) == 3 + gid_map = json.loads((work_dir / "gid_map.json").read_text()) + assert len(gid_map) == 3 + + results_path = tmp_path / "results.jsonl" + exit_code = cli.main( + [ + "score", + "--questions", + str(questions_path), + "--answers-a", + str(answers_a), + "--answers-b", + str(answers_b), + "--work-dir", + str(work_dir), + "--out", + str(results_path), + ] + ) + assert exit_code == 0 + rows = load_jsonl(results_path) + summary = rows[-1] + assert summary["row_type"] == ResultRowType.SUMMARY + assert set(summary["arms"]) == {"old", "new"} + assert summary["noise_floor"] == 0.0 + assert summary["arms"]["new"]["count_pass"] == [1, 1] + + report_path = tmp_path / "report.md" + exit_code = cli.main(["report", "--results", str(results_path), "--out", str(report_path)]) + assert exit_code == 0 + text = report_path.read_text() + assert "old" in text + assert "new" in text + assert "noise floor" in text.lower() + + +def test_judge_rejects_duplicate_arm_labels(tmp_path, monkeypatch): + questions_path, answers_a, _ = _fixture_files(tmp_path) + monkeypatch.setattr(cli, "judge_chat_fn", lambda: lambda _prompt: "{}") + monkeypatch.setattr(cli, "warm_chat", lambda chat: None) + exit_code = cli.main( + [ + "judge", + "--questions", + str(questions_path), + "--answers-a", + str(answers_a), + "--answers-b", + str(answers_a), + "--work-dir", + str(tmp_path / "w"), + ] + ) + assert exit_code == 1 + + +def test_answer_subcommand_writes_checkpointed_answers(tmp_path, monkeypatch): + questions_path, _, _ = _fixture_files(tmp_path) + out = tmp_path / "answers.jsonl" + + import httpx + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/health": + return httpx.Response(200) + return httpx.Response(200, json={"answer": "ok", "sources": [], "cited_sources": []}) + + client = httpx.Client(transport=httpx.MockTransport(handler), base_url="http://test") + monkeypatch.setattr(cli, "make_http_client", lambda: client) + exit_code = cli.main( + [ + "answer", + "--questions", + str(questions_path), + "--base-url", + "http://test", + "--arm", + "old", + "--out", + str(out), + ] + ) + assert exit_code == 0 + assert len(load_jsonl(out)) == 2 diff --git a/tests/evals/test_eval_judging.py b/tests/evals/test_eval_judging.py new file mode 100644 index 000000000..cb5735e84 --- /dev/null +++ b/tests/evals/test_eval_judging.py @@ -0,0 +1,69 @@ +"""Judge plumbing: grade parsing, clamping, and checkpointed judging.""" + +from evals.retrieval.blinding import BlindRow +from evals.retrieval.checkpoint import load_jsonl +from evals.retrieval.judging import judge_rows, parse_grade + + +def _row(gid: str) -> BlindRow: + return BlindRow(gid=gid, question="Where?", source="a.txt", ground="g", answer="a") + + +def test_parse_grade_extracts_json_from_noise(): + text = 'Sure. {"faithfulness": 2, "relevance": 1, "citation": 0} done.' + assert parse_grade(text) == {"faithfulness": 2, "relevance": 1, "citation": 0} + + +def test_parse_grade_clamps_out_of_range_scores(): + text = '{"faithfulness": 9, "relevance": -3, "citation": 1}' + assert parse_grade(text) == {"faithfulness": 2, "relevance": 0, "citation": 1} + + +def test_parse_grade_rejects_missing_dimensions_and_garbage(): + assert parse_grade('{"faithfulness": 2}') is None + assert parse_grade("no json here") is None + assert parse_grade('{"faithfulness": "high", "relevance": 1, "citation": 1}') is None + + +def test_judge_rows_grades_and_checkpoints(tmp_path): + out = tmp_path / "grades.jsonl" + grades = judge_rows( + [_row("g1"), _row("g2")], + lambda _prompt: '{"faithfulness": 2, "relevance": 2, "citation": 1}', + out, + retry_delay=0, + ) + assert set(grades) == {"g1", "g2"} + assert len(load_jsonl(out)) == 2 + + +def test_judge_rows_resumes_and_keeps_existing_grades(tmp_path): + out = tmp_path / "grades.jsonl" + judge_rows( + [_row("g1")], + lambda _prompt: '{"faithfulness": 1, "relevance": 1, "citation": 1}', + out, + retry_delay=0, + ) + judged: list[str] = [] + + def chat(prompt: str) -> str: + judged.append(prompt) + return '{"faithfulness": 2, "relevance": 2, "citation": 2}' + + grades = judge_rows([_row("g1"), _row("g2")], chat, out, retry_delay=0) + assert len(judged) == 1 + assert grades["g1"] == {"faithfulness": 1, "relevance": 1, "citation": 1} + assert grades["g2"] == {"faithfulness": 2, "relevance": 2, "citation": 2} + + +def test_judge_rows_retries_and_skips_unparseable_rows(tmp_path): + calls: list[str] = [] + + def bad_judge(prompt: str) -> str: + calls.append(prompt) + return "not json" + + grades = judge_rows([_row("g1")], bad_judge, tmp_path / "g.jsonl", attempts=2, retry_delay=0) + assert grades == {} + assert len(calls) == 2 diff --git a/tests/evals/test_eval_questions.py b/tests/evals/test_eval_questions.py new file mode 100644 index 000000000..262c8df09 --- /dev/null +++ b/tests/evals/test_eval_questions.py @@ -0,0 +1,128 @@ +"""Question generation: topical authoring, known-item, and count oracles.""" + +import random + +import pytest + +from evals.retrieval import questions as questions_mod +from evals.retrieval.questions import ( + CountOracle, + Question, + QuestionKind, + author_topical, + build_questions, + parse_question, + sample_terms, +) +from evals.retrieval.store_scan import ChunkRow + + +def test_parse_question_takes_first_line_and_strips_quotes(): + assert parse_question('"Where did the fleet anchor?"\nextra') == "Where did the fleet anchor?" + + +@pytest.mark.parametrize("text", ["", "not a question", "Why?", "Too short?"]) +def test_parse_question_rejects_non_questions(text): + assert parse_question(text) is None + + +def test_sample_terms_picks_mid_frequency_terms(): + passages = [(f"s{i}.txt", "common words everywhere") for i in range(10)] + passages += [("s0.txt", "unique zebra sighting"), ("s1.txt", "zebra again here")] + terms = sample_terms(passages, 5, random.Random(1)) + assert "common" not in terms + assert "zebra" in terms + + +def test_sample_terms_respects_count(): + passages = [(f"s{i}.txt", f"word{i}alpha word{i}beta shared") for i in range(20)] + assert len(sample_terms(passages, 3, random.Random(1))) <= 3 + + +def test_author_topical_builds_questions_with_ground_passages(): + picks = [("a.txt", "p" * 500), ("b.txt", "q" * 500)] + calls: list[str] = [] + + def chat(prompt: str) -> str: + calls.append(prompt) + return "What color was the harbor light?" + + authored = author_topical(picks, chat, retry_delay=0) + assert len(authored) == 2 + assert authored[0].kind is QuestionKind.TOPICAL + assert authored[0].source == "a.txt" + assert authored[0].ground_passage == "p" * 500 + assert "p" * 100 in calls[0] + + +def test_author_topical_retries_then_skips_hard_failures(): + attempts: list[int] = [] + + def flaky_chat(prompt: str) -> str: + attempts.append(1) + raise RuntimeError("model busy") + + authored = author_topical([("a.txt", "p" * 500)], flaky_chat, attempts=3, retry_delay=0) + assert authored == [] + assert len(attempts) == 3 + + +def test_author_topical_drops_malformed_questions(): + authored = author_topical([("a.txt", "p" * 500)], lambda _prompt: "no.", retry_delay=0) + assert authored == [] + + +def test_question_round_trips_through_dict(): + question = Question( + qid="ct000", + kind=QuestionKind.COUNT, + question="How many documents mention 'lantern'?", + oracle=CountOracle(term="lantern", chunks=4, sources=2), + ) + assert Question.from_dict(question.to_dict()) == question + + +def test_build_questions_assembles_all_three_kinds(monkeypatch): + rare_words = ["lantern", "beacon", "harbor", "vessel", "anchor", "compass"] + chunk_rows = [ChunkRow(f"doc{i}.txt", f"{rare_words[i]} " + "x" * 450, 0) for i in range(6)] + monkeypatch.setattr(questions_mod, "iter_chunks", lambda _dir: iter(chunk_rows)) + monkeypatch.setattr( + questions_mod, "iter_source_names", lambda _dir: iter(f"doc{i}.txt" for i in range(6)) + ) + + def chat(prompt: str) -> str: + return "Which lantern hung over the pier that night?" + + built = build_questions(lancedb_dir=None, chat=chat, topical=5, known_item=2, count=1, seed=11) + kinds = [q.kind for q in built] + assert kinds.count(QuestionKind.TOPICAL) == 5 + assert kinds.count(QuestionKind.KNOWN_ITEM) == 2 + assert kinds.count(QuestionKind.COUNT) == 1 + count_question = next(q for q in built if q.kind is QuestionKind.COUNT) + assert count_question.oracle is not None + assert count_question.oracle.chunks >= 1 + known = next(q for q in built if q.kind is QuestionKind.KNOWN_ITEM) + assert known.source in known.question + assert known.ground_passage + assert len({q.qid for q in built}) == len(built) + + +def test_build_questions_skips_the_count_scan_when_no_terms_qualify(monkeypatch): + scans: list[int] = [] + + def counted_iter_chunks(_dir): + scans.append(1) + return iter([ChunkRow("doc0.txt", "y" * 450, 0)]) + + monkeypatch.setattr(questions_mod, "iter_chunks", counted_iter_chunks) + monkeypatch.setattr(questions_mod, "iter_source_names", lambda _dir: iter(["doc0.txt"])) + built = build_questions( + lancedb_dir=None, + chat=lambda _p: "Where did the launch dock?", + topical=1, + known_item=1, + count=4, + seed=1, + ) + assert all(q.kind is not QuestionKind.COUNT for q in built) + assert len(scans) == 1 diff --git a/tests/evals/test_eval_report.py b/tests/evals/test_eval_report.py new file mode 100644 index 000000000..fb500af4a --- /dev/null +++ b/tests/evals/test_eval_report.py @@ -0,0 +1,49 @@ +"""Markdown report rendering from results rows.""" + +from evals.retrieval.report import render_report +from evals.retrieval.scoring import ResultRowType + + +def _summary(noise: float = 0.1) -> dict: + return { + "row_type": ResultRowType.SUMMARY, + "noise_floor": noise, + "judged": 4, + "arms": { + "A": { + "means": {"faithfulness": 1.5, "relevance": 1.8, "citation": 1.0}, + "count_pass": [1, 2], + "known_item_pass": [2, 2], + "errors": 0, + }, + "B": { + "means": {"faithfulness": 1.55, "relevance": 1.2, "citation": 1.0}, + "count_pass": [2, 2], + "known_item_pass": [1, 2], + "errors": 1, + }, + }, + } + + +def test_render_report_flags_deltas_within_the_noise_floor(): + report = render_report([_summary()]) + assert "| faithfulness (0-2) | 1.5 | 1.55 | +0.05 (within noise) |" in report + assert "| relevance (0-2) | 1.8 | 1.2 | -0.6 |" in report + + +def test_render_report_includes_exact_truth_and_failures(): + report = render_report([_summary()]) + assert "count questions | 1/2 | 2/2" in report + assert "known-item citation | 2/2 | 1/2" in report + assert "hard failures | 0 | 1" in report + assert "0.1" in report + + +def test_render_report_requires_a_summary_row(): + try: + render_report([{"row_type": ResultRowType.QUESTION}]) + except ValueError as exc: + assert "summary" in str(exc) + else: + raise AssertionError("expected ValueError") diff --git a/tests/evals/test_eval_scoring.py b/tests/evals/test_eval_scoring.py new file mode 100644 index 000000000..69cbc6b70 --- /dev/null +++ b/tests/evals/test_eval_scoring.py @@ -0,0 +1,158 @@ +"""Scoring: noise floor math, per-dimension means, and exact-truth checks.""" + +import pytest + +from evals.retrieval.answers import AnswerRow +from evals.retrieval.questions import CountOracle, Question, QuestionKind +from evals.retrieval.scoring import ( + ResultRowType, + build_results, + count_question_pass, + dimension_means, + known_item_pass, + noise_floor, +) + + +def _grades(f: int, r: int, c: int) -> dict[str, int]: + return {"faithfulness": f, "relevance": r, "citation": c} + + +def test_noise_floor_is_mean_absolute_disagreement_per_dimension(): + rep0 = {"q1": _grades(2, 2, 2), "q2": _grades(1, 1, 1)} + rep1 = {"q1": _grades(2, 1, 0), "q2": _grades(1, 1, 1)} + assert noise_floor(rep0, rep1) == pytest.approx(0.5) + + +def test_noise_floor_ignores_questions_missing_from_either_replicate(): + rep0 = {"q1": _grades(2, 2, 2), "only0": _grades(0, 0, 0)} + rep1 = {"q1": _grades(2, 2, 2), "only1": _grades(2, 2, 2)} + assert noise_floor(rep0, rep1) == 0.0 + + +def test_noise_floor_with_no_shared_questions_is_zero(): + assert noise_floor({}, {}) == 0.0 + + +def test_dimension_means(): + means = dimension_means({"q1": _grades(2, 2, 0), "q2": _grades(0, 1, 1)}) + assert means == {"faithfulness": 1.0, "relevance": 1.5, "citation": 0.5} + assert dimension_means({}) == {"faithfulness": 0.0, "relevance": 0.0, "citation": 0.0} + + +def test_count_question_pass_requires_both_oracle_numbers(): + oracle = CountOracle(term="lantern", chunks=14, sources=3) + assert count_question_pass(oracle, "It appears in 14 chunks across 3 documents.") + assert not count_question_pass(oracle, "It appears in 14 chunks.") + assert not count_question_pass(oracle, "It appears 15 times in 3 documents.") + + +def test_known_item_pass_requires_the_expected_citation(): + row = AnswerRow( + qid="ki000", + arm="A", + answer="It is a book.", + sources=["a.txt", "b.txt"], + cited_sources=["a.txt"], + seconds=0.1, + error=None, + ) + assert known_item_pass("a.txt", row) + assert not known_item_pass("b.txt", row) + + +def _pipeline_fixture(): + questions = [ + Question( + qid="tq000", + kind=QuestionKind.TOPICAL, + question="Where?", + source="a.txt", + ground_passage="ground", + ), + Question( + qid="ki000", + kind=QuestionKind.KNOWN_ITEM, + question="What is a.txt about?", + source="a.txt", + ground_passage="head", + ), + Question( + qid="ct000", + kind=QuestionKind.COUNT, + question="How many documents mention 'lantern'?", + oracle=CountOracle(term="lantern", chunks=2, sources=1), + ), + ] + + def _answer(qid, arm, answer, cited): + return AnswerRow( + qid=qid, + arm=arm, + answer=answer, + sources=cited, + cited_sources=cited, + seconds=0.2, + error=None, + ) + + answers = { + "A": { + "tq000": _answer("tq000", "A", "By the pier.", ["a.txt"]), + "ki000": _answer("ki000", "A", "A harbor log.", ["a.txt"]), + "ct000": _answer("ct000", "A", "2 chunks in 1 document.", []), + }, + "B": { + "tq000": _answer("tq000", "B", "Near the docks.", ["a.txt"]), + "ki000": _answer("ki000", "B", "A harbor log.", ["b.txt"]), + "ct000": _answer("ct000", "B", "About 7 mentions.", []), + }, + } + unblinded = { + "A": {0: {"tq000": _grades(2, 2, 2), "ki000": _grades(2, 2, 2)}}, + "B": { + 0: {"tq000": _grades(1, 1, 1), "ki000": _grades(1, 2, 1)}, + 1: {"tq000": _grades(1, 1, 0), "ki000": _grades(1, 2, 1)}, + }, + } + return questions, answers, unblinded + + +def test_build_results_emits_question_rows_and_a_summary(): + questions, answers, unblinded = _pipeline_fixture() + rows = build_results(questions, answers, unblinded, noise_arm="B") + summary = rows[-1] + assert summary["row_type"] == ResultRowType.SUMMARY + assert summary["noise_floor"] == pytest.approx(1 / 6, abs=1e-3) + arm_a = summary["arms"]["A"] + assert arm_a["means"]["faithfulness"] == 2.0 + assert arm_a["count_pass"] == [1, 1] + assert arm_a["known_item_pass"] == [1, 1] + assert arm_a["errors"] == 0 + arm_b = summary["arms"]["B"] + assert arm_b["count_pass"] == [0, 1] + assert arm_b["known_item_pass"] == [0, 1] + + question_rows = [r for r in rows if r["row_type"] == ResultRowType.QUESTION] + assert len(question_rows) == 6 + topical_a = next(r for r in question_rows if r["qid"] == "tq000" and r["arm"] == "A") + assert topical_a["grades"] == _grades(2, 2, 2) + count_b = next(r for r in question_rows if r["qid"] == "ct000" and r["arm"] == "B") + assert count_b["exact_pass"] is False + + +def test_build_results_counts_hard_failures(): + questions, answers, unblinded = _pipeline_fixture() + answers["B"]["tq000"] = AnswerRow( + qid="tq000", + arm="B", + answer="", + sources=[], + cited_sources=[], + seconds=0.0, + error="ConnectError: refused", + ) + del answers["B"]["ct000"] + rows = build_results(questions, answers, unblinded, noise_arm="B") + summary = rows[-1] + assert summary["arms"]["B"]["errors"] == 2 diff --git a/tests/evals/test_eval_store_scan.py b/tests/evals/test_eval_store_scan.py new file mode 100644 index 000000000..f6e5ab466 --- /dev/null +++ b/tests/evals/test_eval_store_scan.py @@ -0,0 +1,85 @@ +"""Streaming scan helpers: sampling and counting without materializing an index.""" + +import random + +from evals.retrieval.store_scan import ( + ChunkRow, + count_term_hits, + reservoir_sample, + scan_passages_and_heads, +) + + +def _rows(spec: dict[str, list[str]]) -> list[ChunkRow]: + return [ + ChunkRow(source, chunk, index) + for source, chunks in spec.items() + for index, chunk in enumerate(chunks) + ] + + +def test_reservoir_sample_returns_all_when_fewer_than_k(): + assert sorted(reservoir_sample(iter("abc"), 10, random.Random(1))) == ["a", "b", "c"] + + +def test_reservoir_sample_is_deterministic_for_a_seed(): + items = list(range(1000)) + first = reservoir_sample(iter(items), 5, random.Random(7)) + second = reservoir_sample(iter(items), 5, random.Random(7)) + assert first == second + assert len(first) == 5 + assert all(item in items for item in first) + + +def test_reservoir_sample_draws_from_the_whole_stream(): + samples = reservoir_sample(iter(range(10_000)), 50, random.Random(3)) + assert any(item > 5000 for item in samples) + + +def test_count_term_hits_counts_chunks_and_distinct_sources(): + rows = _rows( + { + "a.txt": ["the whale swam", "deep water"], + "b.txt": ["a whale breached", "another Whale sighting"], + "c.txt": ["nothing relevant"], + } + ) + counts = count_term_hits(rows, ["whale", "water", "absent"]) + assert counts["whale"].chunks == 3 + assert counts["whale"].sources == 2 + assert counts["water"] == (1, 1) + assert counts["absent"] == (0, 0) + + +def test_scan_collects_one_passage_per_source_and_respects_min_chars(): + long_a = "a" * 500 + long_b = "b" * 500 + rows = _rows({"a.txt": [long_a, long_a], "b.txt": ["short", long_b]}) + scan = scan_passages_and_heads( + rows, passage_count=5, min_passage_chars=400, head_sources=set(), rng=random.Random(1) + ) + assert sorted(source for source, _ in scan.passages) == ["a.txt", "b.txt"] + assert all(len(passage) >= 400 for _, passage in scan.passages) + + +def test_scan_caps_passages_at_requested_count(): + rows = _rows({f"s{i}.txt": ["x" * 450] for i in range(30)}) + scan = scan_passages_and_heads( + rows, passage_count=4, min_passage_chars=400, head_sources=set(), rng=random.Random(2) + ) + assert len(scan.passages) == 4 + assert len({source for source, _ in scan.passages}) == 4 + + +def test_scan_builds_doc_heads_in_chunk_order(): + rows = [ + ChunkRow("k.txt", "third", 2), + ChunkRow("k.txt", "first", 0), + ChunkRow("k.txt", "second", 1), + ChunkRow("k.txt", "far past the head", 9), + ChunkRow("other.txt", "ignored", 0), + ] + scan = scan_passages_and_heads( + rows, passage_count=0, min_passage_chars=400, head_sources={"k.txt"}, rng=random.Random(1) + ) + assert scan.doc_heads == {"k.txt": "first\nsecond\nthird"}