From acf06ed13effed8a012e2c58e2addbd853b16050 Mon Sep 17 00:00:00 2001 From: Harshit Sharma <66710144+harshitethic@users.noreply.github.com> Date: Thu, 5 Nov 2026 06:09:42 +0530 Subject: [PATCH 1/3] Add an offline top-file retrieval evaluation using the catalog demo --- examples/evaluate_retrieval.py | 56 ++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 examples/evaluate_retrieval.py diff --git a/examples/evaluate_retrieval.py b/examples/evaluate_retrieval.py new file mode 100644 index 0000000..bc9c51b --- /dev/null +++ b/examples/evaluate_retrieval.py @@ -0,0 +1,56 @@ +"""Evaluate the first source file selected by Ask AI on the fictional catalog.""" +import argparse +import json +from pathlib import Path +from tempfile import TemporaryDirectory + +from create_demo_zip import DEMO_FILES +from services.analyzer import build_context + + +# Evidence strings distinguish a definition from an import of the same symbol. +CASES = [ + ("Where is find_title defined?", "src/catalog.py", "def find_title("), + ("Where is test_query defined?", "tests/test_catalog.py", "def test_query("), + ("catalog_handler", "src/api.py", "def catalog_handler("), + ("Which Python version does the project require?", "pyproject.toml", "requires-python"), + ("Where is SKU looked up?", "src/catalog.py", 'product.get("sku")'), +] + + +def evaluate(root): + results = [] + for question, expected, evidence in CASES: + context = build_context(root, question, max_files=1) + header, _, excerpt = context.lstrip("\n").partition("\n") + selected = header[10:-4] if header.startswith("--- FILE: ") and header.endswith(" ---") else None + results.append({ + "question": question, "expected": expected, "selected": selected, + "passed": selected == expected and evidence in excerpt, + }) + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--json", action="store_true", help="print machine-readable case results") + args = parser.parse_args() + with TemporaryDirectory() as directory: + root = Path(directory) + for name, contents in DEMO_FILES.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + results = evaluate(root) + + if args.json: + print(json.dumps(results, indent=2)) + else: + for row in results: + status = "PASS" if row["passed"] else "FAIL" + print(f"{status} {row['question']} expected={row['expected']} selected={row['selected']}") + print(f"{sum(row['passed'] for row in results)}/{len(results)} cases passed") + + +if __name__ == "__main__": + main() From 458b21375003bf26d24947a17f0977042bdd4b08 Mon Sep 17 00:00:00 2001 From: Harshit Sharma <66710144+harshitethic@users.noreply.github.com> Date: Thu, 5 Nov 2026 06:09:42 +0530 Subject: [PATCH 2/3] Measure abstention for absent symbols and expose a regression gate --- examples/evaluate_retrieval.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/evaluate_retrieval.py b/examples/evaluate_retrieval.py index bc9c51b..802ca8a 100644 --- a/examples/evaluate_retrieval.py +++ b/examples/evaluate_retrieval.py @@ -14,6 +14,7 @@ ("Where is test_query defined?", "tests/test_catalog.py", "def test_query("), ("catalog_handler", "src/api.py", "def catalog_handler("), ("Which Python version does the project require?", "pyproject.toml", "requires-python"), + ("Where is the missing_symbol defined?", None, None), ("Where is SKU looked up?", "src/catalog.py", 'product.get("sku")'), ] @@ -26,7 +27,7 @@ def evaluate(root): selected = header[10:-4] if header.startswith("--- FILE: ") and header.endswith(" ---") else None results.append({ "question": question, "expected": expected, "selected": selected, - "passed": selected == expected and evidence in excerpt, + "passed": selected == expected and (evidence is None or evidence in excerpt), }) return results @@ -34,6 +35,7 @@ def evaluate(root): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--json", action="store_true", help="print machine-readable case results") + parser.add_argument("--check", action="store_true", help="exit with status 1 on a retrieval miss") args = parser.parse_args() with TemporaryDirectory() as directory: root = Path(directory) @@ -51,6 +53,8 @@ def main(): print(f"{status} {row['question']} expected={row['expected']} selected={row['selected']}") print(f"{sum(row['passed'] for row in results)}/{len(results)} cases passed") + return int(args.check and not all(row["passed"] for row in results)) + if __name__ == "__main__": - main() + raise SystemExit(main()) From d9cad9b209db2ad981a38d603f38e8a0afac0d6a Mon Sep 17 00:00:00 2001 From: Harshit Sharma <66710144+harshitethic@users.noreply.github.com> Date: Thu, 5 Nov 2026 06:09:42 +0530 Subject: [PATCH 3/3] Document how to reproduce and interpret the offline retrieval evaluation --- docs/independent-evaluation.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/independent-evaluation.md b/docs/independent-evaluation.md index 1a93bf5..86d2a60 100644 --- a/docs/independent-evaluation.md +++ b/docs/independent-evaluation.md @@ -35,3 +35,20 @@ Open an [issue](https://github.com/harshitethic/RepoPilot/issues) with commit SH - [ ] Dependency upgrade compatibility independently verified before merging - [ ] Release screenshots label mocks and scripted browser fixtures accurately - [ ] External review/adoption claims cite independent, reproducible records + +## Check the evidence sent to Ask AI + +From the checkout root, use the backend Python environment to run the catalog +retrieval cases. Each question gets one file of context, which makes a wrong +first choice visible. A positive case checks both the source path and a literal +piece of evidence inside the excerpt. The missing-symbol case expects no file. + +~~~bash +PYTHONPATH=backend python examples/evaluate_retrieval.py +~~~ + +In PowerShell, set `$env:PYTHONPATH = "backend"` before running the Python +command. Use `--json` for per-question records and `--check` for a nonzero exit +status when any case fails. The script creates the fictional catalog in a +temporary directory and exercises retrieval without starting a server or +calling Ollama. Results apply to this small fixture.