Add an offline evaluation for Ask AI source selection

docs/independent-evaluation.md

@@ -35,3 +35,20 @@ Open an [issue](https://github.com/harshitethic/RepoPilot/issues) with commit SH
 - [ ] Dependency upgrade compatibility independently verified before merging
 - [ ] Release screenshots label mocks and scripted browser fixtures accurately
 - [ ] External review/adoption claims cite independent, reproducible records
+
+## Check the evidence sent to Ask AI
+
+From the checkout root, use the backend Python environment to run the catalog
+retrieval cases. Each question gets one file of context, which makes a wrong
+first choice visible. A positive case checks both the source path and a literal
+piece of evidence inside the excerpt. The missing-symbol case expects no file.
+
+~~~bash
+PYTHONPATH=backend python examples/evaluate_retrieval.py
+~~~
+
+In PowerShell, set `$env:PYTHONPATH = "backend"` before running the Python
+command. Use `--json` for per-question records and `--check` for a nonzero exit
+status when any case fails. The script creates the fictional catalog in a
+temporary directory and exercises retrieval without starting a server or
+calling Ollama. Results apply to this small fixture.

examples/evaluate_retrieval.py

@@ -0,0 +1,60 @@
+"""Evaluate the first source file selected by Ask AI on the fictional catalog."""
+import argparse
+import json
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from create_demo_zip import DEMO_FILES
+from services.analyzer import build_context
+
+
+# Evidence strings distinguish a definition from an import of the same symbol.
+CASES = [
+    ("Where is find_title defined?", "src/catalog.py", "def find_title("),
+    ("Where is test_query defined?", "tests/test_catalog.py", "def test_query("),
+    ("catalog_handler", "src/api.py", "def catalog_handler("),
+    ("Which Python version does the project require?", "pyproject.toml", "requires-python"),
+    ("Where is the missing_symbol defined?", None, None),
+    ("Where is SKU looked up?", "src/catalog.py", 'product.get("sku")'),
+]
+
+
+def evaluate(root):
+    results = []
+    for question, expected, evidence in CASES:
+        context = build_context(root, question, max_files=1)
+        header, _, excerpt = context.lstrip("\n").partition("\n")
+        selected = header[10:-4] if header.startswith("--- FILE: ") and header.endswith(" ---") else None
+        results.append({
+            "question": question, "expected": expected, "selected": selected,
+            "passed": selected == expected and (evidence is None or evidence in excerpt),
+        })
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--json", action="store_true", help="print machine-readable case results")
+    parser.add_argument("--check", action="store_true", help="exit with status 1 on a retrieval miss")
+    args = parser.parse_args()
+    with TemporaryDirectory() as directory:
+        root = Path(directory)
+        for name, contents in DEMO_FILES.items():
+            path = root / name
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(contents, encoding="utf-8")
+        results = evaluate(root)
+
+    if args.json:
+        print(json.dumps(results, indent=2))
+    else:
+        for row in results:
+            status = "PASS" if row["passed"] else "FAIL"
+            print(f"{status}  {row['question']}  expected={row['expected']}  selected={row['selected']}")
+        print(f"{sum(row['passed'] for row in results)}/{len(results)} cases passed")
+
+    return int(args.check and not all(row["passed"] for row in results))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())