diff --git a/HF_RELEASE_CHECKLIST.md b/HF_RELEASE_CHECKLIST.md index f28febe..1d2d6b1 100644 --- a/HF_RELEASE_CHECKLIST.md +++ b/HF_RELEASE_CHECKLIST.md @@ -1,7 +1,7 @@ # HUGGING FACE & OPEN-SOURCE RELEASE CHECKLIST: REPONEEDLE > [!NOTE] -> **Historical record (2026-10-28) of the v0.1.0 release checklist.** It has not been updated for later versions. The Hugging Face Space in `hf_artifacts/space/` bundles the 1.1.0 engine, in which the uncalibrated abstention heuristic is always on. +> **Historical record (2026-10-28) of the v0.1.0 release checklist.** It has not been updated for later versions. When this note was added (2026-11-10), the Hugging Face Space in `hf_artifacts/space/` bundled the 1.1.0 engine, in which the uncalibrated abstention heuristic is always on. The Space bundle has since been synchronised with RepoNeedle 1.2.0, where the heuristic is opt-in; see `docs/STATIC_PLAYGROUND_DEPLOYMENT.md`. **Version**: `0.1.0` (Research Preview) **Date**: 2026-10-28 diff --git a/api.py b/api.py index 2e53f47..a81b943 100644 --- a/api.py +++ b/api.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +import logging import sys import time from functools import lru_cache @@ -22,19 +23,24 @@ # app.py handles Gradio as optional; it can be imported without installing Gradio. from app import CORPORA # noqa: E402 +import reponeedle # noqa: E402 from reponeedle.engine import RepoNeedleEngine # noqa: E402 from reponeedle.models import Query, TaskType # noqa: E402 from reponeedle.core.lexical import BM25Index # noqa: E402 from reponeedle.core.graph import RepoGraph # noqa: E402 +# Single source of truth: the version of the bundled engine this API actually imports. +ENGINE_VERSION = reponeedle.__version__ + +logger = logging.getLogger("reponeedle.demo_api") app = FastAPI( title="RepoNeedle Demo API", description=( - "Browser-facing API for the RepoNeedle V1.1.0 demonstration. " + f"Browser-facing API for the RepoNeedle {ENGINE_VERSION} demonstration. " "Uses embedded sample snapshots of Flask and Requests." ), - version="1.1.0", + version=ENGINE_VERSION, ) # Hugging Face Static Spaces are served from a different origin. @@ -56,6 +62,9 @@ class RetrieveRequest(BaseModel): budget_tokens: int = Field(default=4000, ge=500, le=8000) seed_files: list[str] = Field(default_factory=list, max_length=10) query: str = Field(min_length=2, max_length=3000) + # Experimental, uncalibrated out-of-scope abstention heuristic. Off unless the caller opts in + # (the RepoNeedle 1.2.0 default). + enable_experimental_abstention: bool = False @lru_cache(maxsize=2) @@ -78,7 +87,7 @@ def get_baselines(repo_name: str): def root(): return { "name": "RepoNeedle Demo API", - "version": "1.1.0", + "version": ENGINE_VERSION, "health": "/health", "docs": "/docs", "disclosure": "Embedded demonstration snapshots only; not full upstream repositories.", @@ -87,7 +96,7 @@ def root(): @app.get("/health") def health(): - return {"status": "ok", "service": "reponeedle-demo-api", "version": "1.1.0"} + return {"status": "ok", "service": "reponeedle-demo-api", "version": ENGINE_VERSION} @app.post("/api/retrieve") @@ -112,7 +121,11 @@ def retrieve_context(payload: RetrieveRequest): ) started = time.perf_counter() - result = engine.retrieve(query=query_obj, ranker=payload.ranker) + result = engine.retrieve( + query=query_obj, + ranker=payload.ranker, + enable_experimental_abstention=payload.enable_experimental_abstention, + ) elapsed_ms = (time.perf_counter() - started) * 1000 selected = [] @@ -152,6 +165,14 @@ def retrieve_context(payload: RetrieveRequest): else: ranker_note = "Legacy ranking is active (the backwards-compatible default ranker)." + diag = result.abstention + abstention = { + "enabled": bool(diag.enabled) if diag else bool(payload.enable_experimental_abstention), + "status": diag.status if diag else ("evaluated" if payload.enable_experimental_abstention else "disabled"), + "experimental": True, + "calibrated": False, + } + return { "query": query_text, "repository": payload.repo_name, @@ -169,9 +190,12 @@ def retrieve_context(payload: RetrieveRequest): "selected_files": selected, "comparison": comparison, "preview": selected[0] if selected else None, + "abstention": abstention, } except HTTPException: raise except Exception as exc: - # Do not return stack traces or local filesystem details to public callers. - raise HTTPException(status_code=500, detail=f"Retrieval failed: {type(exc).__name__}: {exc}") from exc + # Do not return exception messages, stack traces or local filesystem details to public + # callers; keep the details in the server log only. + logger.exception("Demo retrieval failed") + raise HTTPException(status_code=500, detail="Retrieval failed due to an internal error.") from exc diff --git a/docs/STATIC_PLAYGROUND_DEPLOYMENT.md b/docs/STATIC_PLAYGROUND_DEPLOYMENT.md index f46d14a..b288034 100644 --- a/docs/STATIC_PLAYGROUND_DEPLOYMENT.md +++ b/docs/STATIC_PLAYGROUND_DEPLOYMENT.md @@ -1,16 +1,62 @@ -# RepoNeedle V1.1.0 public demo deployment +# RepoNeedle 1.2.0 public demo deployment -This document describes the public demo deployment. It does not change the installation path for the core Python package. +This document describes the public demo deployment. It does not change the installation path for the core Python package (`pip install reponeedle`). ## Components | Component | Location | Purpose | |---|---|---| | Core engine | `reponeedle/` | CPU-first retrieval library and CLI | -| Static frontend | `hf_artifacts/static_space/` | Browser UI hosted by Hugging Face Static Spaces | -| Demo API | `api.py` | FastAPI wrapper exposing retrieval for the embedded demo corpora | -| Demo snapshots and bundled UI engine | `hf_artifacts/space/` | Small Flask and Requests snapshots and synchronized RepoNeedle package used by the API | -| Dataset | `hf_artifacts/dataset/` | Lightweight synthetic demonstration benchmark | +| Static frontend | `hf_artifacts/static_space/` | Browser UI hosted by a Hugging Face **Static** Space | +| Demo API | `api.py` | FastAPI wrapper exposing retrieval for the embedded demo corpora (hosted on Render) | +| Demo snapshots and bundled engine | `hf_artifacts/space/` | Small Flask and Requests snapshots (`app.py`) and the bundled `reponeedle` package used by the API and the Gradio Space | +| Gradio Space | `hf_artifacts/space/` | Self-contained Gradio Playground that runs the bundled engine in-process | +| Dataset | `hf_artifacts/dataset/` | Lightweight synthetic demonstration benchmark (unchanged in 1.2.0) | + +### Hugging Face repositories + +| Repository | Type | Source in this repository | +|---|---|---| +| `Krishikax/RepoNeedle-Playground` | Space, **static** SDK (frontend) | README badge; `hf_artifacts/static_space/` | +| `RepoNeedle/RepoNeedle-Playground` | Space, **Gradio** SDK | README badge; default target of `scripts/prepare_hf_release.py` (`--org RepoNeedle`) | +| `RepoNeedle/RepoNeedle-Bench` | Dataset | README badge; `hf_artifacts/dataset/`; default target of `scripts/prepare_hf_release.py` | + +Both Spaces are named `RepoNeedle-Playground` but live in different namespaces and use different SDKs. Never publish one directory to the other's repository. + +### How the pieces fit together + +``` +Static Space (index.html) ──HTTPS──► Render API (api.py) ──imports──► hf_artifacts/space/reponeedle (bundled engine) + └─imports──► hf_artifacts/space/app.py (embedded Flask/Requests snapshots) +Gradio Space (app.py) ──in-process──► hf_artifacts/space/reponeedle (bundled engine) +``` + +- The static Space runs no Python. It calls the API URL shown in its **Backend API URL** field (default `https://reponeedle.onrender.com`). +- `api.py` puts `hf_artifacts/space/` first on `sys.path`, so it runs the **bundled** engine, not the copy that `requirements-api.txt` installs from the repository root. +- The bundled engine is an exact copy of `reponeedle/` **except** the optional `reponeedle/mcp/` package, which the demo does not need. `tests/test_hf_space_bundle.py` fails if the two diverge, and `scripts/prepare_hf_release.py` refuses to publish a diverged bundle. After changing `reponeedle/`, copy the changed files into `hf_artifacts/space/reponeedle/`. +- The API's reported version is read from the imported engine's `reponeedle.__version__`, so it always reflects the code actually running. + +## Deployment sequence for 1.2.0 + +The order is API first, then the frontends. The 1.2.0 static UI also works against a 1.1.0 API: the older API ignores the new opt-in field, and its heuristic is always on. + +1. **Merge to `main`** once CI passes. +2. **Render API.** Render builds from `main`. If auto-deploy is enabled for the service, merging starts a deploy; otherwise start a manual deploy in the Render dashboard. Then verify: + - `GET https://YOUR-SERVICE.onrender.com/health` returns `{"status":"ok","service":"reponeedle-demo-api","version":"1.2.0"}`; + - `GET /` reports `"version": "1.2.0"`. +3. **Static Space.** Upload `README.md` and `index.html` from `hf_artifacts/static_space/` to the static Space root. This is a manual step; the release script does not publish the static Space. Verify that the page shows `PLAYGROUND · V1.2.0` and that **Connect API** reports `Connected · RepoNeedle API 1.2.0`. +4. **Gradio Space** (if it is maintained). Run the preflight, then publish the Space only: + ```bash + python scripts/prepare_hf_release.py --check-only --space-only --org + python scripts/prepare_hf_release.py --space-only --org + ``` + - `--space-only` does not upload the dataset. + - The script checks that the bundled engine matches the root package and `pyproject.toml`, never prints the token, and verifies that the Space's head is the uploaded commit. + - It refuses to upload into an existing Space whose SDK is not `gradio`, such as the static frontend Space. The Gradio and static Spaces must be different repositories. + - The Space installs Gradio `sdk_version` 4.29.0 from its README front matter. `requirements.txt` pins `fastapi<0.113`, which keeps Starlette at 0.38.x: Gradio 4.x cannot serve its page with Starlette 1.x. +5. **Dataset.** No upload is needed; the demonstration dataset is unchanged in 1.2.0. + +Do not describe a remote Space or API as upgraded until its live version has been checked as above. ## Deploy the API on Render @@ -29,7 +75,7 @@ The API imports the embedded corpora and engine from `hf_artifacts/space/`. Do n Verify `https://YOUR-SERVICE.onrender.com/health`. A healthy response is: ```json -{"status":"ok","service":"reponeedle-demo-api","version":"1.1.0"} +{"status":"ok","service":"reponeedle-demo-api","version":"1.2.0"} ``` ## Deploy the frontend on Hugging Face @@ -40,22 +86,35 @@ The UI defaults to `https://reponeedle.onrender.com`. Users can change the Backe ## API contract -- `GET /` returns service metadata and a scope disclosure. -- `GET /health` is the health check. -- `POST /api/retrieve` accepts `repo_name`, `ranker`, `task_mode`, `budget_tokens`, `seed_files`, and `query`. +- `GET /` returns service metadata (including `version`) and a scope disclosure. +- `GET /health` is the health check: `{"status": "ok", "service": "reponeedle-demo-api", "version": ""}`. +- `POST /api/retrieve` accepts: + - `repo_name`, `ranker`, `task_mode`, `budget_tokens`, `seed_files` and `query`; + - optional `enable_experimental_abstention` (boolean, default `false`), new in 1.2.0. +- Response fields are unchanged from 1.1.0, plus an additive `abstention` object: `{"enabled", "status", "experimental": true, "calibrated": false}`. `status` is `"disabled"` by default and `"evaluated"` when opted in. - Supported demo repositories: `pallets/flask`, `psf/requests`. - Supported rankers: `legacy` (default), `change_impact`. -- The API validates query length, budget limits, repository choice, and seed count using Pydantic. +- The API validates query length, budget limits, repository choice, seed count and the abstention flag type using Pydantic. +- Internal errors return HTTP 500 with a generic message. Exception details are written to the server log only and are never returned to callers. + +## Experimental abstention (1.2.0 default: off) + +- In 1.2.0 the out-of-scope abstention heuristic is **disabled by default**. The API, the static UI and the Gradio Space return ranked context for every query unless the caller opts in. +- The static UI and the Gradio Space have an unchecked **Experimental abstention** control. +- Choosing the `selective` task does not enable abstention. +- The heuristic is experimental and **uncalibrated**: on the ARB v2 selective split it correctly abstained on 0 of 82 out-of-scope queries. An abstention on a demo query is a diagnostic, not a reliable out-of-scope signal. See `KNOWN_LIMITATIONS.md`. +- In 1.1.0 (the previous demo) the heuristic was always on. ## Limitations and security boundaries - This is a public, unauthenticated demonstration API. Do not send secrets, proprietary source code, or personal data. - CORS is open so the public Hugging Face frontend can call the API. This is not an authentication or abuse-prevention control. - The API only retrieves from the two embedded demonstration snapshots. It does not index arbitrary repositories or expose a general upload endpoint. +- The demos rank in-memory documents (`RepoNeedleEngine.from_documents`). They do not index a directory, so they do not exercise 1.2.0's filesystem indexing safeguards (link containment, directory-link traversal limits) or show indexing diagnostics. - The API does not persist visitor queries. Its in-memory caches reset when the service restarts. - Free hosting may sleep after inactivity and has limited CPU/RAM. Cold starts and request latency can vary. - The dataset is a small synthetic demo dataset, not the complete Agent Retrieval Bench v2 corpus. -- Selective abstention is experimental and should not be presented as reliable; see `KNOWN_LIMITATIONS.md`. +- Demo results are not benchmark results; see the README for the documented evaluations and their qualifications. ## Local development and tests @@ -66,4 +125,6 @@ pip install -e ".[dev,api]" pytest -q ``` -The API tests use FastAPI's `TestClient` and the public-demo corpus shipped with the repository. +- The API tests (`tests/test_public_api.py`) use FastAPI's `TestClient` and the demo corpora shipped with the repository. +- `tests/test_hf_space_bundle.py` checks the bundled engine against the root package and smoke-tests the Gradio app without Gradio installed. +- It also checks the release script's safeguards using a fake Hub client, so nothing is uploaded. diff --git a/hf_artifacts/space/README.md b/hf_artifacts/space/README.md index e3183a2..07290e5 100644 --- a/hf_artifacts/space/README.md +++ b/hf_artifacts/space/README.md @@ -10,16 +10,22 @@ pinned: false license: mit --- -# 🪡 RepoNeedle Playground (V1.1.0) +# 🪡 RepoNeedle Playground (V1.2.0) Interactive demonstration of **RepoNeedle**: Token-Budget-Aware, Workflow-Adaptive Repository Context Retrieval for AI Coding Agents. > ⚠️ **Methodological Disclosure**: This Playground uses embedded demonstration repository snapshots. It is an interactive demonstration, not a reproduction of the complete ARB v2 evaluation. The embedded corpora are lightweight demonstration slices, not full upstream repository checkouts. -### Key Capabilities in V1.1.0: +### What this Playground demonstrates (bundled RepoNeedle 1.2.0 engine): - **Modular Ranking Engine Selection**: Choose between the default `legacy` ranker (V1.0 backwards-compatible multi-signal routing) and the opt-in `change_impact` ranking engine. - **Change-Impact Graph Propagation**: When seed / anchor files are provided, the engine computes seed-relative relationship scoring across static imports, symbols, and calls with Seed File Suppression (SFS). - **Seedless Change-Impact Mode**: When run without seed files, the interface clarifies that lexical/semantic scoring and Test Intent Priors remain active while seed-relative relationship boosts and suppression are inactive. - **Workflow-Adaptive Routing**: Automatic intent classification for `trace2code`, `code2test`, `edit2ripple`, and `comment2context`. - **CUD-Pack Context Optimization**: Deterministic budget enforcement ($C_{\text{actual}} \le B_{\text{configured}}$) across representation tiers (`FULL`, `FOCUS_SPAN`, `SKELETON`). - **Multi-System Baseline Comparison**: Side-by-side comparison with classical BM25 and RepoMap PageRank under identical context constraints. +- **Experimental Abstention (opt-in, off by default)**: In 1.2.0 the out-of-scope abstention heuristic is disabled by default, so ranked context is returned for every query. An unchecked **Enable experimental abstention** box lets you opt in. The heuristic is experimental and uncalibrated: on the ARB v2 selective split it correctly abstained on 0 of 82 out-of-scope queries. + +### Scope of the demonstration + +- The Playground ranks small embedded document snapshots held in memory. It does not index a directory, so it does not exercise 1.2.0's filesystem-level indexing safeguards (link containment, directory-link traversal limits) or report indexing diagnostics. +- The CLI JSON output, `--anchors` validation and the MCP server added in 1.2.0 are not part of this Space. See the [RepoNeedle repository](https://github.com/krishika08/RepoNeedle) for those features and for the documented benchmark results and limitations. diff --git a/hf_artifacts/space/app.py b/hf_artifacts/space/app.py index 87330bb..5bb0f9c 100644 --- a/hf_artifacts/space/app.py +++ b/hf_artifacts/space/app.py @@ -89,6 +89,7 @@ def create_embedded_corpora() -> Dict[str, List[FileDoc]]: 4000, "", "Traceback (most recent call last):\n File \"src/flask/app.py\", line 28, in wsgi_app\n response = self.dispatch_request()\n File \"src/flask/wrappers.py\", line 14, in get_json\nAttributeError: 'NoneType' object has no attribute 'get'", + False, ], [ "pallets/flask", @@ -97,6 +98,7 @@ def create_embedded_corpora() -> Dict[str, List[FileDoc]]: 4000, "src/flask/sessions.py", "Fix cookie attribute handling during session modification in SecureCookieSessionInterface.", + False, ], [ "pallets/flask", @@ -105,6 +107,7 @@ def create_embedded_corpora() -> Dict[str, List[FileDoc]]: 4000, "", "Fix cookie attribute handling during session modification without specifying a seed file.", + False, ], [ "pallets/flask", @@ -113,6 +116,7 @@ def create_embedded_corpora() -> Dict[str, List[FileDoc]]: 4000, "", "Find unit and regression tests for SecureCookieSessionInterface cookie saving and modification.", + False, ], [ "psf/requests", @@ -121,6 +125,7 @@ def create_embedded_corpora() -> Dict[str, List[FileDoc]]: 4000, "", "Find test suites that verify HTTPBasicAuth headers and authentication credentials formatting.", + False, ], [ "pallets/flask", @@ -129,6 +134,8 @@ def create_embedded_corpora() -> Dict[str, List[FileDoc]]: 4000, "", "Fix compilation error in Cargo.toml regarding serde_json feature flag dependency in rust kernel.", + # Explicit opt-in: the experimental abstention heuristic is disabled by default in 1.2.0. + True, ], ] @@ -140,6 +147,7 @@ def run_reponeedle_demo( budget: int, seed_files_input: str, query: str, + enable_experimental_abstention: bool = False, ): if not query.strip(): return ( @@ -158,7 +166,7 @@ def run_reponeedle_demo( # Parse optional seed / anchor files seed_list = [s.strip() for s in seed_files_input.replace("\n", ",").split(",") if s.strip()] - # Construct V1.1.0 Query object + # Construct the Query object query_obj = Query( text=query.strip(), anchor_files=seed_list, @@ -167,7 +175,12 @@ def run_reponeedle_demo( ) t0 = time.perf_counter() - res = engine.retrieve(query=query_obj, ranker=effective_ranker) + abstention_opt_in = bool(enable_experimental_abstention) + res = engine.retrieve( + query=query_obj, + ranker=effective_ranker, + enable_experimental_abstention=abstention_opt_in, + ) elapsed_ms = (time.perf_counter() - t0) * 1000 # Format diagnostic notes based on ranker and seeds @@ -190,19 +203,31 @@ def run_reponeedle_demo( f"- **Behavior**: Standard multi-signal hybrid scoring with workflow routing and AST centrality.\n" ) + if abstention_opt_in: + ranker_note += ( + "- **Experimental Abstention**: **opt-in enabled**. The heuristic is experimental and " + "uncalibrated (it correctly abstained on 0 of 82 out-of-scope queries in the ARB v2 " + "selective split); treat an abstention as a diagnostic, not a reliable out-of-scope signal.\n" + ) + else: + ranker_note += ( + "- **Experimental Abstention**: disabled (the 1.2.0 default). Ranked context is returned " + "even for out-of-scope queries.\n" + ) + # Decision summary if res.abstained: status_md = ( - f"### 🛑 Decision: **ABSTAINED (Selective Abstention)**\n\n" + f"### 🛑 Decision: **ABSTAINED (experimental abstention, opt-in)**\n\n" f"{ranker_note}\n" - f"- **Confidence Score**: `{res.confidence_score:.3f}` (Below threshold)\n" + f"- **Confidence Score**: `{res.confidence_score:.3f}` (below the experimental threshold; uncalibrated)\n" f"- **Inferred Task**: `{res.task_type.value}`\n" f"- **Tokens Consumed**: `0 / {budget}`\n" f"- **Latency**: `{elapsed_ms:.2f} ms`\n\n" f"> **Diagnostic Reason**: {res.explanation}\n" ) rn_rows = [] - file_preview = "System abstained from context retrieval to prevent context window dilution." + file_preview = "No context returned: the opt-in experimental abstention heuristic abstained on this query." else: # Tier counts tier_counts: Dict[str, int] = {} @@ -265,7 +290,7 @@ def build_interface(): theme = gr.themes.Soft(primary_hue="blue") with gr.Blocks(theme=theme, title="RepoNeedle Playground") as demo: gr.Markdown( - "# 🪡 RepoNeedle Playground (V1.1.0)\n" + "# 🪡 RepoNeedle Playground (V1.2.0)\n" "### Token-Budget-Aware, Workflow-Adaptive Repository Context Retrieval for AI Coding Agents\n\n" "> ⚠️ **Methodological Disclosure**: This Playground uses embedded demonstration repository snapshots. " "It is an interactive demonstration, not a reproduction of the complete ARB v2 evaluation. " @@ -308,6 +333,14 @@ def build_interface(): placeholder="Enter an issue description, traceback, or test retrieval request...", lines=4, ) + abstention_checkbox = gr.Checkbox( + label="Enable experimental abstention (opt-in)", + value=False, + info=( + "Off by default in RepoNeedle 1.2.0. The heuristic is experimental and uncalibrated: " + "it correctly abstained on 0 of 82 out-of-scope queries in the ARB v2 selective split." + ), + ) submit_btn = gr.Button("🔍 Retrieve Context", variant="primary") with gr.Column(scale=2): @@ -327,7 +360,7 @@ def build_interface(): gr.Examples( examples=EXAMPLES, - inputs=[repo_select, ranker_select, task_select, budget_slider, seed_input, query_input], + inputs=[repo_select, ranker_select, task_select, budget_slider, seed_input, query_input, abstention_checkbox], outputs=[status_box, table_rn, table_comp, preview_box], fn=run_reponeedle_demo, cache_examples=False, @@ -335,7 +368,7 @@ def build_interface(): submit_btn.click( fn=run_reponeedle_demo, - inputs=[repo_select, ranker_select, task_select, budget_slider, seed_input, query_input], + inputs=[repo_select, ranker_select, task_select, budget_slider, seed_input, query_input, abstention_checkbox], outputs=[status_box, table_rn, table_comp, preview_box], ) diff --git a/hf_artifacts/space/reponeedle/__init__.py b/hf_artifacts/space/reponeedle/__init__.py index 293b600..28335b1 100644 --- a/hf_artifacts/space/reponeedle/__init__.py +++ b/hf_artifacts/space/reponeedle/__init__.py @@ -11,8 +11,10 @@ RankerType, BudgetConfig, ContextTier, + AbstentionDiagnostics, + IndexingDiagnostics, ) -from reponeedle.engine import RepoNeedleEngine +from reponeedle.engine import RepoNeedleEngine, DEFAULT_MAX_FILE_SIZE_BYTES from reponeedle.core.change_impact import ChangeImpactRanker, ChangeImpactConfig from reponeedle.core.packer import ( CUDPackOptimizer, @@ -21,7 +23,7 @@ CodeSpanExtractor, ) -__version__ = "1.1.0" +__version__ = "1.2.0" __all__ = [ "RepoNeedleEngine", "FileDoc", @@ -32,6 +34,9 @@ "RankerType", "BudgetConfig", "ContextTier", + "AbstentionDiagnostics", + "IndexingDiagnostics", + "DEFAULT_MAX_FILE_SIZE_BYTES", "ChangeImpactRanker", "ChangeImpactConfig", "CUDPackOptimizer", diff --git a/hf_artifacts/space/reponeedle/cli.py b/hf_artifacts/space/reponeedle/cli.py index 4812919..fad83b8 100644 --- a/hf_artifacts/space/reponeedle/cli.py +++ b/hf_artifacts/space/reponeedle/cli.py @@ -3,21 +3,100 @@ """ import sys +import os +import json import argparse +from typing import List, Optional from rich.console import Console from rich.table import Table from rich.panel import Panel -from reponeedle.engine import RepoNeedleEngine +from reponeedle.engine import ( + RepoNeedleEngine, + DEFAULT_MAX_FILE_SIZE_BYTES, + validate_max_file_size_bytes, + is_path_within_root, + relpath_escapes_root, +) from reponeedle.models import TaskType, BudgetConfig +# Maximum number of skipped paths listed in human-readable (non-JSON) stderr notices. +# JSON output always contains the complete list. +_MAX_SKIPPED_PATHS_SHOWN = 20 -def main(): - if hasattr(sys.stdout, "reconfigure"): + +def resolve_and_validate_anchors(repo_dir: str, anchors: Optional[List[str]]) -> List[str]: + """ + Validates and resolves anchor file paths against the repository directory. + Returns normalized repository-relative paths with forward slashes. + Raises ValueError if an anchor does not exist or falls outside the repository. + """ + if not anchors: + return [] + + abs_repo = os.path.normpath(os.path.abspath(repo_dir)) + normalized: List[str] = [] + + for a in anchors: + if not a or not a.strip(): + continue + raw_anchor = a.strip() + + # The exact spelling is tried first. On POSIX, a Windows-style spelling ("src\\a.py") is then + # retried with forward slashes, consistent with retrieve(anchor_files=...). A real file whose + # name contains a backslash still takes precedence. Containment checks below apply either way. + spellings = [raw_anchor] + if os.sep == "/" and "\\" in raw_anchor: + spellings.append(raw_anchor.replace("\\", "/")) + + resolved_target = None + for spelling in spellings: + # Interpretation 1: relative to repo_dir (e.g. "auth/session.py" with --repo "dir"). + candidate_1 = os.path.normpath(os.path.join(abs_repo, spelling)) + # Interpretation 2: relative to the current working directory, or absolute. + candidate_2 = os.path.normpath(os.path.abspath(spelling)) + if os.path.isfile(candidate_1): + resolved_target = candidate_1 + elif os.path.isfile(candidate_2): + resolved_target = candidate_2 + if resolved_target is not None: + break + + if resolved_target is None: + raise ValueError( + f"Anchor file '{raw_anchor}' does not exist or is not a regular file within repository '{repo_dir}'." + ) + + # Ensure the resolved file is inside the repository root try: - sys.stdout.reconfigure(encoding="utf-8", errors="replace") - except Exception: - pass + rel_to_repo = os.path.relpath(resolved_target, abs_repo) + except ValueError: + raise ValueError( + f"Anchor file '{raw_anchor}' is located on a different drive than repository '{repo_dir}'." + ) + + # Component-aware traversal check ('..cache/x' is a legitimate name; '../x' is not). + if relpath_escapes_root(rel_to_repo) or os.path.isabs(rel_to_repo): + raise ValueError( + f"Anchor file '{raw_anchor}' resolves outside repository root '{repo_dir}'." + ) + + # Canonical containment (symlinks / junctions), consistent with indexing: an anchor whose + # link target lies outside the repository is rejected, so it cannot name excluded content. + if not is_path_within_root(os.path.realpath(resolved_target), os.path.realpath(abs_repo)): + raise ValueError( + f"Anchor file '{raw_anchor}' is a link that resolves outside repository root '{repo_dir}'." + ) + + clean_rel = rel_to_repo.replace("\\", "/") + if clean_rel not in normalized: + normalized.append(clean_rel) + + return normalized + + +def build_parser() -> argparse.ArgumentParser: + """Builds and returns the argument parser for RepoNeedle CLI.""" parser = argparse.ArgumentParser( prog="reponeedle", description="Token-Budget-Aware, Workflow-Adaptive Repository Context Retrieval for AI Coding Agents.", @@ -64,29 +143,225 @@ def main(): choices=["legacy", "bm25", "arm101", "cascade", "change_impact"], help="Retrieval ranker architecture (default: legacy).", ) + parser.add_argument( + "--json", + action="store_true", + help="Emit results as machine-readable JSON on stdout without terminal formatting or ANSI codes.", + ) + parser.add_argument( + "--anchors", + type=str, + nargs="+", + default=None, + metavar="PATH", + help="One or more seed or anchor file paths for change-impact analysis or ripple queries.", + ) + parser.add_argument( + "--include-context", + action="store_true", + help="Include the full rendered prompt context in the JSON response under 'rendered_context'.", + ) + parser.add_argument( + "--enable-experimental-abstention", + action="store_true", + help=( + "Opt in to the EXPERIMENTAL, uncalibrated out-of-scope abstention heuristic " + "(disabled by default; 0/82 correct abstentions on the ARB v2 selective split). " + "'--task selective' does not enable it." + ), + ) + parser.add_argument( + "--max-file-size-bytes", + type=int, + default=DEFAULT_MAX_FILE_SIZE_BYTES, + metavar="BYTES", + help=( + f"Skip files strictly larger than this many bytes during indexing " + f"(default: {DEFAULT_MAX_FILE_SIZE_BYTES}). Skipped paths are reported in diagnostics." + ), + ) + return parser + + +def _indexing_notice_lines(diag) -> List[str]: + """Human-readable lines describing indexing exclusions (paths only, never contents).""" + lines: List[str] = [] + if diag.skipped_large_files: + lines.append( + f"{diag.skipped_large_files} file(s) larger than {diag.max_file_size_bytes} bytes were not indexed:" + ) + for p in diag.skipped_paths[:_MAX_SKIPPED_PATHS_SHOWN]: + lines.append(f" - {p}") + remaining = len(diag.skipped_paths) - _MAX_SKIPPED_PATHS_SHOWN + if remaining > 0: + lines.append(f" ... and {remaining} more (use --json for the full list)") + if diag.skipped_unreadable_files: + lines.append(f"{diag.skipped_unreadable_files} file(s) could not be read and were not indexed.") + if diag.skipped_symlink_outside_repository: + lines.append( + f"{diag.skipped_symlink_outside_repository} linked file(s) resolve outside the repository " + f"and were not read (reason: symlink_outside_repository):" + ) + for p in diag.symlink_outside_repository_paths[:_MAX_SKIPPED_PATHS_SHOWN]: + lines.append(f" - {p}") + if diag.skipped_directories_outside_repository: + lines.append( + f"{diag.skipped_directories_outside_repository} directory link(s) resolve outside the repository " + f"(or cannot be resolved) and were not traversed (see skipped_directory_paths in --json)." + ) + if diag.max_files_reached: + lines.append(f"Indexing stopped at the max_files limit ({diag.max_files}); remaining files were not indexed.") + return lines + + +def main(argv: Optional[List[str]] = None) -> int: + if hasattr(sys.stdout, "reconfigure"): + try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + + parser = build_parser() + args = parser.parse_args(argv) - args = parser.parse_args() - console = Console() + # Validate repository path + repo_path = os.path.abspath(args.repo) + if not os.path.isdir(repo_path): + sys.stderr.write(f"Error: Repository directory '{args.repo}' does not exist or is not a directory.\n") + return 1 - console.print(Panel(f"[bold cyan]RepoNeedle Context Retrieval[/bold cyan]\nTarget Repository: [green]{args.repo}[/green]\nQuery: [yellow]\"{args.query}\"[/yellow]\nToken Budget: [magenta]{args.budget}[/magenta]\nRanker: [cyan]{args.ranker}[/cyan]", expand=False)) + # Validate and normalize anchor files + try: + normalized_anchors = resolve_and_validate_anchors(args.repo, args.anchors) + except ValueError as err: + sys.stderr.write(f"Error: {err}\n") + return 1 + + # Validate file-size limit before any indexing work + try: + max_file_size_bytes = validate_max_file_size_bytes(args.max_file_size_bytes) + except (TypeError, ValueError) as err: + sys.stderr.write(f"Error: {err}\n") + return 1 config = BudgetConfig( max_tokens=args.budget, allow_skeletons=not args.no_skeletons, mode=args.mode, + enable_experimental_abstention=args.enable_experimental_abstention, ) - with console.status("[bold green]Indexing repository..."): - engine = RepoNeedleEngine.from_directory(args.repo, budget_config=config) + console = None + if not args.json: + console = Console() + console.print( + Panel( + f"[bold cyan]RepoNeedle Context Retrieval[/bold cyan]\n" + f"Target Repository: [green]{args.repo}[/green]\n" + f"Query: [yellow]\"{args.query}\"[/yellow]\n" + f"Token Budget: [magenta]{args.budget}[/magenta]\n" + f"Ranker: [cyan]{args.ranker}[/cyan]" + + (f"\nAnchors: [blue]{', '.join(normalized_anchors)}[/blue]" if normalized_anchors else "") + + ( + "\nAbstention: [yellow]EXPERIMENTAL heuristic enabled (uncalibrated)[/yellow]" + if args.enable_experimental_abstention else "" + ), + expand=False, + ) + ) + with console.status("[bold green]Indexing repository..."): + engine = RepoNeedleEngine.from_directory( + args.repo, budget_config=config, max_file_size_bytes=max_file_size_bytes + ) + # Indexing exclusions are diagnostics: always routed to stderr, never stdout. + notice = _indexing_notice_lines(engine.indexing_diagnostics) + if notice: + Console(stderr=True).print( + Panel("\n".join(notice), title="Indexing notice", border_style="yellow", expand=False), + markup=False, + ) + else: + # The engine logs a summary WARNING to stderr when files are skipped; the full, + # machine-readable list is emitted in the JSON payload under "indexing". + engine = RepoNeedleEngine.from_directory( + args.repo, budget_config=config, max_file_size_bytes=max_file_size_bytes + ) task_type = TaskType(args.task) if args.task != "auto" else None - result = engine.retrieve(query=args.query, budget_tokens=args.budget, task_type=task_type, ranker=args.ranker) + result = engine.retrieve( + query=args.query, + budget_tokens=args.budget, + task_type=task_type, + ranker=args.ranker, + anchor_files=normalized_anchors, + ) + + # Handle output-file writing if requested + if args.output_file: + try: + with open(args.output_file, "w", encoding="utf-8") as f: + f.write(result.rendered_context) + if not args.json and console: + console.print(f"[bold green]Serialized prompt saved to:[/bold green] {args.output_file}") + except Exception as exc: + sys.stderr.write(f"Error saving output file '{args.output_file}': {exc}\n") + return 1 + + # JSON output branch + if args.json: + selected_list = [] + for rank, sf in enumerate(result.selected_files, start=1): + selected_list.append({ + "rank": rank, + "path": sf.path, + "tier": sf.tier.value if hasattr(sf.tier, "value") else str(sf.tier), + "token_count": int(sf.token_count), + "score": round(float(sf.score), 4), + "explanation": sf.explanation or "", + }) + + payload = { + "schema_version": "1.2.0", + "status": "abstained" if result.abstained else "success", + "query": result.query, + "repository": args.repo, + "ranker": result.ranker or args.ranker, + "task_type": result.task_type.value if hasattr(result.task_type, "value") else str(result.task_type), + "abstained": bool(result.abstained), + "explanation": result.explanation or "", + "budget_tokens": int(result.budget_tokens), + "total_tokens": int(result.total_tokens), + "execution_time_ms": round(float(result.execution_time_ms), 2), + "confidence_score": round(float(result.confidence_score), 4), + "selected_files": selected_list, + "abstention": result.abstention.model_dump() if result.abstention is not None else None, + "indexing": engine.indexing_diagnostics.model_dump(), + } + + if args.include_context or args.serialize: + payload["rendered_context"] = result.rendered_context + print(json.dumps(payload, indent=2, ensure_ascii=False)) + return 0 + + # Rich terminal output branch (existing behavior) if result.abstained: - console.print(Panel(f"[bold red]System Abstained[/bold red]\n{result.explanation}\nConfidence Score: {result.confidence_score:.3f}\nExecution Time: {result.execution_time_ms:.1f}ms", border_style="red")) - return + is_heuristic = result.abstention is not None and result.abstention.status == "evaluated" + heading = "System Abstained (experimental heuristic)" if is_heuristic else "System Abstained" + console.print( + Panel( + f"[bold red]{heading}[/bold red]\n" + f"{result.explanation}\n" + f"Confidence Score: {result.confidence_score:.3f}\n" + f"Execution Time: {result.execution_time_ms:.1f}ms", + border_style="red", + ) + ) + return 0 - table = Table(title=f"Retrieved Context ({result.total_tokens}/{result.budget_tokens} tokens) - {result.execution_time_ms:.1f}ms") + table = Table( + title=f"Retrieved Context ({result.total_tokens}/{result.budget_tokens} tokens) - {result.execution_time_ms:.1f}ms" + ) table.add_column("Rank", justify="center", style="cyan") table.add_column("File Path", style="green") table.add_column("Tier", justify="center", style="magenta") @@ -106,11 +381,6 @@ def main(): console.print(table) - if args.output_file: - with open(args.output_file, "w", encoding="utf-8") as f: - f.write(result.rendered_context) - console.print(f"[bold green]Serialized prompt saved to:[/bold green] {args.output_file}") - if args.serialize: console.print("\n[bold cyan]=== Serialized Context Prompt ===[/bold cyan]\n") try: @@ -119,7 +389,8 @@ def main(): enc = sys.stdout.encoding or "utf-8" print(result.rendered_context.encode(enc, errors="replace").decode(enc)) + return 0 -if __name__ == "__main__": - main() +if __name__ == "__main__": + sys.exit(main()) diff --git a/hf_artifacts/space/reponeedle/core/abstention.py b/hf_artifacts/space/reponeedle/core/abstention.py index 1665d75..de69c49 100644 --- a/hf_artifacts/space/reponeedle/core/abstention.py +++ b/hf_artifacts/space/reponeedle/core/abstention.py @@ -1,6 +1,12 @@ """ -Calibrated selective abstention engine for no-gold and out-of-repository queries. -Addresses the selective retrieval calibration gap identified in Agent Retrieval Bench. +EXPERIMENTAL selective abstention heuristic for no-gold and out-of-repository queries. + +Status: experimental, NOT production-calibrated. On the official Agent Retrieval Bench v2 +selective split (N=82 negative samples) it correctly abstained on 0 of 82. Since 1.2.0 the +engine only applies it when explicitly enabled (BudgetConfig.enable_experimental_abstention, +the per-call `enable_experimental_abstention` override, the CLI flag +`--enable-experimental-abstention`, or the MCP retrieve argument of the same name). +See KNOWN_LIMITATIONS.md for details. """ from typing import List, Dict, Set, Tuple @@ -76,7 +82,8 @@ def evaluate_abstention( Evaluates abstention criteria. Returns: abstained (bool): True if system should abstain. - confidence (float): Calibrated confidence score in [0, 1]. + confidence (float): Uncalibrated heuristic score in [0, 1]. It is a diagnostic + signal, not a probability that the repository contains the answer. reason (str): Diagnostic explanation. """ if len(fused_scores) == 0: @@ -114,7 +121,7 @@ def evaluate_abstention( median_score = float(np.median(fused_scores)) margin = max(0.0, max_score - median_score) - # Calibrated composite confidence score + # Composite heuristic confidence score (uncalibrated; weights are fixed, not fitted) confidence = ( 0.30 * z_component + 0.45 * overlap_ratio + diff --git a/hf_artifacts/space/reponeedle/engine.py b/hf_artifacts/space/reponeedle/engine.py index f5743ee..2266f7e 100644 --- a/hf_artifacts/space/reponeedle/engine.py +++ b/hf_artifacts/space/reponeedle/engine.py @@ -5,6 +5,10 @@ import os import time +import logging +import pathlib +import numbers +import stat from typing import List, Optional, Union, Dict, Set import numpy as np import pathspec @@ -19,6 +23,8 @@ BudgetConfig, AblationConfig, ContextTier, + AbstentionDiagnostics, + IndexingDiagnostics, ) from reponeedle.core.function_graph import FunctionGraphEngine from reponeedle.core.arm101 import ARM101Ranker @@ -40,6 +46,78 @@ "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "poetry.lock", ] +# Files strictly larger than this many bytes are excluded from the index (a file exactly at +# the limit is indexed). Exclusions are recorded in RepoNeedleEngine.indexing_diagnostics. +DEFAULT_MAX_FILE_SIZE_BYTES = 500_000 + +logger = logging.getLogger(__name__) + + +def relpath_escapes_root(rel_path: str) -> bool: + """True if a relative path climbs out of its base (first component is '..'). + + Component-aware: names that merely start with '..' (e.g. '..cache/x') do not escape.""" + parts = pathlib.PurePath(rel_path).parts + return bool(parts) and parts[0] == os.pardir + + +def is_path_within_root(path: str, real_root: str) -> bool: + """True if the canonical `path` is `real_root` itself or lies beneath it (component-aware).""" + try: + root_n = os.path.normcase(real_root) + return os.path.commonpath([root_n, os.path.normcase(path)]) == root_n + except ValueError: # different drives / mixed absolute-relative paths + return False + + +def resolve_within_root(path: str, real_root: str) -> Optional[str]: + """ + Resolve `path` to its canonical location (following symlinks and, on Windows, junctions). + + Returns the resolved path if it stays inside `real_root`, or None if it resolves outside. + Raises OSError (or RuntimeError/ValueError) if resolution fails, e.g. broken links or loops. + """ + resolved = os.path.realpath(path, strict=True) + return resolved if is_path_within_root(resolved, real_root) else None + + +def is_directory_link(path: str) -> bool: + """True if `path` is a symlink or a Windows junction (mount-point reparse point). + + ``os.walk(followlinks=False)`` relies on ``os.path.islink``, which is False for Windows + junctions, so junctions would otherwise be descended into. The check uses ``os.lstat`` + directly (the same reparse-tag test as ``os.path.isjunction``, which Python < 3.12 lacks), so + every Python version takes the same path. Fail-closed: if the entry cannot be inspected it + counts as a link, and the directory is not traversed. (``os.path.islink``/``isjunction`` + would instead report False on errors.)""" + try: + st = os.lstat(path) + except (OSError, ValueError): + return True + if stat.S_ISLNK(st.st_mode): + return True + if os.name == "nt": + return getattr(st, "st_reparse_tag", 0) == getattr(stat, "IO_REPARSE_TAG_MOUNT_POINT", 0xA0000003) + return False + + +def validate_max_file_size_bytes(value) -> int: + """ + Validates a file-size limit. The contract is a strictly positive integer number of bytes. + + Raises: + TypeError: If the value is not an integer (bool is rejected). + ValueError: If the value is zero or negative. + """ + if isinstance(value, bool) or not isinstance(value, numbers.Integral): + raise TypeError( + f"max_file_size_bytes must be a positive integer number of bytes, got {type(value).__name__}." + ) + value = int(value) + if value <= 0: + raise ValueError(f"max_file_size_bytes must be a positive integer (> 0), got {value}.") + return value + class RepoNeedleEngine: """ @@ -86,6 +164,9 @@ def __init__( self.change_impact_config = change_impact_config or ChangeImpactConfig() self.change_impact_ranker = ChangeImpactRanker(self.change_impact_config) + # Populated with scan statistics by from_directory(); "documents" source otherwise. + self.indexing_diagnostics = IndexingDiagnostics(source="documents", indexed_files=0) + if docs: self._fit(docs) @@ -116,6 +197,7 @@ def _fit(self, docs: List[FileDoc]): self.repo_graph.fit(clean_docs) self.function_graph.build_graph(clean_docs) self.abstainer.fit(clean_docs) + self.indexing_diagnostics.indexed_files = len(clean_docs) def update_document(self, doc: FileDoc): """ @@ -154,6 +236,7 @@ def update_documents(self, docs: List[FileDoc]): self.repo_graph.fit(clean_docs) self.function_graph.build_graph(clean_docs) self.abstainer.fit(clean_docs) + self.indexing_diagnostics.indexed_files = len(clean_docs) def query(self, *args, **kwargs) -> RetrievalResult: """Alias for retrieve() to provide intuitive querying API.""" @@ -192,10 +275,20 @@ def from_directory( ranker: Optional[Union[str, RankerType]] = RankerType.LEGACY, ranking_engine: Optional[Union[str, RankerType]] = None, change_impact_config: Optional[ChangeImpactConfig] = None, + max_file_size_bytes: int = DEFAULT_MAX_FILE_SIZE_BYTES, ) -> "RepoNeedleEngine": """ Scans a local repository directory, parses files, and builds the retrieval index. + + Files strictly larger than ``max_file_size_bytes`` (default 500,000) are excluded from + the index. Every exclusion (oversized or unreadable) is recorded by repository-relative + path in ``engine.indexing_diagnostics`` and summarized via a WARNING on the + ``reponeedle.engine`` logger (stderr by default). File contents are never recorded. + + Raises: + TypeError / ValueError: If ``max_file_size_bytes`` is not a positive integer. """ + max_file_size_bytes = validate_max_file_size_bytes(max_file_size_bytes) repo_dir = os.path.abspath(repo_dir) ignore_patterns = list(DEFAULT_IGNORE_PATTERNS) if extra_ignore: @@ -214,8 +307,24 @@ def from_directory( spec = pathspec.PathSpec.from_lines("gitwildmatch", ignore_patterns) docs: List[FileDoc] = [] - - for root, dirs, files in os.walk(repo_dir): + skipped_large: List[str] = [] + unreadable: List[str] = [] + outside_root: List[str] = [] + max_files_reached = False + # Canonical repository root: every indexed file must resolve beneath it. + real_root = os.path.realpath(repo_dir) + # Directory traversal is bounded by the real directory tree: links are never followed and + # each physical directory (st_dev, st_ino) is entered at most once. + skipped_dirs: List[str] = [] + skipped_dirs_outside = 0 + visited_dirs: Set = set() + try: + root_st = os.stat(repo_dir) + visited_dirs.add((root_st.st_dev, root_st.st_ino)) + except OSError: + pass + + for root, dirs, files in os.walk(repo_dir, followlinks=False): # Prune ignored directories rel_root = os.path.relpath(root, repo_dir).replace("\\", "/") if rel_root == ".": @@ -227,6 +336,37 @@ def from_directory( and d not in DEFAULT_IGNORE_PATTERNS ] + # Prune directory links / junctions, cycles and directories outside the canonical root. + # Only the in-repository path is recorded; link targets are never listed or read. + kept_dirs = [] + for d in dirs: + child = os.path.join(root, d) + rel_child = f"{rel_root}/{d}" if rel_root else d + outside = False + if is_directory_link(child): + keep = False + try: + outside = resolve_within_root(child, real_root) is None + except (OSError, RuntimeError, ValueError): # broken link or loop + outside = True + else: + # A plain directory beneath an in-root directory is itself in-root; the + # (st_dev, st_ino) set catches cycles that are not links (e.g. bind mounts). + try: + child_st = os.stat(child) + key = (child_st.st_dev, child_st.st_ino) + keep = key not in visited_dirs + if keep: + visited_dirs.add(key) + except OSError: + keep = False + if keep: + kept_dirs.append(d) + else: + skipped_dirs.append(rel_child) + skipped_dirs_outside += int(outside) + dirs[:] = kept_dirs + for file in files: rel_path = os.path.relpath(os.path.join(root, file), repo_dir).replace("\\", "/") if spec.match_file(rel_path): @@ -238,13 +378,31 @@ def from_directory( continue full_path = os.path.join(root, file) + # Containment: resolve links BEFORE touching the target, and read only the + # validated canonical path (never the link itself). Links resolving outside the + # repository are skipped and recorded by their in-repository path only. try: - # Skip large files (> 500KB) - if os.path.getsize(full_path) > 500_000: - continue - with open(full_path, "r", encoding="utf-8", errors="ignore") as f: + resolved = resolve_within_root(full_path, real_root) + except (OSError, RuntimeError, ValueError): # broken link, loop, permission error + unreadable.append(rel_path) + continue + if resolved is None: + outside_root.append(rel_path) + continue + try: + file_size = os.path.getsize(resolved) + except OSError: + unreadable.append(rel_path) + continue + # Exclude oversized files (strictly greater than the limit), but record them. + if file_size > max_file_size_bytes: + skipped_large.append(rel_path) + continue + try: + with open(resolved, "r", encoding="utf-8", errors="ignore") as f: content = f.read() except Exception: + unreadable.append(rel_path) continue # Detect if test file @@ -278,11 +436,13 @@ def from_directory( ) if len(docs) >= max_files: + max_files_reached = True break if len(docs) >= max_files: + max_files_reached = True break - return cls( + engine = cls( docs=docs, budget_config=budget_config, ablation_config=ablation_config, @@ -291,6 +451,64 @@ def from_directory( change_impact_config=change_impact_config, ) + skipped_large.sort() + unreadable.sort() + outside_root.sort() + skipped_dirs.sort() + engine.indexing_diagnostics = IndexingDiagnostics( + source="directory", + indexed_files=len(engine.docs), + max_file_size_bytes=max_file_size_bytes, + skipped_large_files=len(skipped_large), + skipped_paths=skipped_large, + skipped_unreadable_files=len(unreadable), + unreadable_paths=unreadable, + skipped_symlink_outside_repository=len(outside_root), + symlink_outside_repository_paths=outside_root, + skipped_directories=len(skipped_dirs), + skipped_directory_paths=skipped_dirs, + skipped_directories_outside_repository=skipped_dirs_outside, + max_files=max_files, + max_files_reached=max_files_reached, + ) + if outside_root: + logger.warning( + "RepoNeedle skipped %d linked file(s) that resolve outside the repository root " + "(reason: symlink_outside_repository); their targets were not read. See indexing " + "diagnostics (symlink_outside_repository_paths) for the in-repository link paths.", + len(outside_root), + ) + + if skipped_dirs_outside: + logger.warning( + "RepoNeedle did not traverse %d directory link(s) that resolve outside the repository " + "root (or cannot be resolved); their contents were not listed or read. See indexing " + "diagnostics (skipped_directory_paths) for the in-repository paths.", + skipped_dirs_outside, + ) + + # Summaries only (counts + limit): no paths or contents in log records. + if skipped_large: + logger.warning( + "RepoNeedle skipped %d file(s) larger than max_file_size_bytes=%d; they are not " + "indexed or searchable. See indexing diagnostics (skipped_paths) for details.", + len(skipped_large), + max_file_size_bytes, + ) + if unreadable: + logger.warning( + "RepoNeedle could not read %d file(s) during indexing. See indexing diagnostics " + "(unreadable_paths) for details.", + len(unreadable), + ) + if max_files_reached: + logger.warning( + "RepoNeedle stopped indexing at max_files=%d; remaining files were not indexed.", + max_files, + ) + + return engine + def retrieve( self, query: Union[str, Query], @@ -300,13 +518,26 @@ def retrieve( ranker: Optional[Union[str, RankerType]] = None, ranking_engine: Optional[Union[str, RankerType]] = None, change_impact_config: Optional[ChangeImpactConfig] = None, + anchor_files: Optional[List[str]] = None, + enable_experimental_abstention: Optional[bool] = None, ) -> RetrievalResult: """ Executes workflow-adaptive retrieval and packs results within the token budget. Supports controlled ablation overrides and modular ranker architectures. + + enable_experimental_abstention: + Per-call override for the experimental, uncalibrated abstention heuristic. + None (default) uses ``budget_config.enable_experimental_abstention`` (False unless + configured). Passing ``task_type=TaskType.SELECTIVE`` does NOT enable abstention. + Empty repositories always return an abstained result regardless of this flag. """ start_time = time.perf_counter() ablation = ablation_config or self.ablation_config or AblationConfig() + if enable_experimental_abstention is None: + abstention_enabled = bool(getattr(self.budget_config, "enable_experimental_abstention", False)) + else: + abstention_enabled = bool(enable_experimental_abstention) + abstention_threshold = float(self.abstainer.confidence_threshold) raw_ranker = ranking_engine if ranking_engine is not None else ranker if raw_ranker is not None: @@ -322,10 +553,19 @@ def retrieve( f"Unknown ranker '{effective_ranker}'. Valid rankers are: 'legacy', 'bm25', 'arm101', 'cascade', 'change_impact'." ) - if isinstance(query, str): + if anchor_files: + norm_anchors = [a.replace("\\", "/") for a in anchor_files] + if isinstance(query, str): + query_obj = Query(text=query, anchor_files=norm_anchors) + else: + query_obj = query + query_obj.anchor_files = norm_anchors + elif isinstance(query, str): query_obj = Query(text=query) else: query_obj = query + if query_obj.anchor_files: + query_obj.anchor_files = [a.replace("\\", "/") for a in query_obj.anchor_files] if task_type: query_obj.task_type = task_type @@ -349,6 +589,12 @@ def retrieve( execution_time_ms=(time.perf_counter() - start_time) * 1000, ranker=effective_ranker, ranked_files=[], + abstention=AbstentionDiagnostics( + enabled=abstention_enabled, + status="empty_repository", + threshold=abstention_threshold, + reason="Repository contains no candidate files (structural; independent of the experimental heuristic).", + ), ) num_docs = len(self.docs) @@ -394,6 +640,9 @@ def retrieve( workflow_boosts = np.zeros(num_docs, dtype=np.float32) elif effective_ranker == "change_impact": + # AUTO queries report the router-classified task (1.1.0 behaviour on main); explicit task + # types are reported unchanged. Reporting only: the change-impact ranker receives `query_obj`, + # which is not modified here. if query_obj.task_type == TaskType.AUTO: inferred_task, _ = self.router.classify_intent(query_obj) else: @@ -508,15 +757,37 @@ def retrieve( ranked_paths = [self.docs[i].path for i in ranked_indices] - # 3. Selective Abstention Check - abstained, confidence, reason = self.abstainer.evaluate_abstention( + # 3. Experimental Selective Abstention Check (opt-in only). + # The heuristic score is always computed so `confidence_score` keeps its existing meaning + # (an uncalibrated signal), but it only decides the outcome when explicitly enabled. + heuristic_abstained, confidence, reason = self.abstainer.evaluate_abstention( query=query_obj.text, fused_scores=fused_scores, ) exec_time = (time.perf_counter() - start_time) * 1000 - if abstained: + if abstention_enabled: + abstention_diag = AbstentionDiagnostics( + enabled=True, + status="evaluated", + threshold=abstention_threshold, + reason=reason, + ) + outcome_note = f"Experimental abstention check passed (uncalibrated heuristic): {reason}" + else: + abstention_diag = AbstentionDiagnostics( + enabled=False, + status="disabled", + threshold=abstention_threshold, + reason=( + "Experimental abstention disabled (default); out-of-scope queries are not " + "detected and results are returned regardless of query scope." + ), + ) + outcome_note = "Experimental abstention disabled." + + if abstention_enabled and heuristic_abstained: return RetrievalResult( query=query_obj.text, task_type=inferred_task, @@ -526,10 +797,11 @@ def retrieve( total_tokens=0, budget_tokens=eff_budget, candidate_count=num_docs, - explanation=f"Abstained: {reason}", + explanation=f"Abstained: {reason} (Experimental abstention heuristic; not production-calibrated.)", execution_time_ms=exec_time, ranker=effective_ranker, ranked_files=ranked_paths, + abstention=abstention_diag, ) # 4. Budget-Aware Context Packing (ordered by ranked_indices) @@ -572,9 +844,10 @@ def retrieve( total_tokens=total_tokens_used, budget_tokens=eff_budget, candidate_count=num_docs, - explanation=f"Retrieved {len(selected)} files ({total_tokens_used}/{eff_budget} tokens). Ranker: {effective_ranker}. Task: {inferred_task.value}. {reason}", + explanation=f"Retrieved {len(selected)} files ({total_tokens_used}/{eff_budget} tokens). Ranker: {effective_ranker}. Task: {inferred_task.value}. {outcome_note}", execution_time_ms=(time.perf_counter() - start_time) * 1000, ranker=effective_ranker, ranked_files=ranked_paths, rendered_context=rendered_context, + abstention=abstention_diag, ) diff --git a/hf_artifacts/space/reponeedle/models.py b/hf_artifacts/space/reponeedle/models.py index 764ded2..a2c96d5 100644 --- a/hf_artifacts/space/reponeedle/models.py +++ b/hf_artifacts/space/reponeedle/models.py @@ -67,6 +67,64 @@ class ScoredFile(BaseModel): explanation: str = "" +class AbstentionDiagnostics(BaseModel): + """ + Describes how (and whether) the experimental abstention heuristic influenced a retrieval. + + status values: + "disabled" - Heuristic not applied (default). Results are returned regardless of + whether the query is in scope for the repository. + "evaluated" - Caller opted in; the heuristic decided whether to abstain. + "empty_repository" - No candidate files exist; the result is abstained structurally, + independent of the heuristic and of the opt-in flag. + + The heuristic is experimental and NOT production-calibrated (ARB v2 selective split: + 0 of 82 negative samples correctly abstained). `experimental` and `calibrated` are + reported on every result so downstream consumers never mistake it for a calibrated signal. + """ + enabled: bool = False + status: str = "disabled" + experimental: bool = True + calibrated: bool = False + threshold: float = 0.12 + reason: str = "" + + +class IndexingDiagnostics(BaseModel): + """ + Records what was (and was not) indexed. Never stores file contents: only + repository-relative paths and counts. + + source: + "directory" - built by RepoNeedleEngine.from_directory (scan statistics are populated). + "documents" - built from in-memory FileDoc objects (no scan was performed). + """ + source: str = "documents" + indexed_files: int = 0 + max_file_size_bytes: Optional[int] = None + skipped_large_files: int = 0 + skipped_paths: List[str] = Field(default_factory=list) + skipped_unreadable_files: int = 0 + unreadable_paths: List[str] = Field(default_factory=list) + # Files (symlinks or, on Windows, junction members) whose canonical location is outside the + # repository root. Reason code: "symlink_outside_repository". Only the in-repository link path + # is recorded; the target path and contents are never read or stored. + skipped_symlink_outside_repository: int = 0 + symlink_outside_repository_paths: List[str] = Field(default_factory=list) + # Directories that were not traversed. Directory symlinks and Windows junctions are never + # followed (on every platform); files beneath an in-repository link target are still indexed + # through their real path. Directories that would revisit an already-traversed directory + # (cycles) or whose canonical location is outside the repository root are also skipped. + # Only the in-repository path is recorded; link targets are never listed or stored. + skipped_directories: int = 0 + skipped_directory_paths: List[str] = Field(default_factory=list) + # Subset of skipped_directories whose canonical location is outside the repository root + # (or cannot be resolved, e.g. broken links). + skipped_directories_outside_repository: int = 0 + max_files: Optional[int] = None + max_files_reached: bool = False + + class RetrievalResult(BaseModel): query: str task_type: TaskType @@ -81,6 +139,8 @@ class RetrievalResult(BaseModel): ranker: Optional[str] = "legacy" ranked_files: Optional[List[str]] = None rendered_context: str = "" + # Added in 1.2.0 (optional; None for results built outside RepoNeedleEngine.retrieve). + abstention: Optional[AbstentionDiagnostics] = None @property def packed_files(self) -> List[ScoredFile]: @@ -94,6 +154,11 @@ class BudgetConfig(BaseModel): skeleton_token_cap: int = 350 density_exponent: float = 0.75 abstention_threshold: float = 0.12 + # Experimental, uncalibrated out-of-scope abstention heuristic. Disabled by default since 1.2.0. + # When False, retrieval never abstains on heuristic grounds (empty repositories still abstain). + # Selecting TaskType.SELECTIVE does NOT enable it; this flag (or the per-call override on + # RepoNeedleEngine.retrieve) is the only opt-in. + enable_experimental_abstention: bool = False mode: str = "auto" # "auto", "compact", "complete" capacity_elasticity: Optional[float] = None candidate_pool_cutoff: int = 85 diff --git a/hf_artifacts/space/requirements.txt b/hf_artifacts/space/requirements.txt index b8e8253..aeaec28 100644 --- a/hf_artifacts/space/requirements.txt +++ b/hf_artifacts/space/requirements.txt @@ -1,4 +1,7 @@ gradio>=4.20.0,<5.0.0 +# Gradio 4.x fails to serve pages with Starlette 1.x (TemplateResponse signature change); +# fastapi<0.113 keeps Starlette at 0.38.x. Verified with gradio 4.29.0 and 4.44.1. +fastapi<0.113 huggingface_hub<0.25.0 numpy>=1.24.0 scipy>=1.10.0 diff --git a/hf_artifacts/static_space/README.md b/hf_artifacts/static_space/README.md index a29cd61..fdccf14 100644 --- a/hf_artifacts/static_space/README.md +++ b/hf_artifacts/static_space/README.md @@ -8,9 +8,9 @@ pinned: false license: mit --- -# RepoNeedle Playground (V1.1.0) +# RepoNeedle Playground (V1.2.0) -This is the static browser interface for the RepoNeedle V1.1.0 public demonstration. It calls a separately hosted FastAPI service; the static Space does not execute the Python retrieval engine itself. +This is the static browser interface for the RepoNeedle 1.2.0 public demonstration. It calls a separately hosted FastAPI service; the static Space does not execute the Python retrieval engine itself. **Live backend:** `https://reponeedle.onrender.com` **Source repository:** https://github.com/krishika08/RepoNeedle @@ -25,6 +25,8 @@ If you find RepoNeedle useful, consider supporting its continued development. The public Playground uses small embedded demonstration snapshots for `pallets/flask` and `psf/requests`. It does not clone arbitrary GitHub repositories, accept repository uploads, or reproduce the complete ARB v2 evaluation. Benchmark results described in the source repository were obtained in separate documented evaluations; this web demo is not itself a benchmark reproduction. +In RepoNeedle 1.2.0 the experimental abstention heuristic is **off by default**, so ranked context is returned for every query. The **Experimental abstention** checkbox (unchecked by default) opts in. The heuristic is uncalibrated: on the ARB v2 selective split it correctly abstained on 0 of 82 out-of-scope queries. + The browser sends retrieval requests to the public API. Queries are processed in memory and are not persisted by the demo API. The API is unauthenticated and intended for demonstration use, not production workloads. The free backend may sleep after inactivity, causing a cold-start delay. ## Reproducing the Space diff --git a/hf_artifacts/static_space/index.html b/hf_artifacts/static_space/index.html index 9d181e6..5507bfa 100644 --- a/hf_artifacts/static_space/index.html +++ b/hf_artifacts/static_space/index.html @@ -5,7 +5,7 @@ - RepoNeedle Playground · V1.1.0 + RepoNeedle Playground · V1.2.0