refactor(retrieval): clean up legacy MapNav contracts

AGENTS.md

@@ -101,10 +101,10 @@ flowchart TB
     end
 
     subgraph RETRIEVE["⑤ Retrieval (shared)"]
-        Query["GET /v1/retrieval/query"] --> Pipeline["run_retrieval_query"]
+        Query["POST /v1|/v2 retrieval/query"] --> Pipeline["run_retrieval_query"]
         Pipeline --> Classic["classic_topk / small_corpus (use_agentic=False)"]
         Pipeline --> MapNav["mapnav checklist (default / use_agentic≠False)"]
-        Classic --> Channels["3-Channel BM25 (path/content/term)"]
+        Classic --> Channels["map_unit_discovery: path+content BM25 -> RRF"]
         Channels --> Rank["rank_retrieval_candidates"]
         MapNav --> NavSnap["nav_snapshot + run_nav_episode"]
         NavSnap --> Bridge["nav_bridge referenced_chunks"]
@@ -546,7 +546,7 @@ debug CSVs (`preds_*.csv`) are saved alongside for troubleshooting.
 Core retrieval internals are grouped by ownership:
 
 - `execution/`: request shaping, route selection (classic / mapnav / small_corpus), and public response projection.
-- `search/`: lexical channels, scoring, section filters, candidate ranking, and classic `bottom_discovery`.
+- `search/`: `map_unit_discovery` (persisted map-unit BM25 discovery, with a legacy chunk-level PG FTS fallback), scoring, section filters, candidate ranking.
 - `hydration/`: row/path/reference hydration, inline assets, and result assembly.
 - `nav/` + `nav_*.py`: map-nav checklist episode (PLANNER / HARVEST / CONTROL).
 - `trace/`: `DecisionTraceStep` mapping and `TraceRecorder`.
@@ -555,24 +555,33 @@ Core retrieval internals are grouped by ownership:
 
 ### Two Retrieval Modes
 
-Per-request `use_agentic`: `False` → classic 3-channel top-K; `None`/`True` → map-nav (default).
+Per-request `use_agentic`: `False` → classic top-K (map-unit BM25); `None`/`True` → map-nav (default).
 
-#### Classic Mode (3-Channel RRF)
+#### Classic Mode (map-unit BM25 + legacy FTS fallback)
+
+Primary path is `search.map_unit_discovery.map_unit_discovery`: Python BM25Okapi
+over the persisted `document_map_unit_tokens` index, path and content channels
+only, fused by RRF.
 
 ```mermaid
 flowchart LR
-    Q[Query] --> P[Path Channel: BM25 on path_search_text]
-    Q --> C[Content Channel: BM25 on content_search_text]
-    Q --> T[Term Channel: substring on term_search_text]
+    Q[Query] --> P["Path channel: BM25 over document_map_unit_tokens (channel=path)"]
+    Q --> C["Content channel: BM25 over document_map_unit_tokens (channel=content)"]
     P --> RRF["RRF Fusion (k=60)"]
     C --> RRF
-    T --> RRF
     RRF --> Rank[rank_retrieval_candidates]
     Rank --> Assemble[hydration.result_assembly]
 ```
 
-**Channel weights** (default): path=1.0, content=2.0, term=1.5
-**RRF formula**: `score = weight / (k + rank + 1)` per channel, summed across channels.
+**Channel weights** (default): path=1.0, content=2.0. **RRF formula**:
+`score = weight / (k + rank + 1)` per channel, summed across channels, `k=60`.
+
+There is no scored term channel in this primary path. `term_search_text` /
+`term_search_text_lower` are persisted at publish time but are only read by
+the **legacy fallback** (`_legacy_chunk_discovery`), which runs only when a
+revision's map-unit index is missing or incomplete: a single SQL query
+scoring `GREATEST(ts_rank_cd(path_search_tsv), 2 * ts_rank_cd(content_search_tsv))`
+OR `term_search_text LIKE '%query%'`, not three independently-ranked channels.
 
 #### Map-nav Mode (default)
 

docs/design/agent-corpus-schema.md

@@ -0,0 +1,10 @@
+# Agent Corpus Schema
+
+**Status:** Design in progress
+The agent-facing corpus schema and tool-usage guidance has a single source
+of truth. The same text is intended to be shipped verbatim as both the API
+`/mcp` server `instructions` and the `agent_explore` system prompt:
+
+[`packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md`](../../packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md)
+
+Do not copy its content here — edit that file, not this pointer.

packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md

@@ -0,0 +1,135 @@
+# Knowhere Corpus Schema (agent-facing)
+
+Single source of truth for how an agent should understand and explore a
+Knowhere corpus through the tools in this package. This text is intended to
+be shipped verbatim as both the API `/mcp` server instructions and the
+`agent_explore` system prompt — do not duplicate it elsewhere; edit here.
+
+This describes the **published, DB-served corpus** (`documents`,
+`document_sections`, `document_chunks`, `graph_nodes`, `graph_edges`) that
+the tools below are designed to query. It is not the on-disk parse artifact schema
+(`chunks.json` / `doc_nav.json` on disk use different join keys and a
+different path separator) — that schema is for the parser pipeline, not for
+this tool set.
+
+---
+
+## 1. Corpus model
+
+```text
+Namespace
+  └─ Document        (document_id, source_file_name, parse_track)
+       └─ Section     (section_id, parent_section_id, section_path, section_level, summary)
+            └─ Chunk  (chunk_id, chunk_type, content, chunk_metadata)
+```
+
+- **Namespace**: the retrieval scope. One namespace can hold documents parsed
+  by different tracks (see §2) — do not assume a namespace is uniform.
+- **Document**: `document_id` is the stable identifier for every other tool
+  call. `parse_track` is `page_memory` or `chunk` (see §2).
+- **Section**: the navigable tree. `section_path` segments are joined with
+  `" / "` (space-slash-space), one segment per heading/synthetic level.
+- **Chunk**: the retrieval unit. `chunk_type` is `text`, `page`, `image`, or
+  `table`. A section owns **at most one body chunk** (`text` or `page`) —
+  but which sections get one is track-dependent (see §2). `image`/`table`
+  chunks are **not** attached to the section they are conceptually "in" —
+  see §3.
+
+Document-level relationships (`related` edges, including entity-overlap
+scoring) exist in a separate graph — see §4.
+
+## 2. Body chunks: `text` vs `page`, and the `SAME-AS` pointer
+
+A namespace can mix both tracks across documents (e.g. one PDF on the
+`page_memory` track next to one DOCX/XLSX on the `chunk` track). Treat
+`text` and `page` as the same *role* — both are a section's own body — but
+they differ in shape, and in **which sections get a body chunk at all**:
+
+- **`chunk` track** (`text` chunks, e.g. DOCX/XLSX/MD): a section owns a
+  body chunk if it has any content of its own directly under its heading,
+  before any child heading — this can be **any section, leaf or not**. Do
+  not assume a section with children has no body text of its own; check
+  whether it owns a chunk instead of assuming from tree position.
+- **`page_memory` track** (`page` chunks, PDF): only **leaf** sections own a
+  body chunk. Internal/structural sections never carry a `page` chunk
+  themselves — their summaries aggregate from their leaf descendants.
+
+For `page` chunks specifically: one leaf section's body may span one or
+more physical pages. A page's text is stored **once**, under whichever leaf
+is first in reading order to cover that page (the "owner"). Every other
+leaf section that also covers that physical page has, in place of the
+text, a literal marker:
+
+  ```text
+  [SAME-AS <owner_section_path> p<page_num>]
+  ```
+
+  This is a pointer, not a preview. If you need that page's actual text,
+  resolve the marker by reading the owner section's chunk at that page
+  number — do not treat the marker's absence of text as "this section has
+  no content there." `page` chunks also carry `page_nums` (all physical
+  pages they cover) and `page_assets` (rendered page-citation screenshots —
+  these are references for citation, not separate `image` chunks).
+
+  **Format trap**: `<owner_section_path>` inside the marker is written
+  verbatim by the parser and stored as-is — it is the on-disk path
+  (`"<source_file_name>/<Heading>/<Heading>/..."`, plain `/`, filename
+  included), **not** the DB `section_path` you get back from `outline` /
+  `node_filter` / `recall` (which is `" / "`-joined and excludes the
+  filename). Do not string-match the marker directly against a DB
+  `section_path`. Convert it first — `section_path_from_chunk_path()` in
+  `search/lexical_text.py` already does this conversion and is the function
+  to reuse when implementing marker resolution, not a new one.
+
+## 3. Asset chunks: `image` / `table`, and `connect_to`
+
+`image` and `table` chunks are **not children of the section they visually
+belong to**. In the DB they are parked under their document's synthetic
+`Root` section. The real association to a body section is the `connect_to`
+list on the **body chunk**, not a location on the asset:
+
+- `relation: "embeds"` — the body chunk that owns/embeds this asset inline.
+- `relation: "related"` — another body chunk that shares the same source
+  page as a page-track asset, without owning it (`same_as_owner` may name
+  the owning section).
+
+This link is **one-directional** (body → asset). There is no stored
+asset → body back-link; to find which section(s) an asset belongs to, use
+the reverse lookup on the `assets` tool rather than assuming the asset chunk
+itself names its host.
+
+## 4. Document graph
+
+Today the graph only has **document-level** nodes (`node_kind='document'`)
+and undirected `related` edges between documents. Edge scoring prefers
+**typed-entity overlap** between the two documents' aggregated `entities`
+first; it only falls back to free-form TF-IDF keyword overlap when either
+document lacks entities. Either way the edge carries which terms matched
+(`properties.shared_entities` or `properties.shared_keywords`). There are
+no section-level or entity-level graph *nodes* — an entity is not itself a
+queryable node, and there is no entity-to-entity or entity-to-chunk edge,
+only this document-to-document rollup. Use `neighbors` for "what else is
+like this document" (and to see which shared terms justify that link), not
+for anything finer-grained than a document pair.
+
+## 5. Reserved / not yet available
+
+- **Vector channel**: `recall`'s `channels` parameter reserves a `vector`
+  option; it does not exist yet. `recall` today is lexical only.
+
+## 6. Tools and when to use each
+
+| Tool | Use when | Scope | Notes |
+|---|---|---|---|
+| `list_documents` | Starting cold: which documents exist, what are they about | namespace | Returns per-document keywords/summary/type mix/`parse_track`. |
+| `outline` | The task only needs titles/summaries — overview, "what does chapter N cover," picking where to look before reading | one document, or a `section_path` prefix within it | Titles + summaries + `chunk_count`, no body text, no folding. Depth-limited by argument, not by a token budget. Use this to build your own map instead of relying on a pre-folded one. |
+| `node_filter` | The task is a traversal/exclusion predicate — FOR ALL / EXISTS / ANY / NOT — over section titles or summaries ("which docs mention X in a heading," "sections NOT about Y") | one or more documents | Deterministic substring/regex match against `section_path` and `summary` only, not body text. Returns the full matching set and count, never a truncated top-K. If the predicate must run against body text, use `grep` instead. |
+| `grep` | Exact string / regex / identifier / number lookup that must run against body text | scoped by document/section/chunk_type | Returns match count plus snippets, so ANY/ALL logic can also close over body text, not just titles. |
+| `recall` | A fuzzy question where you don't know where the answer lives | namespace or scoped | Ranked candidates (path + content BM25 today; term and vector are separate/reserved — see §5) with path and snippet, not full content. |
+| `read` | You already know which section(s)/chunk(s) to read | one or more sections/chunks | Returns full body content, resolves `SAME-AS` markers into the owner's text, expands `connect_to` assets, and converts `page_assets` into URLs. |
+| `assets` | You need images/tables directly, or need to find which section(s) host a given asset | one or more documents | Forward (by type/query) and reverse (asset → hosting section) lookup — see §3. |
+| `neighbors` | You need related documents in the same namespace | one document | Document-level `related` edges only — see §4. |
+
+General rule: prefer `outline` / `node_filter` to locate before `read`ing
+body text; prefer `grep` over `recall` when you know the exact string you
+are looking for; only fall back to `recall` for genuinely fuzzy questions.

packages/shared-python/shared/services/retrieval/execution/routes.py

@@ -193,7 +193,15 @@ async def _run_classic_topk_route(
 async def _run_mapnav_route(
     context: RetrievalRouteContext,
 ) -> RetrievalRouteOutcome:
-    """Default agentic path: PLANNER + HARVEST + CONTROL (checklist map-nav)."""
+    """Default agentic path: PLANNER + HARVEST + CONTROL (checklist map-nav).
+
+    LEGACY, PENDING REPLACEMENT: ``agent_explore`` will become the default
+    agentic route once it passes its evaluation gate; this route then stays
+    only as the ``RETRIEVAL_AGENTIC_ROUTER=mapnav`` fallback until Phase 5
+    cleanup. Do not add new capabilities here — new agentic-retrieval work
+    belongs in ``shared/services/retrieval/agent_tools/`` and
+    ``shared/services/retrieval/agent_explore/``.
+    """
     process_started = resource.getrusage(resource.RUSAGE_SELF)
     from shared.services.retrieval import nav_llm_backend  # noqa: F401
     from shared.services.retrieval.nav import run_nav_episode

packages/shared-python/shared/services/retrieval/nav/__init__.py

@@ -1,11 +1,24 @@
-"""Recursive-dispatch map navigation for RealData experiments."""
+"""Recursive-dispatch map navigation for RealData experiments.
+
+LEGACY, PENDING REPLACEMENT: this package implements the map-nav
+PLANNER/HARVEST/CONTROL episode
+(``run_nav_episode``), the current default agentic retrieval route via
+``execution.routes._run_mapnav_route``. It will be superseded by
+``agent_explore`` after its evaluation gate, then trimmed in
+Phase 5 to whatever this package still owns and nothing else uses (the BM25
+scorer in ``knowhere_hybrid.py``, the ``NodeFilter`` predicate in
+``nav_node_filter.py``, and snapshot loading are already planned to be
+reused by the new ``agent_tools/``, not deleted). Do not add new PLANNER /
+HARVEST / CONTROL capabilities here; new agentic-retrieval work belongs in
+``shared/services/retrieval/agent_tools/`` and
+``shared/services/retrieval/agent_explore/``.
+"""
 
 from .nav_types import (
     ActionKind,
     NavConfig,
     NavState,
     SubgoalResult,
-    map_mode_enabled,
 )
 from .nav_agent import run_nav_episode
 from .nav_plan import (
@@ -22,7 +35,6 @@
     "NavConfig",
     "NavState",
     "SubgoalResult",
-    "map_mode_enabled",
     "run_nav_episode",
     "Contract",
     "RetrievalPlan",

packages/shared-python/shared/services/retrieval/nav/_compat.py

@@ -41,7 +41,6 @@ class EpisodeResult:
     section_ids: List[str] = field(default_factory=list)
     trajectory_length: int = 0
     truncated_last: bool = False
-    refusal_events: List[Dict[str, object]] = field(default_factory=list)
     phase_timings: Dict[str, float] = field(default_factory=dict)
     stop_reason: str = "completed"
 
@@ -51,10 +50,6 @@ class EpisodeResult:
 HierarchicalTools = Any
 
 
-class Refusal(Exception):
-    """Raised by experimental ToolSpace; unused on the ProviderToolSpace path."""
-
-
 def line_node_id(doc_id: str, line_id: int) -> str:
     """Experiment-corpus helper; ProviderToolSpace never hits this path."""
     raise NotImplementedError(

packages/shared-python/shared/services/retrieval/nav/nav_actions.py

@@ -20,24 +20,12 @@ def _env_enabled(name: str, default: str = "1") -> bool:
     return os.environ.get(name, default).strip().lower() not in {"0", "false", "no", "off"}
 
 
-def _budget_mode(step_idx: int, config: NavConfig, *, max_steps: Optional[int] = None) -> str:
-    episode_steps = int(max_steps if max_steps is not None else config.max_steps)
-    remaining = max(0, episode_steps - step_idx)
-    if remaining <= config.critical_remaining_steps:
-        return "critical"
-    if remaining <= config.tight_remaining_steps:
-        return "tight"
-    return "normal"
-
-
 def build_legal_actions(
     state: NavState,
     projection: Projection,
     *,
-    step_idx: int,
     config: NavConfig,
     depth: int = 0,
-    max_steps: Optional[int] = None,
     ts: Any = None,
 ) -> List[LegalAction]:
     """Every visible node is actionable: COLLECT + DISPATCH (when allowed) + FINISH.
@@ -47,8 +35,6 @@ def build_legal_actions(
     DISPATCH never targets the current scope root (no self-dispatch loop).
     Document / namespace nodes are DISPATCH-only (level registry via ``ts``).
     """
-    episode_steps = int(max_steps if max_steps is not None else config.max_steps)
-    mode = _budget_mode(step_idx, config, max_steps=episode_steps)
     actions: List[LegalAction] = []
     filter_collected = _env_enabled("NAV_FILTER_COLLECTED_SECTIONS")
     collected_sids = set(state.collected_section_ids) | {
@@ -115,7 +101,6 @@ def view_score(view: SectionView) -> float:
             and view.has_children
             and sid not in collected_sids
             and sid != scope_id
-            and mode != "critical"
         ):
             actions.append(
                 LegalAction(