From 3df1c3168e9987ce971605f7eebd2622901fa52c Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Thu, 3 Sep 2026 09:57:22 +0800 Subject: [PATCH 1/3] refactor(retrieval): clean up navigation configuration and remove unused parameters Removed hard-coded parameters from the navigation configuration, including budget modes and projection limits. Updated related functions to streamline the navigation process and improve clarity. Adjusted tests to reflect these changes and ensure functionality remains intact. --- .../shared/services/retrieval/nav/__init__.py | 2 - .../shared/services/retrieval/nav/_compat.py | 5 - .../services/retrieval/nav/nav_actions.py | 15 -- .../services/retrieval/nav/nav_agent.py | 38 ++-- .../services/retrieval/nav/nav_harvest.py | 6 +- .../services/retrieval/nav/nav_hierarchy.py | 14 +- .../services/retrieval/nav/nav_map_scores.py | 2 +- .../services/retrieval/nav/nav_node_filter.py | 19 +- .../shared/services/retrieval/nav/nav_plan.py | 1 - .../services/retrieval/nav/nav_projection.py | 164 +----------------- .../retrieval/nav/nav_scope_filter.py | 26 +-- .../services/retrieval/nav/nav_types.py | 70 +++----- .../services/retrieval/nav/nav_verify.py | 13 +- .../shared/services/retrieval/nav_config.py | 20 +-- .../shared/services/retrieval/trace/mapnav.py | 1 - .../shared/tests/test_nav_node_filter.py | 1 - .../shared/tests/test_nav_node_filter_wire.py | 4 +- .../shared/tests/test_nav_projection_prod.py | 13 -- .../shared/tests/test_nav_scope_filter.py | 9 +- .../shared/tests/test_nav_trace_map.py | 2 +- 20 files changed, 68 insertions(+), 357 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/nav/__init__.py b/packages/shared-python/shared/services/retrieval/nav/__init__.py index ca9233eee..897fd00a9 100644 --- a/packages/shared-python/shared/services/retrieval/nav/__init__.py +++ b/packages/shared-python/shared/services/retrieval/nav/__init__.py @@ -5,7 +5,6 @@ NavConfig, NavState, SubgoalResult, - map_mode_enabled, ) from .nav_agent import run_nav_episode from .nav_plan import ( @@ -22,7 +21,6 @@ "NavConfig", "NavState", "SubgoalResult", - "map_mode_enabled", "run_nav_episode", "Contract", "RetrievalPlan", diff --git a/packages/shared-python/shared/services/retrieval/nav/_compat.py b/packages/shared-python/shared/services/retrieval/nav/_compat.py index 8c6e1644a..7d6e31dba 100644 --- a/packages/shared-python/shared/services/retrieval/nav/_compat.py +++ b/packages/shared-python/shared/services/retrieval/nav/_compat.py @@ -41,7 +41,6 @@ class EpisodeResult: section_ids: List[str] = field(default_factory=list) trajectory_length: int = 0 truncated_last: bool = False - refusal_events: List[Dict[str, object]] = field(default_factory=list) phase_timings: Dict[str, float] = field(default_factory=dict) stop_reason: str = "completed" @@ -51,10 +50,6 @@ class EpisodeResult: HierarchicalTools = Any -class Refusal(Exception): - """Raised by experimental ToolSpace; unused on the ProviderToolSpace path.""" - - def line_node_id(doc_id: str, line_id: int) -> str: """Experiment-corpus helper; ProviderToolSpace never hits this path.""" raise NotImplementedError( diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_actions.py b/packages/shared-python/shared/services/retrieval/nav/nav_actions.py index dc7566228..8af8b07d9 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_actions.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_actions.py @@ -20,24 +20,12 @@ def _env_enabled(name: str, default: str = "1") -> bool: return os.environ.get(name, default).strip().lower() not in {"0", "false", "no", "off"} -def _budget_mode(step_idx: int, config: NavConfig, *, max_steps: Optional[int] = None) -> str: - episode_steps = int(max_steps if max_steps is not None else config.max_steps) - remaining = max(0, episode_steps - step_idx) - if remaining <= config.critical_remaining_steps: - return "critical" - if remaining <= config.tight_remaining_steps: - return "tight" - return "normal" - - def build_legal_actions( state: NavState, projection: Projection, *, - step_idx: int, config: NavConfig, depth: int = 0, - max_steps: Optional[int] = None, ts: Any = None, ) -> List[LegalAction]: """Every visible node is actionable: COLLECT + DISPATCH (when allowed) + FINISH. @@ -47,8 +35,6 @@ def build_legal_actions( DISPATCH never targets the current scope root (no self-dispatch loop). Document / namespace nodes are DISPATCH-only (level registry via ``ts``). """ - episode_steps = int(max_steps if max_steps is not None else config.max_steps) - mode = _budget_mode(step_idx, config, max_steps=episode_steps) actions: List[LegalAction] = [] filter_collected = _env_enabled("NAV_FILTER_COLLECTED_SECTIONS") collected_sids = set(state.collected_section_ids) | { @@ -115,7 +101,6 @@ def view_score(view: SectionView) -> float: and view.has_children and sid not in collected_sids and sid != scope_id - and mode != "critical" ): actions.append( LegalAction( diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_agent.py b/packages/shared-python/shared/services/retrieval/nav/nav_agent.py index 7df6eca34..68b2a721b 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_agent.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_agent.py @@ -10,7 +10,6 @@ from ._compat import HierarchicalTools from ._compat import Chunk from ._compat import ( - Refusal, ToolSpace, ) from .nav_address import ( @@ -34,6 +33,9 @@ _unit_score_for_evidence_chunk = unit_score_for_evidence_chunk _logger = logging.getLogger(__name__) +# Large-scope title-only threshold = episode evidence budget x this mult. +_SCOPE_SUMMARY_BUDGET_MULT = 3.0 + def _chunks_to_retrieved_nodes(chunks: List[Chunk]) -> List[str]: """Stable unit ids: prefer chunk/node id (Knowhere ``chunk_id``).""" @@ -133,8 +135,7 @@ def _line_order(pool: List[Chunk]) -> List[Chunk]: def _collect_subtree(ts: ToolSpace, action: LegalAction, state: NavState, config: NavConfig) -> List[Tuple[Chunk, float]]: """Hydrate ``section_id ∪ descendants`` in document order. - No collect-time top-K / unit-score truncation — final size is controlled by - compose ``budget_chars`` progressive trim (MAP-NAV subtree collect). + Final size is controlled by compose ``budget_chars`` progressive trim. """ sid = action.section_id if not sid: @@ -145,23 +146,12 @@ def _collect_subtree(ts: ToolSpace, action: LegalAction, state: NavState, config if not doc: return [] materialize = getattr(ts, "_materialize_leaf_path_chunks", None) - if callable(materialize): - pool = list(materialize(sid, doc)) - if pool: - return _collect_in_doc_order(pool, config) - rc = ts.read_chunks(sid, state.query, doc_id=doc, k=int(config.collect_k)) - if isinstance(rc, Refusal): - state.refusal_events.append( - { - "tool": "collect", - "section_id": sid, - "status": rc.status, - "message": rc.message, - "available_sections": list(rc.available_sections), - } - ) + if not callable(materialize): return [] - return [(h.chunk, float(h.score) + float(config.read_score_bonus)) for h in rc] + pool = list(materialize(sid, doc)) + if not pool: + return [] + return _collect_in_doc_order(pool, config) def _mark_collected_branch( @@ -221,7 +211,7 @@ def _direct_child_ids(ts: ToolSpace, section_id: str, doc_id: str) -> List[str]: rows: List[Any] = [] if callable(children_fn): try: - rows = list(children_fn(sid, doc_id, limit=100000) or []) + rows = list(children_fn(sid, doc_id) or []) except Exception: rows = [] if not rows: @@ -370,9 +360,10 @@ def _run_nav_episode_body( # Tie the large-scope title-only threshold to the real evidence budget # (budget_chars x mult): a scope whose full summary map would dwarf the final # evidence budget is shown title-only, nudging DISPATCH over broad COLLECT. - mult = float(getattr(cfg, "scope_inline_summary_budget_mult", 0.0) or 0.0) - if mult > 0.0 and int(budget_chars) > 0: - cfg.scope_inline_summary_char_limit = max(1, int(budget_chars * mult)) + if int(budget_chars) > 0: + cfg.scope_inline_summary_char_limit = max( + 1, int(budget_chars * _SCOPE_SUMMARY_BUDGET_MULT) + ) retrieval_t0 = time.perf_counter() if toolspace is not None: ts = toolspace @@ -510,7 +501,6 @@ def _run_nav_episode_body( section_ids=list(section_ids), trajectory_length=len(steps), truncated_last=fill.truncated_last, - refusal_events=list(state.refusal_events), phase_timings={ "retrieval_framework_seconds": retrieval_seconds, "compose_seconds": compose_seconds, diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_harvest.py b/packages/shared-python/shared/services/retrieval/nav/nav_harvest.py index dbd276384..9fc894e38 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_harvest.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_harvest.py @@ -29,7 +29,7 @@ from .nav_compose import parse_collect_confidence from .nav_plan import Subgoal from .nav_policy import _extract_json_obj # reuse: same tolerant JSON extraction -from .nav_projection import build_projection +from .nav_projection import build_map from .nav_types import ActionKind, LegalAction, NavConfig, NavState, Projection _HARVEST_PURPOSE_DEPTH0 = "nav_harvest_v1" @@ -312,7 +312,7 @@ def _harvest_node( max_depth = max(0, int(getattr(config, "max_harvest_depth", 0) or 0)) show_harvested = bool(config.is_checklist) subgoal_dismissed = state.subgoal_dismissed_section_ids.get(subgoal.id, set()) - projection = build_projection( + projection = build_map( ts, doc_id=state.doc_id, query=query, @@ -326,7 +326,7 @@ def _harvest_node( allowed_section_ids=allowed_section_ids, ) actions = build_legal_actions( - state, projection, step_idx=0, config=config, depth=depth, ts=ts + state, projection, config=config, depth=depth, ts=ts ) actionable = [a for a in actions if a.kind != ActionKind.FINISH] if not actionable: diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py b/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py index ead3dfc6e..577de0147 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py @@ -5,8 +5,8 @@ top-level sections, a node's children, a node's structural metadata (title/summary/chunk count), a node's ancestor/descendant ids, and a node's full-subtree text as one evidence unit. Everything else this codebase's -``ToolSpace`` exposes (BM25/dense scoring, ``read_chunks``, ``_idx``, -``corpus_doc_ids``) is optional — every caller already reaches it through +``ToolSpace`` exposes (BM25/dense scoring, ``_idx``, ``corpus_doc_ids``) is +optional — every caller already reaches it through ``getattr(ts, "...", None)`` / ``callable(...)`` guards, so omitting it only degrades ranking quality, never breaks the pipeline. @@ -145,11 +145,9 @@ def get_structure(self, section_id: str) -> dict: } def _children_for_section_path( - self, section_id: str, doc_id: str, limit: Optional[int] = None + self, section_id: str, doc_id: str ) -> List[dict]: child_ids = [str(c) for c in self._provider.children(section_id)] - if limit is not None: - child_ids = child_ids[: max(0, int(limit))] # ``node_meta`` may materialize a lazy subtree to calculate chunk # counts. Tree traversal needs only the child id/title; avoid an N+1 # payload load while building the scoring tree. @@ -311,12 +309,6 @@ def _materialize_leaf_path_chunks(self, section_id: str, doc_id: str) -> List[An out.sort(key=lambda c: (min(c.line_ids or (0,)), c.node_id)) return out - def read_chunks( - self, section_id: str, query: str, *, doc_id: str, k: int - ) -> List[Any]: - del section_id, query, doc_id, k - return [] - def load_persisted_score_corpus( self, doc_ids: Sequence[str], diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py index eb207ce3a..35ffd890d 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py @@ -115,7 +115,7 @@ def _children_ids(ts: Any, section_id: str, doc_id: str) -> List[str]: return [ str(r.get("section_id") or "").strip() for r in rows if r.get("section_id") ] - rows = children_fn(section_id, doc_id, limit=100000) + rows = children_fn(section_id, doc_id) return [str(r.get("section_id") or "").strip() for r in rows if r.get("section_id")] diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_node_filter.py b/packages/shared-python/shared/services/retrieval/nav/nav_node_filter.py index 9de897603..45b2c1eb1 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_node_filter.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_node_filter.py @@ -2,9 +2,7 @@ Agent-authored predicates run on ``path`` (filename + title chain via ``path_titles``) and ``summary``. Field predicates AND together; terms inside -one field OR together. No top-K and no result truncation for substring -matches. Regex is bounded by pattern length, node count, and compile/search -exceptions. +one field OR together. No top-K and no result truncation. """ from __future__ import annotations @@ -17,7 +15,6 @@ FilterField = Literal["path", "summary"] _MAX_REGEX_PATTERN_LEN = 256 -_MAX_REGEX_NODES = 100_000 @dataclass(frozen=True) @@ -37,7 +34,6 @@ class FilterResult: matched_section_ids: List[str] matched_doc_ids: List[str] cardinality: int - truncated: bool = False failed_predicates: List[str] = field(default_factory=list) @@ -73,7 +69,6 @@ def apply_node_filter( matched_section_ids=[], matched_doc_ids=[], cardinality=0, - truncated=False, failed_predicates=failed, ) summaries = _load_summaries(ts) @@ -81,16 +76,9 @@ def apply_node_filter( matched_docs: List[str] = [] seen_sections: set[str] = set() seen_docs: set[str] = set() - visited = 0 - truncated = False - uses_regex = any(pred.match == "regex" for pred in nf.predicates) for doc_id in wanted: for sid, owner_doc, is_doc_node in _iter_doc_nodes(ts, doc_id): - visited += 1 - if uses_regex and visited > _MAX_REGEX_NODES: - truncated = True - break path_text = _path_text(ts, sid, owner_doc) summary_text = "" if is_doc_node else str(summaries.get(sid) or "") if not is_doc_node and not summary_text: @@ -110,14 +98,11 @@ def apply_node_filter( if owner_doc and owner_doc not in seen_docs: seen_docs.add(owner_doc) matched_docs.append(owner_doc) - if truncated: - break return FilterResult( matched_section_ids=matched_sections, matched_doc_ids=matched_docs, cardinality=len(matched_sections), - truncated=truncated, failed_predicates=failed, ) @@ -131,8 +116,6 @@ def render_submap_observation( """Hit-count line plus every matched node (path + summary).""" del doc_ids header = f"hits={result.cardinality}" - if result.truncated: - header = f"{header} truncated=true" if result.failed_predicates: header = f"{header} failed_predicates={len(result.failed_predicates)}" if result.cardinality == 0: diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_plan.py b/packages/shared-python/shared/services/retrieval/nav/nav_plan.py index cf7aef876..49c211704 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_plan.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_plan.py @@ -602,7 +602,6 @@ def build_planning_observation( actions = build_legal_actions( state, projection, - step_idx=0, config=plan_cfg, depth=0, ts=ts, diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_projection.py b/packages/shared-python/shared/services/retrieval/nav/nav_projection.py index b4ffd5794..536ec1bb1 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_projection.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_projection.py @@ -1,11 +1,9 @@ from __future__ import annotations -import math -import re from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Set -from .nav_types import NavConfig, Projection, SectionView, map_mode_enabled +from .nav_types import NavConfig, Projection, SectionView try: from section_summary_store import get_summary as _store_get_summary @@ -42,47 +40,7 @@ def _section_summary_for_map( return "" -def _tokens(text: str) -> set[str]: - return set(re.findall(r"[\w\u4e00-\u9fff]+", (text or "").lower())) - - -def _lexical_score(query: str, text: str) -> float: - q = _tokens(query) - if not q: - return 0.0 - t = _tokens(text) - if not t: - return 0.0 - inter = len(q & t) - if inter == 0: - return 0.0 - return float(inter) / math.sqrt(float(len(q) * len(t))) - - -def _section_view_from_structure( - ts: Any, - section_id: str, - *, - query: str, - depth_from_scope: int, - summary_chars: int, -) -> SectionView: - st = ts.get_structure(section_id) - preview = str(st.get("preview") or "").replace("\n", " ")[:summary_chars] - children = st.get("children") or [] - return SectionView( - section_id=str(st.get("section_id") or section_id), - level=int(st.get("level") or 0) if str(st.get("level") or "").isdigit() else 0, - preview=preview, - score=_lexical_score(query, f"{section_id} {preview}"), - n_lines=int(st.get("n_lines") or 0), - has_children=bool(children), - depth_from_scope=depth_from_scope, - title=preview[:80] if preview else section_id, - ) - - -def _children(ts: Any, section_id: str, *, limit: int) -> List[dict]: +def _children(ts: Any, section_id: str) -> List[dict]: children_fn = getattr(ts, "_children_for_section_path", None) if callable(children_fn): loc = getattr(ts, "_idx", None) @@ -95,13 +53,13 @@ def _children(ts: Any, section_id: str, *, limit: int) -> List[dict]: synth = getattr(ts, "_synthetic_doc_id", lambda _x: None)(section_id) doc_id = synth or "" if doc_id: - rows = children_fn(section_id, doc_id, limit=max(1, int(limit))) + rows = children_fn(section_id, doc_id) if isinstance(rows, list): return [c for c in rows if isinstance(c, dict)] st = ts.get_structure(section_id) children = st.get("children") or [] if isinstance(children, list): - return [c for c in children if isinstance(c, dict)][: max(0, int(limit))] + return [c for c in children if isinstance(c, dict)] return [] @@ -310,8 +268,6 @@ def _build_map_tree( *, root_ids: List[str], map_scores: Dict[str, float], - children_limit: int, - max_nodes: int = 20000, collected_section_ids: Optional[Set[str]] = None, dismissed_section_ids: Optional[Set[str]] = None, harvested_section_ids: Optional[Dict[str, str]] = None, @@ -319,7 +275,6 @@ def _build_map_tree( ) -> List[_MapNode]: roots: List[_MapNode] = [] seen: Set[str] = set() - node_count = 0 harvested = dict(harvested_section_ids or {}) keep = set(keep_ids) if keep_ids is not None else None # collected = branch done (sid ∪ descendants already marked by caller). @@ -331,13 +286,11 @@ def _build_map_tree( ) | set(dismissed_section_ids or ()) def make_node(section_id: str, depth: int, parent_id: Optional[str]) -> Optional[_MapNode]: - nonlocal node_count - if not section_id or section_id in seen or node_count >= max_nodes: + if not section_id or section_id in seen: return None if section_id in gone: return None seen.add(section_id) - node_count += 1 try: st = ts.get_structure(section_id) except Exception: @@ -376,7 +329,7 @@ def append_visible_descendants( # Collapsed leaf: this line alone represents the covered branch. node.harvested_by = str(harvested[section_id]) else: - for row in _children(ts, section_id, limit=children_limit): + for row in _children(ts, section_id): child_id = str(row.get("section_id") or "").strip() if child_id: append_visible_descendants( @@ -554,7 +507,6 @@ def build_map( ts, root_ids=root_ids, map_scores=scores, - children_limit=max(1, int(config.map_children_limit)), collected_section_ids=collected_section_ids, dismissed_section_ids=dismissed_section_ids, harvested_section_ids=harvested_section_ids, @@ -583,7 +535,7 @@ def build_map( and _estimate_actionable_total(roots, with_summary=True) <= scope_summary_limit ) - char_limit = max(1, int(config.map_char_limit or config.projection_char_limit)) + char_limit = max(1, int(config.map_char_limit)) _apply_budget_hide( roots, char_limit=char_limit, @@ -617,107 +569,5 @@ def build_map( ) -def build_projection( - ts: Any, - *, - doc_id: str, - query: str, - scope: Optional[str], - config: NavConfig, - map_scores: Optional[Dict[str, float]] = None, - collected_section_ids: Optional[Set[str]] = None, - dismissed_section_ids: Optional[Set[str]] = None, - highlight_ids: Optional[List[str]] = None, - extra_hidden_ids: Optional[Set[str]] = None, - harvested_section_ids: Optional[Dict[str, str]] = None, - allowed_section_ids: Optional[Set[str]] = None, -) -> Projection: - if map_mode_enabled(config): - return build_map( - ts, - doc_id=doc_id, - query=query, - scope=scope, - config=config, - map_scores=map_scores, - collected_section_ids=collected_section_ids, - dismissed_section_ids=dismissed_section_ids, - highlight_ids=highlight_ids, - extra_hidden_ids=extra_hidden_ids, - harvested_section_ids=harvested_section_ids, - allowed_section_ids=allowed_section_ids, - ) - - # Minimal non-map fallback (legacy shallow projection) — kept for ablation only. - visible: List[SectionView] = [] - lines: List[str] = [] - truncated = False - - def add_line(text: str) -> None: - nonlocal truncated - if truncated: - return - candidate_len = sum(len(x) + 1 for x in lines) + len(text) - if candidate_len > config.projection_char_limit: - lines.append("... [projection truncated]") - truncated = True - return - lines.append(text) - - add_line(f"doc_id={doc_id}") - add_line(f"scope={scope or ''}") - - if scope: - root_ids = [scope] - else: - root_ids = _top_sections(ts, doc_id) - - collected = set(collected_section_ids or ()) - root_ids = root_ids[: max(1, config.projection_child_limit)] - frontier: List[tuple[str, int]] = [(sid, 0) for sid in root_ids] - seen: set[str] = set() - while frontier: - sid, depth = frontier.pop(0) - if sid in seen or sid in collected: - continue - seen.add(sid) - try: - view = _section_view_from_structure( - ts, - sid, - query=query, - depth_from_scope=depth, - summary_chars=config.summary_chars, - ) - except Exception: - continue - visible.append(view) - indent = " " * depth - leaf_tag = " [Leaf]" if not view.has_children else "" - title = view.preview[:80] if view.preview else view.section_id - add_line( - f"{indent}[{view.section_id}] {title}{leaf_tag}" - ) - if view.preview: - add_line(f"{indent} Preview: \"{view.preview[:80]}\"") - if depth + 1 >= max(1, config.projection_depth): - continue - child_rows = _children(ts, sid, limit=max(0, config.projection_child_limit)) - for child in child_rows: - child_id = str(child.get("section_id") or "").strip() - if child_id and child_id not in seen: - frontier.append((child_id, depth + 1)) - - visible.sort(key=lambda v: (-v.score, v.depth_from_scope, v.section_id)) - return Projection( - doc_id=doc_id, - scope=scope, - text="\n".join(lines), - visible_sections=visible, - truncated=truncated, - map_mode=False, - ) - - def top_visible_sections(projection: Projection, *, limit: int) -> List[SectionView]: return list(projection.visible_sections[: max(0, limit)]) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_scope_filter.py b/packages/shared-python/shared/services/retrieval/nav/nav_scope_filter.py index c0be92e9a..886a05276 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_scope_filter.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_scope_filter.py @@ -2,7 +2,7 @@ The policy writes or revises a ``NodeFilter``. Apply is deterministic. The loop stops at ``filter_max_rounds``, token exhaustion, explicit fallback, or -a done-in-band settle. Decision is cardinality-driven; the agent may override. +agent ``done``. The agent may override settle; otherwise hits → scoped harvest. """ from __future__ import annotations @@ -85,8 +85,6 @@ def run_scope_filter( from .nav_token_budget import nav_token_budget_exhausted, stamp_step_detail max_rounds = max(1, int(getattr(config, "filter_max_rounds", 3) or 3)) - min_hits = max(0, int(getattr(config, "filter_min_hits", 1) or 0)) - max_hits = max(min_hits, int(getattr(config, "filter_max_hits", 40) or 0)) wanted = [str(did).strip() for did in doc_ids if str(did).strip()] map_text = str(map_observation or "").strip() or _compact_map(ts, wanted) current = seed_filter @@ -132,7 +130,6 @@ def run_scope_filter( result = apply_node_filter(ts, wanted, current) last_result = result last_obs = render_submap_observation(ts, result, doc_ids=wanted) - in_band = min_hits <= result.cardinality <= max_hits if steps_out is not None: from ._compat import AgentStep @@ -148,7 +145,6 @@ def run_scope_filter( {p.field for p in current.predicates} ), "cardinality": result.cardinality, - "truncated": result.truncated, "failed_predicates": list(result.failed_predicates), "matched_section_ids": list(result.matched_section_ids), "action": "", @@ -165,11 +161,9 @@ def run_scope_filter( steps_out[-1].detail["action"] = "max_rounds" return _settle( result, - in_band=in_band, agent_decision=last_decision, - min_hits=min_hits, rounds=round_idx, - reason=("max_rounds" if in_band else "max_rounds_out_of_band"), + reason="max_rounds", steps_out=steps_out, ) @@ -224,9 +218,7 @@ def run_scope_filter( if kind == "done": return _settle( result, - in_band=in_band, agent_decision=last_decision, - min_hits=min_hits, rounds=round_idx, reason=last_reason or "done", steps_out=steps_out, @@ -236,12 +228,9 @@ def run_scope_filter( current = nxt assert last_result is not None - in_band = min_hits <= last_result.cardinality <= max_hits return _settle( last_result, - in_band=in_band, agent_decision=last_decision, - min_hits=min_hits, rounds=max_rounds, reason=last_reason or "max_rounds", steps_out=steps_out, @@ -251,27 +240,22 @@ def run_scope_filter( def _settle( result: FilterResult, *, - in_band: bool, agent_decision: Optional[ScopeDecision], - min_hits: int, rounds: int, reason: str, steps_out: Optional[List[Any]], ) -> ScopeFilterOutcome: - if not in_band or result.cardinality <= 0: + if result.cardinality <= 0: decision: ScopeDecision = "fallback" - settle_reason = reason or "out_of_band" + settle_reason = reason or "no_hits" elif agent_decision in _DECISIONS: decision = agent_decision settle_reason = reason or "agent" if decision == "fallback": settle_reason = reason or "agent_fallback" - elif result.cardinality <= min_hits: - decision = "collect_all" - settle_reason = reason or "small_cardinality" else: decision = "scoped_harvest" - settle_reason = reason or "medium_cardinality" + settle_reason = reason or "hits" if steps_out: steps_out[-1].detail["decision"] = decision steps_out[-1].detail["reason"] = settle_reason diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_types.py b/packages/shared-python/shared/services/retrieval/nav/nav_types.py index aebee1428..987a7e604 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_types.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_types.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os from dataclasses import dataclass, field from enum import Enum from typing import Any, Dict, List, Literal, Optional, Tuple @@ -14,33 +13,8 @@ class ActionKind(str, Enum): FINISH = "finish" -def map_mode_enabled(config: "NavConfig | None" = None) -> bool: - """True when map-first observation/actions are active. - - When ``config`` is provided it is authoritative (Knowhere production binds - ``map_mode`` on ``NavConfig``). Env ``NAV_MAP_MODE`` is only for EXP scripts - that call without a config. - """ - if config is not None: - return bool(getattr(config, "map_mode", False)) - return os.environ.get("NAV_MAP_MODE", "0").strip().lower() not in { - "0", - "false", - "no", - "off", - "", - } - - @dataclass class NavConfig: - projection_depth: int = 2 - projection_child_limit: int = 8 - projection_char_limit: int = 8000 - summary_chars: int = 120 - max_steps: int = 8 - collect_k: int = 64 - search_k: int = 40 collect_top_k: int = 6 # rescue-K for highlights (not action quota) read_score_bonus: float = 10.0 policy: str = "rule" @@ -49,22 +23,18 @@ class NavConfig: llm_model: str = "" llm_temperature: float = 0.0 llm_max_tokens: int = 256 - critical_remaining_steps: int = 1 - tight_remaining_steps: int = 2 - # Map-first mode (also gated by NAV_MAP_MODE env). + # Map-first observation/actions. map_mode: bool = False map_char_limit: int = 5000 # display budget (fold threshold); only hard display limit - map_children_limit: int = 10000 # Recursive dispatch. enable_recursive_dispatch: bool = True max_dispatch_depth: int = 3 subagent_model: str = "" # Scoped maps whose estimated (with-summary) size exceeds this threshold drop # inline summaries (title-only), nudging the agent to DISPATCH deeper rather - # than broadly COLLECT the whole parent. Default 1500 == evidence budget 500 x3. - # run_nav_episode re-derives it from the episode's evidence budget x mult below. + # than broadly COLLECT the whole parent. run_nav_episode re-derives it from + # the episode's evidence budget (budget_chars x _SCOPE_SUMMARY_BUDGET_MULT). scope_inline_summary_char_limit: int = 1500 - scope_inline_summary_budget_mult: float = 3.0 # COMPOSE child score = own_unit + compose_confidence_weight * collect_confidence # (see nav_compose._child_final_score); drives group_key / within-group rank. compose_confidence_weight: float = 0.5 @@ -95,14 +65,9 @@ class NavConfig: max_waves: int = 0 # Structural recursion depth cap for harvest() (checklist mode). max_harvest_depth: int = 3 - # Retired: plan_control now shows full prebuilt section summaries (already - # head/tail clipped at summary-build time), not a raw-evidence char cut. - plan_control_digest_chars: int = 600 # WHERE node filter (pre-harvest). Off until orchestrate enables a subgoal. enable_node_filter: bool = False filter_max_rounds: int = 3 - filter_min_hits: int = 1 - filter_max_hits: int = 40 @property def is_checklist(self) -> bool: @@ -111,14 +76,6 @@ def is_checklist(self) -> bool: @classmethod def from_dict(cls, data: Dict[str, Any]) -> "NavConfig": flat = dict(data) - budget_modes = flat.pop("budget_modes", {}) or {} - if isinstance(budget_modes, dict): - flat["critical_remaining_steps"] = int( - budget_modes.get("critical_remaining_steps", cls.critical_remaining_steps) - ) - flat["tight_remaining_steps"] = int( - budget_modes.get("tight_remaining_steps", cls.tight_remaining_steps) - ) # Retired product flags (dropped after the navigate loop was removed). for dead in ( "expand_top_k", @@ -129,6 +86,9 @@ def from_dict(cls, data: Dict[str, Any]) -> "NavConfig": "map_collapse_min_score", "dispatch_group_size", "dispatch_max_workers", + "projection_depth", + "projection_child_limit", + "projection_char_limit", "enable_contract_verify", "enable_per_subgoal_illumination", "enable_goal_conditioned_folding", @@ -158,6 +118,23 @@ def from_dict(cls, data: Dict[str, Any]) -> "NavConfig": "enable_depth0_oversize_to_dispatch", "depth0_oversize_char_limit", "compose_group_rank_max_chars", + # Never wired to any consumer. + "search_k", + "plan_control_digest_chars", + # Legacy shallow-projection leftovers. + "summary_chars", + "map_children_limit", + # Budget-mode step gating: build_legal_actions always ran at + # step_idx=0, so critical/tight never triggered. + "max_steps", + "budget_modes", + "critical_remaining_steps", + "tight_remaining_steps", + # Folded into nav_agent._SCOPE_SUMMARY_BUDGET_MULT. + "scope_inline_summary_budget_mult", + "collect_k", + "filter_min_hits", + "filter_max_hits", ): flat.pop(dead, None) flat["mode"] = "checklist" @@ -255,7 +232,6 @@ class NavState: collected_section_ids: set[str] = field(default_factory=set) blocked_collect_section_ids: set[str] = field(default_factory=set) action_history: List[Dict[str, Any]] = field(default_factory=list) - refusal_events: List[Dict[str, Any]] = field(default_factory=list) dismissed_section_ids: set[str] = field(default_factory=set) # Explicit COLLECT confidence by section_id; hydration-only descendants stay 0. collect_confidence: Dict[str, float] = field(default_factory=dict) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_verify.py b/packages/shared-python/shared/services/retrieval/nav/nav_verify.py index 50f1a09e4..aa198ab96 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_verify.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_verify.py @@ -91,7 +91,7 @@ def extract_slots_llm( f"Slots to fill: {json.dumps(names, ensure_ascii=False)}\n" f"Contract: {contract_kind}" + (f", cardinality={cardinality}" if cardinality is not None else "") - + f"\n\n=== Evidence ===\n{evidence_text[:6000]}\n=== End Evidence ===\n" + + f"\n\n=== Evidence ===\n{evidence_text}\n=== End Evidence ===\n" ) import time @@ -248,17 +248,12 @@ def apply_bindings_from_result( return out -def build_evidence_text_from_chunks(chunks: Any, *, limit: int = 8000) -> str: - """Concatenate (chunk, score) texts up to a char budget.""" +def build_evidence_text_from_chunks(chunks: Any) -> str: + """Concatenate (chunk, score) texts in order.""" parts: List[str] = [] - total = 0 for chunk, _score in list(chunks or []): text = str(getattr(chunk, "text", "") or getattr(chunk, "content", "") or "") if not text.strip(): continue - if total >= limit: - break - take = text[: max(0, limit - total)] - parts.append(take) - total += len(take) + parts.append(text) return "\n".join(parts) diff --git a/packages/shared-python/shared/services/retrieval/nav_config.py b/packages/shared-python/shared/services/retrieval/nav_config.py index df5e9b5e5..1bf06c5e0 100644 --- a/packages/shared-python/shared/services/retrieval/nav_config.py +++ b/packages/shared-python/shared/services/retrieval/nav_config.py @@ -18,16 +18,8 @@ MAPNAV_PLANNER_THINK_MAX_TOKENS = 16_384 MAPNAV_TRACE_RAW_CHARS = 2_000 -# Hard-coded enabled product config (checklist + map). Matches EXP -# config/nav_default.json + cfg_shared() (mode=checklist, map_mode=True). +# Hard-coded enabled product config (checklist + map). _PRODUCTION_NAV_DICT: dict[str, Any] = { - "projection_depth": 2, - "projection_child_limit": 8, - "projection_char_limit": 8000, - "summary_chars": 120, - "max_steps": 8, - "collect_k": 64, - "search_k": 40, "collect_top_k": 6, "read_score_bonus": 10.0, "policy": "llm", @@ -36,18 +28,11 @@ "llm_max_tokens": 256, "planner_llm_max_tokens": 1024, "harvest_llm_max_tokens": 1024, - "budget_modes": { - "critical_remaining_steps": 1, - "tight_remaining_steps": 2, - }, "map_mode": True, "map_char_limit": 5000, - "map_children_limit": 10000, "enable_recursive_dispatch": True, "max_dispatch_depth": 3, "subagent_model": MAPNAV_MODEL, - "scope_inline_summary_char_limit": 1500, - "scope_inline_summary_budget_mult": 3.0, "compose_confidence_weight": 0.5, "mode": "checklist", "planning_map_char_limit": 10000, @@ -60,11 +45,8 @@ "max_replans": 1, "max_waves": 0, "max_harvest_depth": 3, - "plan_control_digest_chars": 600, "enable_node_filter": True, "filter_max_rounds": 3, - "filter_min_hits": 1, - "filter_max_hits": 40, } diff --git a/packages/shared-python/shared/services/retrieval/trace/mapnav.py b/packages/shared-python/shared/services/retrieval/trace/mapnav.py index 6a2e83244..3d97659d7 100644 --- a/packages/shared-python/shared/services/retrieval/trace/mapnav.py +++ b/packages/shared-python/shared/services/retrieval/trace/mapnav.py @@ -208,7 +208,6 @@ def _map_one( "predicates": detail.get("predicates") or [], "fields": detail.get("fields") or [], "cardinality": detail.get("cardinality"), - "truncated": detail.get("truncated"), "failed_predicates": detail.get("failed_predicates") or [], "matched_section_ids": detail.get("matched_section_ids") or [], "round": detail.get("round"), diff --git a/packages/shared-python/shared/tests/test_nav_node_filter.py b/packages/shared-python/shared/tests/test_nav_node_filter.py index 7e059d6c4..ef662b646 100644 --- a/packages/shared-python/shared/tests/test_nav_node_filter.py +++ b/packages/shared-python/shared/tests/test_nav_node_filter.py @@ -97,7 +97,6 @@ def test_path_filter_returns_complete_set_across_documents() -> None: node_filter([field_predicate("path", ["AAPL", "Apple"])]), ) - assert result.truncated is False assert result.failed_predicates == [] assert result.matched_doc_ids == ["doc_apple"] assert "sec_q3" in result.matched_section_ids diff --git a/packages/shared-python/shared/tests/test_nav_node_filter_wire.py b/packages/shared-python/shared/tests/test_nav_node_filter_wire.py index c3d0d7ebb..2615d71dc 100644 --- a/packages/shared-python/shared/tests/test_nav_node_filter_wire.py +++ b/packages/shared-python/shared/tests/test_nav_node_filter_wire.py @@ -12,7 +12,7 @@ ) from shared.services.retrieval.nav.nav_orchestrate import _execute_subgoal_harvest_once from shared.services.retrieval.nav.nav_plan import RetrievalPlan, Subgoal -from shared.services.retrieval.nav.nav_projection import build_projection +from shared.services.retrieval.nav.nav_projection import build_map from shared.services.retrieval.nav.nav_scope_filter import ScopeFilterOutcome from shared.services.retrieval.nav.nav_types import NavConfig, NavState @@ -296,7 +296,7 @@ def fake_harvest(*args: Any, **kwargs: Any) -> Any: def test_projection_keeps_allowed_and_ancestors() -> None: ts = _ts() cfg = _cfg() - projection = build_projection( + projection = build_map( ts, doc_id="", query="profit", diff --git a/packages/shared-python/shared/tests/test_nav_projection_prod.py b/packages/shared-python/shared/tests/test_nav_projection_prod.py index 9d74127bc..e9b5e4e93 100644 --- a/packages/shared-python/shared/tests/test_nav_projection_prod.py +++ b/packages/shared-python/shared/tests/test_nav_projection_prod.py @@ -18,27 +18,14 @@ _section_summary_for_map, build_map, ) -from shared.services.retrieval.nav.nav_types import NavConfig, map_mode_enabled from shared.services.retrieval.nav_config import build_nav_config from shared.services.retrieval.nav_snapshot import build_nav_snapshot -def test_map_mode_enabled_trusts_config_over_env(monkeypatch: Any) -> None: - monkeypatch.setenv("NAV_MAP_MODE", "0") - cfg_on = NavConfig(map_mode=True) - cfg_off = NavConfig(map_mode=False) - assert map_mode_enabled(cfg_on) is True - assert map_mode_enabled(cfg_off) is False - monkeypatch.setenv("NAV_MAP_MODE", "1") - assert map_mode_enabled(cfg_off) is False - assert map_mode_enabled(None) is True - - def test_build_nav_config_authoritative_for_production() -> None: cfg = build_nav_config() assert cfg.map_mode is True assert cfg.mode == "checklist" - assert map_mode_enabled(cfg) is True def test_section_summary_falls_back_to_provider_structure() -> None: diff --git a/packages/shared-python/shared/tests/test_nav_scope_filter.py b/packages/shared-python/shared/tests/test_nav_scope_filter.py index b762e085e..2228d362c 100644 --- a/packages/shared-python/shared/tests/test_nav_scope_filter.py +++ b/packages/shared-python/shared/tests/test_nav_scope_filter.py @@ -82,8 +82,6 @@ def _cfg(**kwargs: Any) -> NavConfig: data = { "enable_node_filter": True, "filter_max_rounds": 3, - "filter_min_hits": 1, - "filter_max_hits": 40, "llm_model": "test-model", "llm_max_tokens": 256, } @@ -194,7 +192,7 @@ def test_too_many_hits_tighten(monkeypatch: Any) -> None: ) out = run_scope_filter( _ts(), - _cfg(filter_max_hits=1), + _cfg(), query="everything", doc_ids=["doc_apple", "doc_other"], seed_filter=node_filter([field_predicate("path", ["Root"])]), @@ -231,7 +229,7 @@ def test_max_rounds_hard_stop(monkeypatch: Any) -> None: ) assert out.decision == "fallback" assert out.rounds == 2 - assert out.reason == "max_rounds_out_of_band" + assert out.reason == "max_rounds" def test_cardinality_drives_scoped_harvest(monkeypatch: Any) -> None: @@ -241,12 +239,11 @@ def test_cardinality_drives_scoped_harvest(monkeypatch: Any) -> None: ) out = run_scope_filter( _ts(), - _cfg(filter_min_hits=1, filter_max_hits=40), + _cfg(), query="apple", doc_ids=["doc_apple", "doc_other"], seed_filter=node_filter([field_predicate("path", ["AAPL"])]), ) - # filename + Root + Q3 → more than min_hits → scoped_harvest assert out.decision == "scoped_harvest" assert out.rounds == 1 assert "sec_q3" in out.settled_section_ids diff --git a/packages/shared-python/shared/tests/test_nav_trace_map.py b/packages/shared-python/shared/tests/test_nav_trace_map.py index e25f6cfe2..55a2e0500 100644 --- a/packages/shared-python/shared/tests/test_nav_trace_map.py +++ b/packages/shared-python/shared/tests/test_nav_trace_map.py @@ -140,7 +140,7 @@ def test_node_filter_steps_map_and_count_tokens() -> None: "cardinality": 2, "action": "done", "decision": "collect_all", - "reason": "small_cardinality", + "reason": "hits", "matched_section_ids": ["sec_q3"], "token_limit": 100000, "tokens_used_total": 80, From 9157fe0d02ed8a40ce6b3723cb558d59b0ea6ae5 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Thu, 3 Sep 2026 10:19:35 +0800 Subject: [PATCH 2/3] refactor(retrieval): update navigation configuration parameters and clean up code Removed the map_mode parameter from the navigation configuration and adjusted related functions to reflect this change. Increased max_dispatch_depth and max_harvest_depth values for improved navigation performance. Updated tests to ensure consistency with the new configuration settings. --- .../shared/services/retrieval/nav/nav_agent.py | 2 +- .../shared/services/retrieval/nav/nav_plan.py | 4 ++++ .../shared/services/retrieval/nav/nav_projection.py | 1 - .../shared/services/retrieval/nav/nav_types.py | 10 ++++------ .../shared/services/retrieval/nav_config.py | 8 +++----- .../shared/tests/test_nav_bridge_config.py | 3 +-- .../shared/tests/test_nav_node_filter_wire.py | 1 - .../shared/tests/test_nav_projection_prod.py | 1 - 8 files changed, 13 insertions(+), 17 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_agent.py b/packages/shared-python/shared/services/retrieval/nav/nav_agent.py index 68b2a721b..c7334ff9b 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_agent.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_agent.py @@ -347,7 +347,7 @@ def _run_nav_episode_body( load_llm_env() require_llm_env(context="Nav Agent") cfg = config or NavConfig(policy="llm") - if cfg.map_mode and cfg.llm_max_tokens < 256: + if cfg.llm_max_tokens < 256: cfg.llm_max_tokens = 256 nav_policy = (policy or cfg.policy or "llm").strip().lower() if nav_policy != "llm": diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_plan.py b/packages/shared-python/shared/services/retrieval/nav/nav_plan.py index 49c211704..6ff770022 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_plan.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_plan.py @@ -445,6 +445,8 @@ def parse_retrieval_plan( if not rq: rq = need or query produces = _as_str_list(row.get("produces")) + # TODO: revisit one-slot-per-subgoal hard clip (produces[:1]) — + # prompt + validate also enforce it; may want multi-slot later. if len(produces) > 1: produces = produces[:1] subgoals.append( @@ -469,6 +471,7 @@ def parse_retrieval_plan( s.prefer_after = [d for d in s.prefer_after if d in known and d != s.id] _apply_slot_dependency_inference(subgoals) + # TODO: same one-slot-per-subgoal policy as parse above; revisit with multi-slot. for s in subgoals: if len(s.produces) > 1: s.produces = s.produces[:1] @@ -556,6 +559,7 @@ def validate_retrieval_plan(plan: RetrievalPlan) -> Tuple[bool, str]: return False, f"bad_prefer_after:{s.id}->{d}" if s.contract.kind not in _CONTRACT_KINDS: return False, f"bad_contract:{s.id}" + # TODO: revisit multi_produces rejection with one-slot-per-subgoal clip. if len(s.produces) > 1: return False, f"multi_produces:{s.id}" for ref in unbound_slots(s.retrieval_query) + unbound_slots(s.need): diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_projection.py b/packages/shared-python/shared/services/retrieval/nav/nav_projection.py index 536ec1bb1..16b7f2a46 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_projection.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_projection.py @@ -563,7 +563,6 @@ def build_map( visible_sections=visible_sorted, truncated=truncated, id_to_section=id_map, - map_mode=True, tree_sections=list(tree_visible), highlight_ids=list(hits), ) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_types.py b/packages/shared-python/shared/services/retrieval/nav/nav_types.py index 987a7e604..323dac6a9 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_types.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_types.py @@ -23,12 +23,10 @@ class NavConfig: llm_model: str = "" llm_temperature: float = 0.0 llm_max_tokens: int = 256 - # Map-first observation/actions. - map_mode: bool = False map_char_limit: int = 5000 # display budget (fold threshold); only hard display limit # Recursive dispatch. enable_recursive_dispatch: bool = True - max_dispatch_depth: int = 3 + max_dispatch_depth: int = 5 subagent_model: str = "" # Scoped maps whose estimated (with-summary) size exceeds this threshold drop # inline summaries (title-only), nudging the agent to DISPATCH deeper rather @@ -64,7 +62,7 @@ class NavConfig: # Checklist: 0 = no extra wave cap (stop when no ready subgoals). max_waves: int = 0 # Structural recursion depth cap for harvest() (checklist mode). - max_harvest_depth: int = 3 + max_harvest_depth: int = 5 # WHERE node filter (pre-harvest). Off until orchestrate enables a subgoal. enable_node_filter: bool = False filter_max_rounds: int = 3 @@ -135,12 +133,13 @@ def from_dict(cls, data: Dict[str, Any]) -> "NavConfig": "collect_k", "filter_min_hits", "filter_max_hits", + "map_mode", ): flat.pop(dead, None) flat["mode"] = "checklist" allowed = {f.name for f in cls.__dataclass_fields__.values()} cfg = cls(**{k: v for k, v in flat.items() if k in allowed}) - if cfg.map_mode and cfg.llm_max_tokens < 256: + if cfg.llm_max_tokens < 256: cfg.llm_max_tokens = 256 return cfg @@ -174,7 +173,6 @@ class Projection: visible_sections: List[SectionView] truncated: bool = False # True if any budget-hidden nodes id_to_section: Dict[str, str] = field(default_factory=dict) - map_mode: bool = False tree_sections: List[SectionView] = field(default_factory=list) highlight_ids: List[str] = field(default_factory=list) diff --git a/packages/shared-python/shared/services/retrieval/nav_config.py b/packages/shared-python/shared/services/retrieval/nav_config.py index 1bf06c5e0..edf03dcb5 100644 --- a/packages/shared-python/shared/services/retrieval/nav_config.py +++ b/packages/shared-python/shared/services/retrieval/nav_config.py @@ -28,10 +28,9 @@ "llm_max_tokens": 256, "planner_llm_max_tokens": 1024, "harvest_llm_max_tokens": 1024, - "map_mode": True, "map_char_limit": 5000, "enable_recursive_dispatch": True, - "max_dispatch_depth": 3, + "max_dispatch_depth": 5, "subagent_model": MAPNAV_MODEL, "compose_confidence_weight": 0.5, "mode": "checklist", @@ -44,7 +43,7 @@ "subgoal_max_attempts": 2, "max_replans": 1, "max_waves": 0, - "max_harvest_depth": 3, + "max_harvest_depth": 5, "enable_node_filter": True, "filter_max_rounds": 3, } @@ -56,9 +55,8 @@ def nav_evidence_chars() -> int: def build_nav_config() -> NavConfig: - """Checklist + map_mode production config (enabled items only).""" + """Checklist + map production config (enabled items only).""" cfg = NavConfig.from_dict(dict(_PRODUCTION_NAV_DICT)) cfg.mode = "checklist" - cfg.map_mode = True cfg.policy = "llm" return cfg diff --git a/packages/shared-python/shared/tests/test_nav_bridge_config.py b/packages/shared-python/shared/tests/test_nav_bridge_config.py index 076b66d3c..89e8469d4 100644 --- a/packages/shared-python/shared/tests/test_nav_bridge_config.py +++ b/packages/shared-python/shared/tests/test_nav_bridge_config.py @@ -28,12 +28,11 @@ def test_build_nav_config_is_checklist_map_trim_stack() -> None: cfg = build_nav_config() assert cfg.mode == "checklist" assert cfg.is_checklist - assert cfg.map_mode is True assert cfg.policy == "llm" assert cfg.subgoal_max_attempts == 2 assert cfg.max_replans == 1 assert cfg.max_waves == 0 - assert cfg.max_harvest_depth == 3 + assert cfg.max_harvest_depth == 5 assert not hasattr(cfg, "compose_packing_mode") assert cfg.llm_model == MAPNAV_MODEL assert cfg.planner_model == MAPNAV_MODEL diff --git a/packages/shared-python/shared/tests/test_nav_node_filter_wire.py b/packages/shared-python/shared/tests/test_nav_node_filter_wire.py index 2615d71dc..f31297362 100644 --- a/packages/shared-python/shared/tests/test_nav_node_filter_wire.py +++ b/packages/shared-python/shared/tests/test_nav_node_filter_wire.py @@ -73,7 +73,6 @@ def _cfg(**kwargs: Any) -> NavConfig: data = { "enable_node_filter": True, "mode": "checklist", - "map_mode": True, "llm_model": "test-model", } data.update(kwargs) diff --git a/packages/shared-python/shared/tests/test_nav_projection_prod.py b/packages/shared-python/shared/tests/test_nav_projection_prod.py index e9b5e4e93..1d5219d67 100644 --- a/packages/shared-python/shared/tests/test_nav_projection_prod.py +++ b/packages/shared-python/shared/tests/test_nav_projection_prod.py @@ -24,7 +24,6 @@ def test_build_nav_config_authoritative_for_production() -> None: cfg = build_nav_config() - assert cfg.map_mode is True assert cfg.mode == "checklist" From 7f450775751411401872332df7431939197cd03f Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Tue, 8 Sep 2026 11:35:08 +0800 Subject: [PATCH 3/3] docs(retrieval): define agent corpus contract --- AGENTS.md | 31 ++-- docs/design/agent-corpus-schema.md | 10 ++ .../retrieval/agent_tools/CORPUS_SCHEMA.md | 135 ++++++++++++++++++ .../services/retrieval/execution/routes.py | 10 +- .../shared/services/retrieval/nav/__init__.py | 16 ++- .../shared/services/retrieval/nav_config.py | 6 + 6 files changed, 195 insertions(+), 13 deletions(-) create mode 100644 docs/design/agent-corpus-schema.md create mode 100644 packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md diff --git a/AGENTS.md b/AGENTS.md index 00c31b182..503a16ad8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -101,10 +101,10 @@ flowchart TB end subgraph RETRIEVE["⑤ Retrieval (shared)"] - Query["GET /v1/retrieval/query"] --> Pipeline["run_retrieval_query"] + Query["POST /v1|/v2 retrieval/query"] --> Pipeline["run_retrieval_query"] Pipeline --> Classic["classic_topk / small_corpus (use_agentic=False)"] Pipeline --> MapNav["mapnav checklist (default / use_agentic≠False)"] - Classic --> Channels["3-Channel BM25 (path/content/term)"] + Classic --> Channels["map_unit_discovery: path+content BM25 -> RRF"] Channels --> Rank["rank_retrieval_candidates"] MapNav --> NavSnap["nav_snapshot + run_nav_episode"] NavSnap --> Bridge["nav_bridge referenced_chunks"] @@ -546,7 +546,7 @@ debug CSVs (`preds_*.csv`) are saved alongside for troubleshooting. Core retrieval internals are grouped by ownership: - `execution/`: request shaping, route selection (classic / mapnav / small_corpus), and public response projection. -- `search/`: lexical channels, scoring, section filters, candidate ranking, and classic `bottom_discovery`. +- `search/`: `map_unit_discovery` (persisted map-unit BM25 discovery, with a legacy chunk-level PG FTS fallback), scoring, section filters, candidate ranking. - `hydration/`: row/path/reference hydration, inline assets, and result assembly. - `nav/` + `nav_*.py`: map-nav checklist episode (PLANNER / HARVEST / CONTROL). - `trace/`: `DecisionTraceStep` mapping and `TraceRecorder`. @@ -555,24 +555,33 @@ Core retrieval internals are grouped by ownership: ### Two Retrieval Modes -Per-request `use_agentic`: `False` → classic 3-channel top-K; `None`/`True` → map-nav (default). +Per-request `use_agentic`: `False` → classic top-K (map-unit BM25); `None`/`True` → map-nav (default). -#### Classic Mode (3-Channel RRF) +#### Classic Mode (map-unit BM25 + legacy FTS fallback) + +Primary path is `search.map_unit_discovery.map_unit_discovery`: Python BM25Okapi +over the persisted `document_map_unit_tokens` index, path and content channels +only, fused by RRF. ```mermaid flowchart LR - Q[Query] --> P[Path Channel: BM25 on path_search_text] - Q --> C[Content Channel: BM25 on content_search_text] - Q --> T[Term Channel: substring on term_search_text] + Q[Query] --> P["Path channel: BM25 over document_map_unit_tokens (channel=path)"] + Q --> C["Content channel: BM25 over document_map_unit_tokens (channel=content)"] P --> RRF["RRF Fusion (k=60)"] C --> RRF - T --> RRF RRF --> Rank[rank_retrieval_candidates] Rank --> Assemble[hydration.result_assembly] ``` -**Channel weights** (default): path=1.0, content=2.0, term=1.5 -**RRF formula**: `score = weight / (k + rank + 1)` per channel, summed across channels. +**Channel weights** (default): path=1.0, content=2.0. **RRF formula**: +`score = weight / (k + rank + 1)` per channel, summed across channels, `k=60`. + +There is no scored term channel in this primary path. `term_search_text` / +`term_search_text_lower` are persisted at publish time but are only read by +the **legacy fallback** (`_legacy_chunk_discovery`), which runs only when a +revision's map-unit index is missing or incomplete: a single SQL query +scoring `GREATEST(ts_rank_cd(path_search_tsv), 2 * ts_rank_cd(content_search_tsv))` +OR `term_search_text LIKE '%query%'`, not three independently-ranked channels. #### Map-nav Mode (default) diff --git a/docs/design/agent-corpus-schema.md b/docs/design/agent-corpus-schema.md new file mode 100644 index 000000000..a680fe019 --- /dev/null +++ b/docs/design/agent-corpus-schema.md @@ -0,0 +1,10 @@ +# Agent Corpus Schema + +**Status:** Design in progress +The agent-facing corpus schema and tool-usage guidance has a single source +of truth. The same text is intended to be shipped verbatim as both the API +`/mcp` server `instructions` and the `agent_explore` system prompt: + +[`packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md`](../../packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md) + +Do not copy its content here — edit that file, not this pointer. diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md b/packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md new file mode 100644 index 000000000..496b16074 --- /dev/null +++ b/packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md @@ -0,0 +1,135 @@ +# Knowhere Corpus Schema (agent-facing) + +Single source of truth for how an agent should understand and explore a +Knowhere corpus through the tools in this package. This text is intended to +be shipped verbatim as both the API `/mcp` server instructions and the +`agent_explore` system prompt — do not duplicate it elsewhere; edit here. + +This describes the **published, DB-served corpus** (`documents`, +`document_sections`, `document_chunks`, `graph_nodes`, `graph_edges`) that +the tools below are designed to query. It is not the on-disk parse artifact schema +(`chunks.json` / `doc_nav.json` on disk use different join keys and a +different path separator) — that schema is for the parser pipeline, not for +this tool set. + +--- + +## 1. Corpus model + +```text +Namespace + └─ Document (document_id, source_file_name, parse_track) + └─ Section (section_id, parent_section_id, section_path, section_level, summary) + └─ Chunk (chunk_id, chunk_type, content, chunk_metadata) +``` + +- **Namespace**: the retrieval scope. One namespace can hold documents parsed + by different tracks (see §2) — do not assume a namespace is uniform. +- **Document**: `document_id` is the stable identifier for every other tool + call. `parse_track` is `page_memory` or `chunk` (see §2). +- **Section**: the navigable tree. `section_path` segments are joined with + `" / "` (space-slash-space), one segment per heading/synthetic level. +- **Chunk**: the retrieval unit. `chunk_type` is `text`, `page`, `image`, or + `table`. A section owns **at most one body chunk** (`text` or `page`) — + but which sections get one is track-dependent (see §2). `image`/`table` + chunks are **not** attached to the section they are conceptually "in" — + see §3. + +Document-level relationships (`related` edges, including entity-overlap +scoring) exist in a separate graph — see §4. + +## 2. Body chunks: `text` vs `page`, and the `SAME-AS` pointer + +A namespace can mix both tracks across documents (e.g. one PDF on the +`page_memory` track next to one DOCX/XLSX on the `chunk` track). Treat +`text` and `page` as the same *role* — both are a section's own body — but +they differ in shape, and in **which sections get a body chunk at all**: + +- **`chunk` track** (`text` chunks, e.g. DOCX/XLSX/MD): a section owns a + body chunk if it has any content of its own directly under its heading, + before any child heading — this can be **any section, leaf or not**. Do + not assume a section with children has no body text of its own; check + whether it owns a chunk instead of assuming from tree position. +- **`page_memory` track** (`page` chunks, PDF): only **leaf** sections own a + body chunk. Internal/structural sections never carry a `page` chunk + themselves — their summaries aggregate from their leaf descendants. + +For `page` chunks specifically: one leaf section's body may span one or +more physical pages. A page's text is stored **once**, under whichever leaf +is first in reading order to cover that page (the "owner"). Every other +leaf section that also covers that physical page has, in place of the +text, a literal marker: + + ```text + [SAME-AS p] + ``` + + This is a pointer, not a preview. If you need that page's actual text, + resolve the marker by reading the owner section's chunk at that page + number — do not treat the marker's absence of text as "this section has + no content there." `page` chunks also carry `page_nums` (all physical + pages they cover) and `page_assets` (rendered page-citation screenshots — + these are references for citation, not separate `image` chunks). + + **Format trap**: `` inside the marker is written + verbatim by the parser and stored as-is — it is the on-disk path + (`"///..."`, plain `/`, filename + included), **not** the DB `section_path` you get back from `outline` / + `node_filter` / `recall` (which is `" / "`-joined and excludes the + filename). Do not string-match the marker directly against a DB + `section_path`. Convert it first — `section_path_from_chunk_path()` in + `search/lexical_text.py` already does this conversion and is the function + to reuse when implementing marker resolution, not a new one. + +## 3. Asset chunks: `image` / `table`, and `connect_to` + +`image` and `table` chunks are **not children of the section they visually +belong to**. In the DB they are parked under their document's synthetic +`Root` section. The real association to a body section is the `connect_to` +list on the **body chunk**, not a location on the asset: + +- `relation: "embeds"` — the body chunk that owns/embeds this asset inline. +- `relation: "related"` — another body chunk that shares the same source + page as a page-track asset, without owning it (`same_as_owner` may name + the owning section). + +This link is **one-directional** (body → asset). There is no stored +asset → body back-link; to find which section(s) an asset belongs to, use +the reverse lookup on the `assets` tool rather than assuming the asset chunk +itself names its host. + +## 4. Document graph + +Today the graph only has **document-level** nodes (`node_kind='document'`) +and undirected `related` edges between documents. Edge scoring prefers +**typed-entity overlap** between the two documents' aggregated `entities` +first; it only falls back to free-form TF-IDF keyword overlap when either +document lacks entities. Either way the edge carries which terms matched +(`properties.shared_entities` or `properties.shared_keywords`). There are +no section-level or entity-level graph *nodes* — an entity is not itself a +queryable node, and there is no entity-to-entity or entity-to-chunk edge, +only this document-to-document rollup. Use `neighbors` for "what else is +like this document" (and to see which shared terms justify that link), not +for anything finer-grained than a document pair. + +## 5. Reserved / not yet available + +- **Vector channel**: `recall`'s `channels` parameter reserves a `vector` + option; it does not exist yet. `recall` today is lexical only. + +## 6. Tools and when to use each + +| Tool | Use when | Scope | Notes | +|---|---|---|---| +| `list_documents` | Starting cold: which documents exist, what are they about | namespace | Returns per-document keywords/summary/type mix/`parse_track`. | +| `outline` | The task only needs titles/summaries — overview, "what does chapter N cover," picking where to look before reading | one document, or a `section_path` prefix within it | Titles + summaries + `chunk_count`, no body text, no folding. Depth-limited by argument, not by a token budget. Use this to build your own map instead of relying on a pre-folded one. | +| `node_filter` | The task is a traversal/exclusion predicate — FOR ALL / EXISTS / ANY / NOT — over section titles or summaries ("which docs mention X in a heading," "sections NOT about Y") | one or more documents | Deterministic substring/regex match against `section_path` and `summary` only, not body text. Returns the full matching set and count, never a truncated top-K. If the predicate must run against body text, use `grep` instead. | +| `grep` | Exact string / regex / identifier / number lookup that must run against body text | scoped by document/section/chunk_type | Returns match count plus snippets, so ANY/ALL logic can also close over body text, not just titles. | +| `recall` | A fuzzy question where you don't know where the answer lives | namespace or scoped | Ranked candidates (path + content BM25 today; term and vector are separate/reserved — see §5) with path and snippet, not full content. | +| `read` | You already know which section(s)/chunk(s) to read | one or more sections/chunks | Returns full body content, resolves `SAME-AS` markers into the owner's text, expands `connect_to` assets, and converts `page_assets` into URLs. | +| `assets` | You need images/tables directly, or need to find which section(s) host a given asset | one or more documents | Forward (by type/query) and reverse (asset → hosting section) lookup — see §3. | +| `neighbors` | You need related documents in the same namespace | one document | Document-level `related` edges only — see §4. | + +General rule: prefer `outline` / `node_filter` to locate before `read`ing +body text; prefer `grep` over `recall` when you know the exact string you +are looking for; only fall back to `recall` for genuinely fuzzy questions. diff --git a/packages/shared-python/shared/services/retrieval/execution/routes.py b/packages/shared-python/shared/services/retrieval/execution/routes.py index 6ed992937..3f6a82703 100644 --- a/packages/shared-python/shared/services/retrieval/execution/routes.py +++ b/packages/shared-python/shared/services/retrieval/execution/routes.py @@ -193,7 +193,15 @@ async def _run_classic_topk_route( async def _run_mapnav_route( context: RetrievalRouteContext, ) -> RetrievalRouteOutcome: - """Default agentic path: PLANNER + HARVEST + CONTROL (checklist map-nav).""" + """Default agentic path: PLANNER + HARVEST + CONTROL (checklist map-nav). + + LEGACY, PENDING REPLACEMENT: ``agent_explore`` will become the default + agentic route once it passes its evaluation gate; this route then stays + only as the ``RETRIEVAL_AGENTIC_ROUTER=mapnav`` fallback until Phase 5 + cleanup. Do not add new capabilities here — new agentic-retrieval work + belongs in ``shared/services/retrieval/agent_tools/`` and + ``shared/services/retrieval/agent_explore/``. + """ process_started = resource.getrusage(resource.RUSAGE_SELF) from shared.services.retrieval import nav_llm_backend # noqa: F401 from shared.services.retrieval.nav import run_nav_episode diff --git a/packages/shared-python/shared/services/retrieval/nav/__init__.py b/packages/shared-python/shared/services/retrieval/nav/__init__.py index 897fd00a9..50aa1b62d 100644 --- a/packages/shared-python/shared/services/retrieval/nav/__init__.py +++ b/packages/shared-python/shared/services/retrieval/nav/__init__.py @@ -1,4 +1,18 @@ -"""Recursive-dispatch map navigation for RealData experiments.""" +"""Recursive-dispatch map navigation for RealData experiments. + +LEGACY, PENDING REPLACEMENT: this package implements the map-nav +PLANNER/HARVEST/CONTROL episode +(``run_nav_episode``), the current default agentic retrieval route via +``execution.routes._run_mapnav_route``. It will be superseded by +``agent_explore`` after its evaluation gate, then trimmed in +Phase 5 to whatever this package still owns and nothing else uses (the BM25 +scorer in ``knowhere_hybrid.py``, the ``NodeFilter`` predicate in +``nav_node_filter.py``, and snapshot loading are already planned to be +reused by the new ``agent_tools/``, not deleted). Do not add new PLANNER / +HARVEST / CONTROL capabilities here; new agentic-retrieval work belongs in +``shared/services/retrieval/agent_tools/`` and +``shared/services/retrieval/agent_explore/``. +""" from .nav_types import ( ActionKind, diff --git a/packages/shared-python/shared/services/retrieval/nav_config.py b/packages/shared-python/shared/services/retrieval/nav_config.py index edf03dcb5..48067194a 100644 --- a/packages/shared-python/shared/services/retrieval/nav_config.py +++ b/packages/shared-python/shared/services/retrieval/nav_config.py @@ -3,6 +3,12 @@ Values live on ``NavConfig`` (model / thinking / token_limit / evidence chars). Vendored nav binds them for the episode via ``nav_llm_runtime`` + ``nav_token_episode``; no process-wide env seeding and no Knowhere wallet/agentic knobs. + +LEGACY, PENDING REPLACEMENT: this config only feeds the map-nav +PLANNER/HARVEST/CONTROL episode +(``shared.services.retrieval.nav``). It is not read by ``agent_explore`` +and is scheduled for Phase 5 cleanup once map-nav is +no longer the default route. """ from __future__ import annotations