diff --git a/apps/api/app/api/v1/routes/retrieval.py b/apps/api/app/api/v1/routes/retrieval.py index 508f9bf8e..5629e733e 100644 --- a/apps/api/app/api/v1/routes/retrieval.py +++ b/apps/api/app/api/v1/routes/retrieval.py @@ -13,6 +13,7 @@ from shared.core.database import get_db from shared.models.schemas.retrieval_namespace import normalize_retrieval_namespace from shared.services.retrieval.app_service import run_retrieval_query +from shared.services.retrieval.settings import DEFAULT_TOP_K router = APIRouter(tags=["Retrieval"]) @@ -29,7 +30,7 @@ class RetrievalQueryRequest(BaseModel): description="Effective namespace; defaults to default", ) query: str - top_k: int = 10 + top_k: int = DEFAULT_TOP_K exclude_document_ids: list[str] = Field(default_factory=list) exclude_sections: list[ExcludeSection] = Field(default_factory=list) data_type: int = Field( diff --git a/apps/api/app/mcp/retrieval_server.py b/apps/api/app/mcp/retrieval_server.py index 2f5846fa0..4223ccca7 100644 --- a/apps/api/app/mcp/retrieval_server.py +++ b/apps/api/app/mcp/retrieval_server.py @@ -13,6 +13,7 @@ from shared.core.database import get_db_context from shared.models.schemas.retrieval_namespace import normalize_retrieval_namespace from shared.services.retrieval.app_service import run_retrieval_query +from shared.services.retrieval.settings import DEFAULT_TOP_K DbFactory = Callable[[], AsyncContextManager[AsyncSession]] KNOWHERE_NAMESPACE_HEADER = "x-knowhere-namespace" @@ -51,36 +52,20 @@ def resolve_mcp_namespace(*, ctx: Context | None) -> str: def to_mcp_query_response(response: dict[str, Any]) -> dict[str, Any]: - results: list[dict[str, Any]] = [] - for row in response.get("results", []): - if not isinstance(row, dict): - continue - - source_value = row.get("source") - source = source_value if isinstance(source_value, dict) else row - result: dict[str, Any] = { - "content": row.get("content"), - "source_file_name": source.get("source_file_name"), - "section_path": source.get("section_path"), - "chunk_type": row.get("chunk_type"), - } - if row.get("asset_url"): - result["asset_url"] = row["asset_url"] - results.append(result) - - mcp_response: dict[str, Any] = { + """Project the internal retrieval response to the MCP agent contract. + + MCP returns exactly 3 PRIMARY fields: + - evidence_text: hierarchical evidence tree for LLM consumption + - referenced_chunks: structured chunk references for citation / follow-up + - decision_trace: navigation decisions including terminal stop/failure + """ + return { "query": response.get("query"), - "results": results, "evidence_text": response.get("evidence_text") or "", + "referenced_chunks": response.get("referenced_chunks") or [], + "decision_trace": response.get("decision_trace") or [], } - if response.get("stop_reason") is not None: - mcp_response["stop_reason"] = response["stop_reason"] - if response.get("decision_trace") is not None: - mcp_response["decision_trace"] = response["decision_trace"] - - return mcp_response - async def resolve_mcp_user_id(*, ctx: Context | None, db: AsyncSession) -> str: request = get_mcp_request(ctx) @@ -101,8 +86,10 @@ def create_retrieval_mcp_server( "knowhere-retrieval", instructions=( "Use this server to search published documents. " - "It returns evidence_text and ranked snippets, but never final answers. " - "Downstream agents should synthesize from the returned evidence." + "It returns evidence_text (hierarchical evidence tree), " + "referenced_chunks (structured chunk citations), and " + "decision_trace (navigation decisions). " + "Downstream agents should synthesize answers from evidence_text." ), streamable_http_path=streamable_http_path, stateless_http=True, @@ -112,17 +99,53 @@ def create_retrieval_mcp_server( @server.tool( name="retrieval.query", description=( - "Search published documents and return relevant snippets plus unified " - "evidence_text for downstream answer synthesis." + "Search published documents. Returns evidence_text (hierarchical " + "evidence for LLM consumption), referenced_chunks (cited chunk " + "metadata for follow-up queries), and decision_trace (navigation " + "decisions including stop/failure reasons). " + "Include navigation intent directly in your query text — the " + "engine will automatically locate the right documents and sections." ), ) async def query_documents( - query: Annotated[str, Field(description="What you want to search for.")], + query: Annotated[ + str, + Field(description=( + "What you want to search for. You may include navigation " + "hints naturally, e.g. 'find tables in chapter 3 of the " + "safety report'. The engine understands document structure." + )), + ], top_k: Annotated[ - int, Field(description="Maximum number of results to return.") - ] = 5, + int, + Field(description=( + "Number of candidate chunks for initial discovery. " + "The final output is budget-controlled; this only affects " + "the discovery recall pool. Usually no need to adjust." + )), + ] = DEFAULT_TOP_K, + exclude_document_ids: Annotated[ + list[str], + Field(description=( + "Document IDs to exclude from this query. " + "Use document_id values from prior referenced_chunks." + )), + ] = [], + exclude_sections: Annotated[ + list[dict[str, str]], + Field(description=( + "Sections to exclude. Each item: " + '{"document_id": "...", "section_path": "..."}.' + )), + ] = [], ctx: Context | None = None, ) -> dict: + # TODO(intent-step): When the Intent Understanding step is + # implemented, it will parse `query` here to extract structured + # hints (document_hint, scope_hint, content_type_hint) and set + # data_type / signal_paths / filter_mode / exclude_document_ids + # automatically before calling run_retrieval_query. + # See: shared/services/retrieval/intent/ (to be created) namespace = resolve_mcp_namespace(ctx=ctx) async with db_factory() as db: user_id = await resolve_mcp_user_id(ctx=ctx, db=db) @@ -132,8 +155,9 @@ async def query_documents( namespace=namespace, query=query, top_k=top_k, - exclude_document_ids=[], - exclude_sections=[], + exclude_document_ids=exclude_document_ids, + exclude_sections=[item for item in exclude_sections], + use_agentic=True, ) return to_mcp_query_response(response) diff --git a/apps/api/tests/contract/test_agentic_discovery_selection_contract.py b/apps/api/tests/contract/test_agentic_discovery_selection_contract.py index 12cfdae97..7d9069183 100644 --- a/apps/api/tests/contract/test_agentic_discovery_selection_contract.py +++ b/apps/api/tests/contract/test_agentic_discovery_selection_contract.py @@ -6,7 +6,7 @@ def test_root_discovery_hint_is_projected_for_llm_selection() -> None: - hint_lines, hint_by_path = _project_discovery_hints( + hint_lines, hint_by_path, excluded_hints = _project_discovery_hints( [ { "section_path": "Root", diff --git a/packages/shared-python/shared/services/retrieval/agentic/core/trace.py b/packages/shared-python/shared/services/retrieval/agentic/core/trace.py index bc6ccaa02..7925bcb03 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/core/trace.py +++ b/packages/shared-python/shared/services/retrieval/agentic/core/trace.py @@ -16,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from shared.services.retrieval.agentic.core.types import AgentRunConfig, ToolResult +from shared.services.retrieval.settings import DEFAULT_TOP_K def _now_utc() -> datetime: @@ -47,7 +48,7 @@ def __init__( namespace: str, query: str, config: AgentRunConfig, - top_k: int = 10, + top_k: int = DEFAULT_TOP_K, data_type: int = 1, filters: dict[str, Any] | None = None, parent_run_id: str | None = None, diff --git a/packages/shared-python/shared/services/retrieval/agentic/discovery/phase.py b/packages/shared-python/shared/services/retrieval/agentic/discovery/phase.py index 49f932c9c..e5110a708 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/discovery/phase.py +++ b/packages/shared-python/shared/services/retrieval/agentic/discovery/phase.py @@ -13,6 +13,7 @@ from shared.services.retrieval.agentic.core.trace import TraceRecorder from shared.services.retrieval.agentic.core.types import AgentState, CandidateDoc, ToolResult from shared.services.retrieval.llm_adapter import LLMFn +from shared.services.retrieval.search.lexical_text import normalize_section_path async def run_initial_discovery( @@ -76,6 +77,9 @@ async def run_initial_discovery( f"status={discovery_result.status} latency={discovery_result.latency_ms}ms" ) + # Build per-document discovery signals for KG soft-prompting + discovery_signals = build_discovery_signals(discovery_rows) + if bootstrap_llm_fn is not None: await _select_documents( db, @@ -87,42 +91,39 @@ async def run_initial_discovery( query=query, exclude_document_ids=exclude_document_ids, bootstrap_llm_fn=bootstrap_llm_fn, + discovery_signals=discovery_signals, ) return discovery_rows -async def register_discovery_documents( - db: AsyncSession, - *, - state: AgentState, - discovery_by_doc: dict[str, list[dict[str, Any]]], -) -> None: - selected_doc_ids = {doc.document_id for doc in state.selected_docs} - for doc_id in discovery_by_doc: - if doc_id in selected_doc_ids or doc_id in state.ever_explored_doc_ids: - continue - doc_stmt = ( - select(Document.document_id, Document.source_file_name, Document.current_job_result_id) - .where(Document.document_id == doc_id) +def build_discovery_signals( + discovery_rows: list[dict[str, Any]], +) -> dict[str, list[str]]: + """Build per-document discovery signals from bottom discovery results. + + Returns a mapping of ``{doc_id: [path1, path2, ...]}`` for documents + where keyword/semantic search found potentially relevant section paths. + These signals are injected as soft hints into the KG document selection + prompt, allowing the LLM to make an informed decision rather than + force-injecting documents. + """ + signals: dict[str, list[str]] = {} + seen: dict[str, set[str]] = {} + for row in discovery_rows: + doc_id = row.get("document_id", "") + section_path = normalize_section_path( + str(row.get("section_path", "") or "").strip() ) - doc_result = await db.execute(doc_stmt) - row_data = doc_result.first() - if row_data is None: + if not doc_id or not section_path or section_path == "Root": continue - did, fname, job_result_id = row_data - state.selected_docs.append( - CandidateDoc( - document_id=did, - source_file_name=fname or did, - confidence=0.4, - reason="discovery_auto (not in KG selection)", - source="discovery_auto", - ) - ) - state.doc_id_to_name[did] = fname or did - if job_result_id: - state.doc_job_map[did] = job_result_id + if doc_id not in seen: + seen[doc_id] = set() + signals[doc_id] = [] + if section_path not in seen[doc_id]: + seen[doc_id].add(section_path) + signals[doc_id].append(section_path) + return signals async def _select_documents( @@ -136,6 +137,7 @@ async def _select_documents( query: str, exclude_document_ids: list[str], bootstrap_llm_fn: LLMFn, + discovery_signals: dict[str, list[str]] | None = None, ) -> None: try: kg_result = await tools.kg_document_select( @@ -146,6 +148,7 @@ async def _select_documents( llm_fn=bootstrap_llm_fn, exclude_document_ids=list(state.ever_explored_doc_ids | set(exclude_document_ids)), budget_snapshot=state.ledger.snapshot() if state.ledger else None, + discovery_signals=discovery_signals, ) except BudgetExceeded: logger.info(" agentic: bootstrap budget exhausted during document selection") diff --git a/packages/shared-python/shared/services/retrieval/agentic/discovery/selection.py b/packages/shared-python/shared/services/retrieval/agentic/discovery/selection.py index fbb4c1d90..45ae3010a 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/discovery/selection.py +++ b/packages/shared-python/shared/services/retrieval/agentic/discovery/selection.py @@ -2,6 +2,7 @@ from __future__ import annotations import time +from dataclasses import dataclass, field from typing import Any from loguru import logger @@ -22,7 +23,16 @@ from shared.services.retrieval.llm_adapter import LLMFn -_MAX_DISCOVERY_PER_DOC = 3 +_MAX_DISCOVERY_PER_DOC = 10 + + +@dataclass +class DiscoverySelectResult: + """Result of discovery_select_step: node + dedup metadata.""" + + node: DocTreeNode + excluded_hints: list[dict[str, str]] = field(default_factory=list) + candidate_count: int = 0 async def discovery_select_step( @@ -37,22 +47,36 @@ async def discovery_select_step( discovery_hints: list[dict[str, Any]], exclude_paths: set[str] | None = None, budget_snapshot: dict | None = None, -) -> DocTreeNode: +) -> DiscoverySelectResult: """Select and hydrate discovery-found sections after BFS navigation.""" node = DocTreeNode(scope_path=None) if not discovery_hints: - return node + return DiscoverySelectResult(node=node) hints = discovery_hints[:_MAX_DISCOVERY_PER_DOC] t0 = time.monotonic() try: - hint_lines, hint_by_path = _project_discovery_hints( + hint_lines, hint_by_path, excluded_hints = _project_discovery_hints( hints, exclude_paths=exclude_paths, ) + if excluded_hints: + logger.info( + f' discovery_select_step doc="{doc_name}": ' + f"{len(excluded_hints)} hints excluded by navigation COLLECT: " + + ", ".join( + f'"{h["path"]}" (covered by "{h["covered_by"]}")' + for h in excluded_hints[:3] + ) + + (f" (+{len(excluded_hints) - 3} more)" if len(excluded_hints) > 3 else "") + ) if not hint_lines: - return node + return DiscoverySelectResult( + node=node, + excluded_hints=excluded_hints, + candidate_count=len(hints), + ) selections: list[dict[str, Any]] = [] if hint_lines: @@ -100,20 +124,30 @@ async def discovery_select_step( f" discovery_select_step done: hydrated={len(node.leaf_content)} " f"latency={latency}ms" ) - return node + return DiscoverySelectResult( + node=node, + excluded_hints=excluded_hints, + candidate_count=len(hints), + ) except BudgetExceeded: raise except Exception as exc: logger.error(f" discovery_select_step failed for doc={document_id}: {exc}") - return node + return DiscoverySelectResult(node=node) def _project_discovery_hints( hints: list[dict[str, Any]], *, exclude_paths: set[str] | None, -) -> tuple[list[str], dict[str, dict]]: +) -> tuple[list[str], dict[str, dict], list[dict[str, str]]]: + """Project discovery hints into prompt lines, filtering excluded paths. + + Returns ``(hint_lines, hint_by_path, excluded_hints)`` where + *excluded_hints* records each path that was dropped and which + navigation-collected path covered it. + """ exclude_set = { normalize_section_path(path) for path in (exclude_paths or set()) @@ -121,11 +155,14 @@ def _project_discovery_hints( } hint_lines: list[str] = [] hint_by_path: dict[str, dict] = {} + excluded_hints: list[dict[str, str]] = [] for hint in hints: section_path = normalize_section_path(hint.get("section_path", "")) if not section_path: continue - if _is_covered_by_exclude(section_path, exclude_set): + covered_by = _find_covering_path(section_path, exclude_set) + if covered_by is not None: + excluded_hints.append({"path": section_path, "covered_by": covered_by}) continue if section_path in hint_by_path: continue @@ -136,22 +173,22 @@ def _project_discovery_hints( if summary: hint_lines.append(f" {summary[:300]}") - return hint_lines, hint_by_path + return hint_lines, hint_by_path, excluded_hints -def _is_covered_by_exclude(path: str, exclude_set: set[str]) -> bool: - """Check if *path* is covered by any entry in *exclude_set*. +def _find_covering_path(path: str, exclude_set: set[str]) -> str | None: + """Return the exclude-set entry that covers *path*, or ``None``. A path is covered if it exactly matches an exclude entry, OR if any exclude entry is a prefix of this path (i.e. the parent path was already collected by navigation). """ if path in exclude_set: - return True + return path for excluded in exclude_set: if path.startswith(excluded + " / "): - return True - return False + return excluded + return None def _build_discovery_selection_prompt( diff --git a/packages/shared-python/shared/services/retrieval/agentic/discovery/tools.py b/packages/shared-python/shared/services/retrieval/agentic/discovery/tools.py index 29cae7783..1a4a2447e 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/discovery/tools.py +++ b/packages/shared-python/shared/services/retrieval/agentic/discovery/tools.py @@ -189,6 +189,7 @@ async def kg_document_select( query: str, llm_fn: LLMFn | None, exclude_document_ids: list[str], + discovery_signals: dict[str, list[str]] | None = None, **_kwargs: Any, ) -> ToolResult: """Select candidate documents from document-level KG.""" @@ -215,6 +216,12 @@ async def kg_document_select( latency_ms=latency, ) + # Inject discovery signals as soft hints into the overview + if discovery_signals: + overview_text = _inject_discovery_signals( + overview_text, discovery_signals, + ) + file_prompt = FILE_SELECT_PROMPT.format( overview=overview_text, query=query, @@ -284,3 +291,40 @@ async def kg_document_select( latency = int((time.monotonic() - t0) * 1000) logger.error(f" agentic.kg_document_select failed: {exc}") return ToolResult(status="error", error=str(exc), latency_ms=latency) + + +def _inject_discovery_signals( + overview_text: str, + signals: dict[str, list[str]], + *, + max_paths_per_doc: int = 5, +) -> str: + """Inject discovery hint lines into the document overview text. + + For each document that has discovery signals, append hint lines + directly after the document's overview entry. The LLM sees these + as advisory information — it is free to select or ignore the document. + """ + if not signals: + return overview_text + + lines = overview_text.split("\n") + result: list[str] = [] + for line in lines: + result.append(line) + # Match overview lines: "- [doc_xxx] filename chunks=..." + if not line.startswith("- ["): + continue + bracket_end = line.find("]") + if bracket_end < 0: + continue + doc_id = line[3:bracket_end] + paths = signals.get(doc_id) + if not paths: + continue + display_paths = paths[:max_paths_per_doc] + hints_line = ", ".join(f'"{p}"' for p in display_paths) + if len(paths) > max_paths_per_doc: + hints_line += f" (+{len(paths) - max_paths_per_doc} more)" + result.append(f" 🔍 Discovery hints: {hints_line}") + return "\n".join(result) diff --git a/packages/shared-python/shared/services/retrieval/agentic/evidence/builder.py b/packages/shared-python/shared/services/retrieval/agentic/evidence/builder.py index ee0e68e7f..c81874017 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/evidence/builder.py +++ b/packages/shared-python/shared/services/retrieval/agentic/evidence/builder.py @@ -140,16 +140,19 @@ async def render_evidence( evidence_parts: list[str] = [] for doc_id, doc_tree in doc_trees.items(): - # Only render if there is actual evidence (hydrated chunks or outline content). - if doc_tree.has_leaf_content() or doc_tree.has_content(): - doc_name = doc_id_to_name.get(doc_id, doc_id) - rendered = render_unified_doc_tree( - doc_tree, - doc_name, - asset_lookup=asset_url_map, - ) - if rendered.strip(): - evidence_parts.append(rendered) + # Only render if there is actual hydrated evidence (chunks collected + # via COLLECT or discovery). Outline-only trees (e.g. navigation + # STOP with empty collect) must not leak into evidence_text. + if not doc_tree.has_leaf_content(): + continue + doc_name = doc_id_to_name.get(doc_id, doc_id) + rendered = render_unified_doc_tree( + doc_tree, + doc_name, + asset_lookup=asset_url_map, + ) + if rendered.strip(): + evidence_parts.append(rendered) return "\n\n".join(evidence_parts) if evidence_parts else "" diff --git a/packages/shared-python/shared/services/retrieval/agentic/evidence/renderer.py b/packages/shared-python/shared/services/retrieval/agentic/evidence/renderer.py index c600218dd..42c543f72 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/evidence/renderer.py +++ b/packages/shared-python/shared/services/retrieval/agentic/evidence/renderer.py @@ -87,13 +87,13 @@ def min_sort(path: str) -> float: if path in node.children: # Non-leaf node with own content: render heading, then # self chunks, then child subtree (merged rendering). - parts.append(f"{indent}▸ {title}") + parts.append(f"{indent}▸ [L{depth + 1}] {title}") render_leaf_chunks(parts, node.leaf_content[path], sub_indent, asset_lookup=asset_lookup) child_text = render_unified_doc_tree(node.children[path], doc_name, depth + 1, asset_lookup=asset_lookup) if child_text.strip(): parts.append(child_text) else: - parts.append(f"{indent}▸ [Leaf] {title}") + parts.append(f"{indent}▸ [L{depth + 1}] {title} [Leaf]") render_leaf_chunks(parts, node.leaf_content[path], sub_indent, asset_lookup=asset_lookup) elif render_type == "orphan_child": @@ -103,7 +103,7 @@ def min_sort(path: str) -> float: # Only render the orphan heading if the child has content. # Prevents empty orphan nodes from polluting evidence_text. if child_text.strip(): - parts.append(f"{indent}▸ {title}") + parts.append(f"{indent}▸ [L{depth + 1}] {title}") parts.append(child_text) return "\n".join(parts) diff --git a/packages/shared-python/shared/services/retrieval/agentic/navigation/document.py b/packages/shared-python/shared/services/retrieval/agentic/navigation/document.py index 167615c7c..230cffc25 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/navigation/document.py +++ b/packages/shared-python/shared/services/retrieval/agentic/navigation/document.py @@ -38,6 +38,9 @@ from shared.services.retrieval.agentic.navigation.selection_hydration import ( hydrate_path_selections_into_node, ) +from shared.services.retrieval.agentic.discovery.selection import ( + DiscoverySelectResult, +) from shared.services.retrieval.llm_adapter import LLMFn @@ -95,19 +98,16 @@ async def _navigate_document( return doc_name = doc.source_file_name or self._state.doc_id_to_name.get(doc.document_id, "") - is_discovery_only_doc = doc.source == "discovery_auto" root = DocTreeNode(scope_path=None) doc_pending_assets: list[dict[str, Any]] = [] # Phase 2A: Collector Agent navigation (summary-only, no content hydration) - collected_paths: list[dict[str, Any]] = [] - if not is_discovery_only_doc: - doc_pending_assets, collected_paths = await self._navigate_collector( - doc=doc, - root=root, - doc_name=doc_name, - job_result_id=job_result_id, - ) + doc_pending_assets, collected_paths = await self._navigate_collector( + doc=doc, + root=root, + doc_name=doc_name, + job_result_id=job_result_id, + ) # Phase 2B: Discovery hints (independent hydration path) await self._hydrate_discovery_hints( @@ -118,7 +118,7 @@ async def _navigate_document( ) # Phase 2C: Batch hydrate all collected paths - if not is_discovery_only_doc and collected_paths: + if collected_paths: await self._hydrate_collected( doc=doc, root=root, @@ -127,7 +127,7 @@ async def _navigate_document( ) # Phase 2D: Reconcile assets into hydrated tree - if not is_discovery_only_doc and doc_pending_assets: + if doc_pending_assets: self._reconcile_pending_assets( doc=doc, root=root, @@ -322,11 +322,14 @@ async def _hydrate_collected( limit_depth=False, ) if section_items: - # Filter out the scope node itself to avoid duplicate - # title rendering (parent outline already shows it). + # Filter out the scope node itself AND ancestor/sibling + # items. load_child_sections returns ancestor context + # for navigation prompts, but for evidence rendering the + # child node only needs its own descendants. child_node.outline_items = [ si for si in section_items if si.get("path") != path + and si.get("path", "").startswith(path + " / ") ] # Build sub-tree from outline hierarchy and re-reparent # so chunks are correctly nested (e.g. L3 under L2). @@ -417,13 +420,14 @@ async def _hydrate_discovery_hints( discovery_exclude_paths = _build_discovery_exclude_set( root, collected_paths or [] ) + doc_discovery_llm_fn = self._llm_budget.for_discovery( cast(LLMFn, self._llm_fn), doc_id=doc.document_id, low_priority=root.has_content(), ) try: - discovery_node = await tools.discovery_select_step( + result = await tools.discovery_select_step( self._db, document_id=doc.document_id, query=self._query, @@ -439,9 +443,12 @@ async def _hydrate_discovery_hints( logger.info(" agentic: planning budget exhausted during discovery selection") if self._trace_enabled: self._trace.record_budget_stop("planning_exhausted") - discovery_node = DocTreeNode(scope_path=None) + result = DiscoverySelectResult(node=DocTreeNode(scope_path=None)) self._state.step_count += 1 + discovery_node = result.node + excluded_hints = result.excluded_hints + if self._trace_enabled: self._trace.record_step( "discovery_select_step", @@ -451,6 +458,7 @@ async def _hydrate_discovery_hints( "document_id": doc.document_id, "hints_count": len(doc_hints), "hydrated_count": len(discovery_node.leaf_content), + "excluded_count": len(excluded_hints), }, ), decision_reason=f"discovery_{doc.source_file_name}", @@ -461,9 +469,11 @@ async def _hydrate_discovery_hints( "document_id": doc.document_id, "action": "select" if discovery_node.has_content() else "skip", "reason": "", - "candidate_count": len(doc_hints), + "candidate_count": result.candidate_count, "hydrated_count": len(discovery_node.leaf_content), "selected_paths": list(discovery_node.leaf_content.keys()), + "excluded_hints": excluded_hints, + "exclude_set": sorted(discovery_exclude_paths), }) root.merge(discovery_node) if self._state.ledger is not None: @@ -623,6 +633,7 @@ def _build_discovery_exclude_set( return exclude + def _ensure_child_node(root: DocTreeNode, path: str) -> None: """Create an intermediate child node for *path* if it doesn't exist. @@ -696,6 +707,22 @@ def _build_outline_subtree(node: DocTreeNode) -> None: if path not in node.children: node.children[path] = DocTreeNode(scope_path=path) + # Reparent existing children that are descendants of newly-created + # parents. E.g. if node.children already has "A / B" and we just + # created "A", move "A / B" under "A". This prevents "A / B" from + # appearing as an orphan_child at the wrong tree depth. + for parent_path in parent_paths: + if parent_path in pre_existing: + continue # Don't reparent into pre-existing nodes + parent_node = node.children[parent_path] + to_move = [ + cp for cp in list(node.children.keys()) + if cp != parent_path + and cp.startswith(parent_path + " / ") + ] + for cp in to_move: + parent_node.children[cp] = node.children.pop(cp) + # Split outline_items: keep items at this level, move descendants # into newly-created children only (skip pre-existing ones). kept: list[dict] = [] diff --git a/packages/shared-python/shared/services/retrieval/agentic/navigation/section_prompt_projection.py b/packages/shared-python/shared/services/retrieval/agentic/navigation/section_prompt_projection.py index 2b4c04fd3..d7130fa31 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/navigation/section_prompt_projection.py +++ b/packages/shared-python/shared/services/retrieval/agentic/navigation/section_prompt_projection.py @@ -73,12 +73,30 @@ def _render_item(item: dict, include_summary: bool, collected: set[str]) -> str: if include_summary and show_summary and summary: sub_indent = " " * level - clipped = truncate_content_preview(summary, head=80, tail=0) + display_summary = _enrich_section_covers_summary(summary) + clipped = truncate_content_preview(display_summary, head=80, tail=0) lines.append(f"{sub_indent}{clipped}") return "\n".join(lines) +def _enrich_section_covers_summary(summary: str) -> str: + """Inject sub-section count into 'This section covers:' summaries. + + Transforms: + 'This section covers: A, B, C' + into: + 'This section covers 3 sub-sections: A, B, C' + """ + prefix = "This section covers: " + if not summary.startswith(prefix): + return summary + body = summary[len(prefix):] + sub_sections = [s.strip() for s in body.split(", ") if s.strip()] + count = len(sub_sections) + return f"This section covers {count} sub-sections: {body}" + + def _is_path_collected(path: str, collected: set[str]) -> bool: """Check if path itself or any ancestor is in the collected set.""" if path in collected: diff --git a/packages/shared-python/shared/services/retrieval/agentic/navigation/tools.py b/packages/shared-python/shared/services/retrieval/agentic/navigation/tools.py index 65ab2fae1..530777768 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/navigation/tools.py +++ b/packages/shared-python/shared/services/retrieval/agentic/navigation/tools.py @@ -26,7 +26,6 @@ from shared.services.retrieval.agentic.core.budget import BudgetExceeded from shared.services.retrieval.agentic.prompts import ( COLLECTOR_PROMPT, - DISCOVERY_SELECT_PROMPT, format_budget_block, parse_collector_response, ) @@ -192,62 +191,3 @@ async def navigate_step( logger.error(f" navigate_step failed for doc={document_id}: {exc}") return NavigateStepResult.stop(scope_paths[0] if scope_paths else None) - -async def discovery_select_step( - db: AsyncSession, - *, - document_id: str, - query: str, - llm_fn: LLMFn, - user_id: str, - namespace: str, - doc_name: str = "", - discovery_hints: list[dict[str, Any]], - exclude_paths: set[str] | None = None, - budget_snapshot: dict | None = None, -) -> DocTreeNode: - """Select discovery hint paths via LLM and hydrate them.""" - from shared.services.retrieval.agentic.prompts import parse_action_response - from shared.services.retrieval.agentic.navigation.selection_hydration import ( - hydrate_path_selections_into_node, - ) - - excluded = exclude_paths or set() - filtered_hints = [ - hint for hint in discovery_hints - if hint.get("path", "") not in excluded - ] - if not filtered_hints: - return DocTreeNode(scope_path=None) - - items_text = "\n".join( - f'- path="{hint.get("path", "")}" score={hint.get("score", 0):.2f}' - for hint in filtered_hints - ) - prompt = DISCOVERY_SELECT_PROMPT.format( - doc_name=doc_name or document_id, - budget_block=format_budget_block(budget_snapshot), - items=items_text, - query=query, - ) - - response = await llm_fn(prompt) - parsed = parse_action_response(response) - selections = parsed.get("selections", []) - - node = DocTreeNode(scope_path=None) - if selections: - path_selections = [ - {"path": sel["path"], "confidence": sel.get("confidence", 0.7), "hydrate_mode": "chunks"} - for sel in selections - ] - await hydrate_path_selections_into_node( - db, - node=node, - path_selections=path_selections, - user_id=user_id, - namespace=namespace, - document_id=document_id, - ) - - return node diff --git a/packages/shared-python/shared/services/retrieval/agentic/orchestrator.py b/packages/shared-python/shared/services/retrieval/agentic/orchestrator.py index 80123b00b..590304606 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/orchestrator.py +++ b/packages/shared-python/shared/services/retrieval/agentic/orchestrator.py @@ -23,7 +23,6 @@ from shared.services.retrieval.agentic.core.budget import BudgetLedger from shared.services.retrieval.agentic.discovery.phase import ( - register_discovery_documents, run_initial_discovery, ) from shared.services.retrieval.agentic.navigation.document import DocumentNavigationRunner @@ -42,6 +41,7 @@ AgenticResult, ) from shared.services.retrieval.llm_adapter import LLMFn +from shared.services.retrieval.settings import DEFAULT_TOP_K class RetrievalAgent: @@ -68,7 +68,7 @@ async def run( user_id: str, namespace: str, query: str, - top_k: int = 10, + top_k: int = DEFAULT_TOP_K, llm_fn: LLMFn | None = None, exclude_document_ids: list[str] | None = None, exclude_sections: list[dict[str, str]] | None = None, @@ -203,11 +203,6 @@ async def run( continue discovery_by_doc.setdefault(doc_id, []).append(row) - await register_discovery_documents( - db, - state=state, - discovery_by_doc=discovery_by_doc, - ) if state.ledger is not None: await state.ledger.allocate_doc_caps({ diff --git a/packages/shared-python/shared/services/retrieval/agentic/prompts.py b/packages/shared-python/shared/services/retrieval/agentic/prompts.py index 62c52a288..1d4cbf60f 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/prompts.py +++ b/packages/shared-python/shared/services/retrieval/agentic/prompts.py @@ -12,6 +12,9 @@ {budget_block} Below is a document corpus overview showing all available documents, their navigation summaries, chunk counts, and media counts. +Some documents may show "🔍 Discovery hints" — these are preliminary keyword +matches from bottom-up search. Consider them as additional signals but make +your own judgment on document relevance. === Document Corpus Overview === {overview} @@ -99,6 +102,10 @@ Set "outline": true on a collect entry to collect only the section structure (titles and summaries) without full chunk content. Use for overview/structure queries. Do not include any explanation outside the JSON. + +IMPORTANT: +1. All agent-generated text (e.g., "reason" and other free-text fields) MUST be written in English. +2. Document content and section paths MUST remain in their original language. """ diff --git a/packages/shared-python/shared/services/retrieval/agentic/tools.py b/packages/shared-python/shared/services/retrieval/agentic/tools.py index 97e76c5e1..78ffc555c 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/tools.py +++ b/packages/shared-python/shared/services/retrieval/agentic/tools.py @@ -9,8 +9,9 @@ from sqlalchemy.ext.asyncio import AsyncSession -from shared.services.retrieval.agentic.core.types import DocTreeNode, ToolResult +from shared.services.retrieval.agentic.core.types import ToolResult from shared.services.retrieval.agentic.discovery import selection as discovery_selection +from shared.services.retrieval.agentic.discovery.selection import DiscoverySelectResult from shared.services.retrieval.agentic.discovery import tools as discovery_tools from shared.services.retrieval.agentic.navigation import assets as asset_tools from shared.services.retrieval.agentic.navigation import tools as navigation_tools @@ -60,6 +61,7 @@ async def kg_document_select( query: str, llm_fn: LLMFn | None, exclude_document_ids: list[str], + discovery_signals: dict[str, list[str]] | None = None, **kwargs: Any, ) -> ToolResult: return await discovery_tools.kg_document_select( @@ -69,6 +71,7 @@ async def kg_document_select( query=query, llm_fn=llm_fn, exclude_document_ids=exclude_document_ids, + discovery_signals=discovery_signals, **kwargs, ) @@ -135,7 +138,7 @@ async def discovery_select_step( discovery_hints: list[dict[str, Any]], exclude_paths: set[str] | None = None, budget_snapshot: dict | None = None, -) -> DocTreeNode: +) -> DiscoverySelectResult: return await discovery_selection.discovery_select_step( db, document_id=document_id, diff --git a/packages/shared-python/shared/services/retrieval/execution/plan.py b/packages/shared-python/shared/services/retrieval/execution/plan.py index 2dda78192..2916c4166 100644 --- a/packages/shared-python/shared/services/retrieval/execution/plan.py +++ b/packages/shared-python/shared/services/retrieval/execution/plan.py @@ -69,6 +69,18 @@ def __init__(self, request: RetrievalQuery) -> None: async def execute(self) -> dict[str, Any]: request = self.request + # TODO(intent-step): Insert Intent Understanding step here. + # Before any retrieval runs, parse `request.query` with LLM + + # KG overview + section tree to extract structured navigation + # hints (document_hint, scope_hint, content_type_hint). + # Use extracted hints to override request.data_type, + # request.signal_paths, request.filter_mode, and narrow + # request.exclude_document_ids. This pre-trims the search + # space so Discovery/DocSelect/Navigation operate on a + # focused subgraph. Only activate when hints are detected; + # pure semantic queries skip this step. + # See: shared/services/retrieval/intent/ (to be created) + start_time = time.monotonic() _log_retrieval_start( query=request.query, diff --git a/packages/shared-python/shared/services/retrieval/execution/routes.py b/packages/shared-python/shared/services/retrieval/execution/routes.py index 2bb24958e..c46bab9bf 100644 --- a/packages/shared-python/shared/services/retrieval/execution/routes.py +++ b/packages/shared-python/shared/services/retrieval/execution/routes.py @@ -156,6 +156,18 @@ async def _run_agentic_route( for step in workflow_result.steps: if step.decision_trace: all_decision_trace.extend(step.decision_trace) + + # Embed stop/failure into decision_trace as terminal entry + stop_reason = response.get("stop_reason") or "" + failure_reason = response.get("failure_reason") or "" + if stop_reason or failure_reason: + all_decision_trace.append({ + "phase": "terminal", + "action": "complete", + "stop_reason": stop_reason, + "failure_reason": failure_reason, + }) + if all_decision_trace: response["decision_trace"] = all_decision_trace diff --git a/packages/shared-python/shared/services/retrieval/search/channels.py b/packages/shared-python/shared/services/retrieval/search/channels.py index bd881769e..13b392aba 100644 --- a/packages/shared-python/shared/services/retrieval/search/channels.py +++ b/packages/shared-python/shared/services/retrieval/search/channels.py @@ -99,6 +99,12 @@ def _build_extra_filters( params[f'_act_{i}'] = ct if signal_paths: + # TODO(intent-step): Current implementation uses OR across + # signal_paths keywords. The Intent Step will need hierarchical + # AND (prefix) matching, e.g. signal_paths=["第一章/1.1/(2)"] + # should match only paths containing ALL segments in order. + # Consider adding a `filter_strategy` param: "keyword_or" (current) + # vs "path_prefix" (for Intent Step resolved paths). ilike_parts = [] for i, kw in enumerate(signal_paths): key = f'_sig_{i}' diff --git a/packages/shared-python/shared/services/retrieval/settings.py b/packages/shared-python/shared/services/retrieval/settings.py index 49f2461b5..dfc55bf00 100644 --- a/packages/shared-python/shared/services/retrieval/settings.py +++ b/packages/shared-python/shared/services/retrieval/settings.py @@ -5,6 +5,7 @@ CHANNEL_WEIGHT_TERM = 1.5 INTERNAL_RECALL_K_MULTIPLIER = 2 RRF_K = 60 +DEFAULT_TOP_K = 10 DATA_TYPE_ALLOWED_CHUNK_TYPES: dict[int, set[str] | None] = { 1: None, diff --git a/packages/shared-python/shared/services/retrieval/workflow/planner.py b/packages/shared-python/shared/services/retrieval/workflow/planner.py index ae413f658..f2581298e 100644 --- a/packages/shared-python/shared/services/retrieval/workflow/planner.py +++ b/packages/shared-python/shared/services/retrieval/workflow/planner.py @@ -54,6 +54,10 @@ Return ONLY a JSON object matching this schema (think first, then answer): {schema} + +IMPORTANT: +1. "reasoning_summary" MUST be written in English; +2. user query and sub-queries (if any) MUST remain in the user's original language. """