Resolve Issue #113: Agentic decision trace + Root node channel fix

AGENTS.md

@@ -537,7 +537,7 @@ Core retrieval internals are grouped by ownership:
 - `hydration/`: row/path/reference hydration, inline assets, and result assembly.
 - `graph/`: document graph publication/query support.
 - `stats/`: retrieval hit recording.
-- `workflow/`: query planning, step execution, synthesis, and wallet state.
+- `workflow/`: query planning, retrieve step execution, and wallet state.
 - `agentic/core/`: agentic run types, token budgets, runtime config, and traces.
 - `agentic/discovery/`: bottom discovery and document selection.
 - `agentic/navigation/`: section-tree navigation, selection hydration, and asset tools.
@@ -569,15 +569,15 @@ flowchart LR
 
 The agentic pipeline uses `WorkflowOrchestrator` to handle complex queries via a DAG-based planning and budget-constrained execution engine:
 
-1. **Planning (`PlannerAgent`)**: The query is analyzed and decomposed into a DAG of steps.
+1. **Planning (`PlannerAgent`)**: The query is analyzed and decomposed into a DAG of retrieval steps.
    - Simple queries generate a single `retrieve` step.
-   - Complex queries are broken into multiple `retrieve` steps followed by a final `synthesize` step.
-2. **Budget Ledger (`BudgetLedger`)**: A strict token budget mechanism is enforced across the entire DAG execution (e.g., `AGENTIC_MAX_BUDGET=30000`). If the budget is exhausted, the pipeline halts safely and returns the best-effort evidence collected so far.
+   - Complex queries are broken into multiple `retrieve` steps. KNOWHERE does not plan answer synthesis steps.
+2. **Budget Ledger (`BudgetLedger`)**: A strict token budget mechanism is enforced across the entire DAG execution. If the budget is exhausted, the pipeline halts safely and returns the best-effort evidence collected so far.
 3. **Execution (`RetrievalAgent`)**: For each `retrieve` step, a multi-phase navigation engine runs:
    - **Phase 1 (Discovery)**: 3-channel RRF keyword search and KG document selection.
    - **Phase 2 (Navigation)**: Constrained Breadth-First Search (BFS) over the document's section tree. Discovered orphan leaves are merged into the tree to prevent data loss.
-   - **Phase 3 (Verdict)**: The LLM evaluates the collected structural outlines + hydrated chunks. Triggers a revision round (max 2) if `NOT_FOUND`.
-4. **Synthesis**: The LLM synthesizes a final `answer_text` and precise citations (`referenced_chunks`) using the unified evidence tree.
+   - **Phase 3 (Evidence Rendering)**: The hydrated document tree is rendered as `evidence_text`.
+4. **Evidence-Only Contract**: Retrieval responses always expose `evidence_text` as the primary output. `answer_text` is retained only as a deprecated empty string. Downstream agents decide whether the evidence is sufficient and synthesize answers outside KNOWHERE.
 
 ### Tree Rendering & Hydration
 
@@ -682,8 +682,7 @@ cd apps/worker && uv run worker.py     # Celery worker
 
 | Script | Purpose |
 |:---|:---|
-| `debug_parse.py` | End-to-end parsing with `MockRedis`, `LOCAL_DEBUG=1` |
-| `debug_hierarchy_llm.py` | Test heading recognition LLM calls |
+| `debug_parse.py` | Unified parsing debug: all formats, `--stop-at profile/hierarchy/full`, `--run-db` |
 | `debug_agentic_e2e.py` | End-to-end agentic retrieval test |
 | `debug_profiler.py` | Document profiler testing |
 | `debug_toc_detection.py` | TOC detection and hierarchy building |

apps/api/.env.example

@@ -84,8 +84,10 @@ ARK_API_KEY=
 # IMAGE_MODEL=qwen3.5-flash
 # IMAGE_MODEL_MAX=qwen3.5-flash
 
-# Optional retrieval overrides have code defaults. Set RETRIEVAL_AGENTIC_ENABLED=false
-# only when you need to fall back to legacy 3-channel RRF mode.
+# Optional retrieval overrides have code defaults. Retrieval is evidence-only:
+# evidence_text is the primary output and answer_text is always empty. Set
+# RETRIEVAL_AGENTIC_ENABLED=false only when you need to fall back to legacy
+# 3-channel RRF mode.
 
 # File handling defaults
 SUPPORTED_EXTENSIONS=.doc,.docx,.pdf,.txt,.xls,.xlsx,.csv,.pptx,.jpg,.jpeg,.png,.md

apps/api/app/api/v1/routes/retrieval.py

@@ -17,10 +17,6 @@
 router = APIRouter(tags=["Retrieval"])
 
 
-def _is_none(value: object) -> bool:
-    return value is None
-
-
 class ExcludeSection(BaseModel):
     document_id: str
     section_path: str
@@ -84,12 +80,30 @@ class RetrievalQueryResponse(BaseModel):
     namespace: str
     query: str
     router_used: str
-    answer_text: str | None = None
+    evidence_text: str = Field(
+        default="",
+        description="Hierarchical evidence text. Primary output for downstream agents.",
+    )
+    answer_text: str = Field(
+        default="",
+        description=(
+            "DEPRECATED. Always empty; KNOWHERE no longer generates answers. "
+            "Use evidence_text and synthesize answers downstream."
+        ),
+    )
     referenced_chunks: list[dict] = Field(default_factory=list)
     results: list[dict] = Field(default_factory=list)
-    evidence_text: str | None = Field(default=None, exclude_if=_is_none)
-    stop_reason: str | None = Field(default=None, exclude_if=_is_none)
-    failure_reason: str | None = Field(default=None, exclude_if=_is_none)
+    stop_reason: str | None = None
+    failure_reason: str | None = None
+    decision_trace: list[dict] | None = Field(
+        default=None,
+        description=(
+            "Per-step navigation decisions from agentic retrieval. "
+            "Each entry has phase, document, action, reason, stop_type, "
+            "selected_paths, and hydrated_count. Use this to understand "
+            "why KNOWHERE stopped or made specific navigation choices."
+        ),
+    )
 
 
 @router.post("/query", response_model=RetrievalQueryResponse)

apps/api/app/mcp/retrieval_server.py

@@ -71,10 +71,13 @@ def to_mcp_query_response(response: dict[str, Any]) -> dict[str, Any]:
     mcp_response: dict[str, Any] = {
         "query": response.get("query"),
         "results": results,
+        "evidence_text": response.get("evidence_text") or "",
     }
-    # Forward evidence_text for agentic mode
-    if response.get("evidence_text") is not None:
-        mcp_response["evidence_text"] = response["evidence_text"]
+
+    if response.get("stop_reason") is not None:
+        mcp_response["stop_reason"] = response["stop_reason"]
+    if response.get("decision_trace") is not None:
+        mcp_response["decision_trace"] = response["decision_trace"]
 
     return mcp_response
 
@@ -98,7 +101,8 @@ def create_retrieval_mcp_server(
         "knowhere-retrieval",
         instructions=(
             "Use this server to search published documents. "
-            "If you need information before answering, try searching with this tool."
+            "It returns evidence_text and ranked snippets, but never final answers. "
+            "Downstream agents should synthesize from the returned evidence."
         ),
         streamable_http_path=streamable_http_path,
         stateless_http=True,
@@ -107,7 +111,10 @@ def create_retrieval_mcp_server(
 
     @server.tool(
         name="retrieval.query",
-        description="Search published documents and return relevant snippets.",
+        description=(
+            "Search published documents and return relevant snippets plus unified "
+            "evidence_text for downstream answer synthesis."
+        ),
     )
     async def query_documents(
         query: Annotated[str, Field(description="What you want to search for.")],

apps/api/tests/contract/test_agentic_discovery_selection_contract.py

@@ -0,0 +1,96 @@
+from shared.services.retrieval.agentic.core.types import DocTreeNode
+from shared.services.retrieval.agentic.discovery.selection import (
+    _build_discovery_path_selections,
+    _project_discovery_hints,
+)
+
+
+def test_root_discovery_hint_is_projected_for_llm_selection() -> None:
+    hint_lines, hint_by_path = _project_discovery_hints(
+        [
+            {
+                "section_path": "Root",
+                "chunk_id": "chunk_root_relevant",
+                "summary": "document-level market chart",
+            }
+        ],
+        exclude_paths=None,
+    )
+
+    assert hint_lines == [
+        '▸ path="Root"',
+        "    document-level market chart",
+    ]
+    assert hint_by_path["Root"]["chunk_id"] == "chunk_root_relevant"
+
+
+def test_root_discovery_hint_without_llm_selection_does_not_hydrate() -> None:
+    node = DocTreeNode()
+
+    path_selections, chunk_refs = _build_discovery_path_selections(
+        selections=[],
+        hint_by_path={
+            "Root": {
+                "section_path": "Root",
+                "chunk_id": "chunk_root_relevant",
+            }
+        },
+        document_id="doc_root",
+        node=node,
+    )
+
+    assert path_selections == []
+    assert chunk_refs == []
+    assert node.confidence == {}
+
+
+def test_explicit_root_discovery_selection_with_chunk_id_uses_exact_chunk_ref() -> None:
+    node = DocTreeNode()
+
+    path_selections, chunk_refs = _build_discovery_path_selections(
+        selections=[{"path": "Root", "confidence": 0.91}],
+        hint_by_path={
+            "Root": {
+                "section_path": "Root",
+                "chunk_id": "chunk_root_relevant",
+            }
+        },
+        document_id="doc_root",
+        node=node,
+    )
+
+    assert path_selections == []
+    assert chunk_refs == [
+        {
+            "document_id": "doc_root",
+            "chunk_id": "chunk_root_relevant",
+            "section_path": "Root",
+        }
+    ]
+    assert node.confidence["Root"] == 0.91
+
+
+def test_explicit_root_discovery_selection_without_chunk_id_keeps_path_fallback() -> None:
+    node = DocTreeNode()
+
+    path_selections, chunk_refs = _build_discovery_path_selections(
+        selections=[{"path": "Root", "confidence": 0.7}],
+        hint_by_path={
+            "Root": {
+                "section_path": "Root",
+                "chunk_id": "",
+            }
+        },
+        document_id="doc_root",
+        node=node,
+    )
+
+    assert path_selections == [
+        {
+            "path": "Root",
+            "confidence": 0.7,
+            "hydrate_mode": "self_only",
+        }
+    ]
+    assert chunk_refs == []
+    assert node.confidence["Root"] == 0.7

apps/api/tests/contract/test_legacy_evidence_renderer_contract.py

@@ -0,0 +1,43 @@
+from shared.services.retrieval.hydration.legacy_evidence import render_legacy_evidence_text
+
+
+def test_render_legacy_evidence_text_should_group_documents_and_sections() -> None:
+    rows = [
+        {
+            "chunk_id": "c2",
+            "content": "second section content",
+            "sort_order": 2,
+            "source": {
+                "source_file_name": "alpha.pdf",
+                "section_path": "Alpha / Two",
+            },
+        },
+        {
+            "chunk_id": "c1",
+            "content": "first section content\nwith more detail",
+            "sort_order": 1,
+            "source": {
+                "source_file_name": "alpha.pdf",
+                "section_path": "Alpha / One",
+            },
+        },
+        {
+            "chunk_id": "c3",
+            "content": "<table><tr><td>metric</td></tr></table>",
+            "source_file_name": "beta.pdf",
+            "section_path": "Beta / Table",
+        },
+    ]
+
+    evidence_text = render_legacy_evidence_text(rows)
+
+    assert "[Document] alpha.pdf" in evidence_text
+    assert "[Document] beta.pdf" in evidence_text
+    assert "▸ Alpha / One" in evidence_text
+    assert "▸ Alpha / Two" in evidence_text
+    assert "    ┈ first section content" in evidence_text
+    assert "    ┈ with more detail" in evidence_text
+    assert "    ┈ <table><tr><td>metric</td></tr></table>" in evidence_text
+    assert "\u3010\u6587\u6863\u3011" not in evidence_text
+    assert "[\u8868\u683c\u5185\u5bb9]" not in evidence_text
+    assert "[\u56fe\u7247" not in evidence_text

apps/api/tests/contract/test_retrieval_contract.py

@@ -183,7 +183,7 @@ async def fake_retrieval_run(
         captured_requests.append(kwargs)
         return AgenticResult(
             evidence_text="policy evidence",
-            answer_text="policy answer",
+            answer_text="",
             referenced_chunks=[
                 {
                     "chunk_id": policy_document["chunk_id"],
@@ -350,8 +350,9 @@ async def test_should_return_empty_results_for_an_empty_query(
         "namespace": "default",
         "query": "",
         "router_used": "empty_query_filtered",
+        "evidence_text": "",
+        "answer_text": "",
         "results": [],
-        "answer_text": None,
         "referenced_chunks": [],
     }
 
@@ -741,7 +742,7 @@ async def run_request(
                 namespace=request.namespace,
                 query=request.query,
                 router_used="workflow_single_step",
-                answer_text="foreign reference answer",
+                answer_text="",
                 referenced_chunks=[
                     {
                         "chunk_id": foreign_document["chunk_id"],
@@ -820,7 +821,7 @@ async def run_request(
                 namespace=request.namespace,
                 query=request.query,
                 router_used="workflow_single_step",
-                answer_text="mismatched section answer",
+                answer_text="",
                 referenced_chunks=[
                     {
                         "chunk_id": visible_chunk["chunk_id"],
@@ -911,7 +912,7 @@ async def fake_retrieval_run(
         document = first_document if query == "first shared reference" else second_document
         return AgenticResult(
             evidence_text=f"evidence for {document['document_id']}",
-            answer_text=f"answer for {document['document_id']}",
+            answer_text="",
             referenced_chunks=[
                 {
                     "chunk_id": shared_chunk_id,
@@ -1022,7 +1023,7 @@ async def fake_retrieval_run(
         chunk = first_chunk if query == "first shared section" else second_chunk
         return AgenticResult(
             evidence_text=f"evidence for {chunk['section_path']}",
-            answer_text=f"answer for {chunk['section_path']}",
+            answer_text="",
             referenced_chunks=[
                 {
                     "chunk_id": shared_chunk_id,

apps/worker/.env.example

@@ -89,8 +89,10 @@ ARK_API_KEY=
 # IMAGE_MODEL=qwen3.5-flash
 # IMAGE_MODEL_MAX=qwen3.5-flash
 
-# Optional retrieval overrides have code defaults. Set RETRIEVAL_AGENTIC_ENABLED=false
-# only when you need to fall back to legacy 3-channel RRF mode.
+# Optional retrieval overrides have code defaults. Retrieval is evidence-only:
+# evidence_text is the primary output and answer_text is always empty. Set
+# RETRIEVAL_AGENTIC_ENABLED=false only when you need to fall back to legacy
+# 3-channel RRF mode.
 
 # Required for specific features: billing and analytics
 BILLING_ENABLED=false

apps/worker/app/services/document_parser/formats/markdown/parser.py

@@ -70,7 +70,13 @@ def is_skip(line):
 
 
 def heading_md_relocate(md_lines, heading_preds):
-    """Relocate markdown headings based on predicted levels (sxjg simplified logic)"""
+    """Relocate markdown headings based on predicted levels (sxjg simplified logic)
+
+    When ``_apply_merge_signals`` has merged continuation rows (level='<')
+    into a preceding heading, the DataFrame's ``heading`` column contains the
+    merged text while ``md_lines`` still has the original truncated text.
+    For positive-level headings, we use the DataFrame's ``heading`` to ensure the merged text is emitted.
+    """
 
     def remove_hash(txt):
         return re.sub(r"^\s*(#+)\s*", "", txt)
@@ -85,9 +91,13 @@ def remove_hash(txt):
             if pred_level < 0:
                 line_txt = remove_hash(line_txt)
             else:
-                # sxjg simplified: remove all #, then add correct number of #
-                clean_text = line_txt.lstrip("#").lstrip()
-                line_txt = f"{'#' * int(pred_level)} {clean_text}"
+                # Use the DataFrame heading text which may have been updated
+                # by _apply_merge_signals (continuation rows appended).
+                heading_text = str(pred_level_df["heading"].iloc[0]).strip()
+                if not heading_text:
+                    # Fallback: strip original line's '#' prefix
+                    heading_text = line_txt.lstrip("#").lstrip()
+                line_txt = f"{'#' * int(pred_level)} {heading_text}"
         # update lines
         md_lines[lid] = line_txt.strip()