fix: resolve E2E agentic retrieval bugs and implement dual-priority scoring

apps/worker/app/services/document_parser/doc_parser.py

@@ -608,6 +608,21 @@ def iter_block_items(doc_data):
                 # <v:imagedata> piece-by-piece loses textual overlay and positioning. 
                 # Future plan: Use LibreOffice headless conversion to render the entire document 
                 # and map the perfectly rendered images back to the layout via text anchors.
+                #
+                # Temporary: detect VML-only paragraphs and inject a placeholder so the
+                # paragraph isn't silently swallowed, leaving its parent section empty.
+                if not text and not seen_rids:
+                    # No text and no DrawingML images — check for VML content
+                    vml_groups = elem.xpath(".//v:group", namespaces=ns)
+                    vml_images_check = elem.xpath(".//v:imagedata", namespaces=ns)
+                    if vml_groups or vml_images_check:
+                        vml_placeholder = "[VML graphic \u2014 extraction not yet supported]"
+                        yield ele_num, vml_placeholder, "PTXT", None
+                        ele_num += 1
+                        logger.debug(
+                            f"Injected VML placeholder for paragraph with "
+                            f"{len(vml_groups)} v:group, {len(vml_images_check)} v:imagedata"
+                        )
                 """
                 # images (VML: <v:imagedata>) — convert to PNG
                 from PIL import Image as PILImage
@@ -942,6 +957,13 @@ def convert_doc2dics(
     for _, row in leaf_dics.iterrows():
         key = row["path_identifier"]
 
+        # Skip leaf nodes with no actual content (empty heading-only sections)
+        content_lst = row["content_lst"]
+        joined = "\n".join(content_lst).strip()
+        if not joined:
+            logger.debug(f"Skipping empty leaf node: {key}")
+            continue
+
         # Build tentative path to check for duplicates
         tentative_path = doc_name + split_char + key
 
@@ -954,7 +976,7 @@ def convert_doc2dics(
             path_counter[tentative_path] = 1
 
         path_keys.append((doc_name + split_char + key))
-        bottom_content = "\n".join(row["content_lst"])
+        bottom_content = joined
         bottom_tokens = tokenize2stw_remove(
             [bottom_content], base_llm_paras["stopwords"]
         )

apps/worker/app/services/document_parser/md_parser.py

@@ -318,7 +318,10 @@ def parse_md(
     os.makedirs(tb_dir, exist_ok=True)
     img_dir = os.path.join(output_dir, "images")
     if os.path.isdir(img_dir):
-        shutil.rmtree(img_dir)
+        # Only remove parse_md's own output (image-N-*) from previous runs
+        for fname in os.listdir(img_dir):
+            if re.match(r"^image-\d+", fname):
+                os.remove(os.path.join(img_dir, fname))
     os.makedirs(img_dir, exist_ok=True)
 
     # initialize vars

packages/shared-python/shared/services/retrieval/__init__.py

@@ -1,4 +1,4 @@
-from .app_service import list_lexical_chunks, merge_channels_rrf, run_retrieval_query
+from .app_service import merge_channels_rrf, run_retrieval_query
 from .cache_service import (
     bump_retrieval_namespace_cache_version,
     get_cached_retrieval_query_result,
@@ -13,7 +13,6 @@
 __all__ = [
     "create_retrieval_llm_fn",
     "run_retrieval_query",
-    "list_lexical_chunks",
     "merge_channels_rrf",
     "DocumentGraphService",
     "GraphQueryService",

packages/shared-python/shared/services/retrieval/agent_navigate.py

@@ -33,6 +33,10 @@
 Do not include any explanation.
 """
 
+_VALID_HYDRATE_MODES = frozenset({
+    'outline', 'chunks', 'assets_only', 'image_only', 'table_only',
+})
+
 _SCOPE_NAV_PROMPT = """\
 You are a document navigation assistant.
 
@@ -41,6 +45,7 @@
 
 Below are candidate section paths at this scope level (up to 2 depth levels).
 Indented items are sub-items of the item above.
+Each item shows text/image/table counts.
 Select section paths directly. A selected section path represents the chunks
 under that section subtree; do not ask to drill deeper.
 
@@ -51,9 +56,19 @@
 User query: {query}
 
 Select the most relevant section paths (at most {max_select}).
+If NO section path is relevant to the query, you MUST return an empty array []. Do not force-select irrelevant sections.
 Prefer specific sub-items over broad parents when both are listed and the sub-item is sufficient.
+
+For each selected path, assign a confidence score (0.0 to 1.0) where 1.0 means exactly answers the query and 0.5 means tangentially related.
+Also choose a hydrate_mode:
+- "chunks"       (default) return all text/image/table chunks
+- "outline"      return only section title + summary, no chunk content
+- "assets_only"  return only image and table chunks
+- "image_only"   return only image chunks
+- "table_only"   return only table chunks
+
 Return ONLY a JSON array:
-[{{"path": "section/path", "confidence": 0.9}}, ...]
+[{{"path": "section/path", "confidence": <float>, "hydrate_mode": "chunks"}}, ...]
 Do not include any explanation.
 """
 
@@ -106,24 +121,27 @@ def _parse_chunk_path_selections(text: str) -> list[dict[str, Any]]:
     """Parse chunk path selections from LLM output.
 
     Accepts either a legacy JSON array of strings or a structured array of
-    objects with `path` and optional `confidence`.
+    objects with `path`, optional `confidence`, and optional `hydrate_mode`.
     """
     payload = _extract_json_array_payload(text)
     selections: list[dict[str, Any]] = []
     for item in payload:
         if isinstance(item, str):
             path = item.strip()
             if path:
-                selections.append({'path': path, 'confidence': None})
+                selections.append({'path': path, 'confidence': None, 'hydrate_mode': 'chunks'})
             continue
         if not isinstance(item, dict):
             continue
         path = str(item.get('path') or item.get('chunk_path') or '').strip()
         if not path:
             continue
+        raw_mode = str(item.get('hydrate_mode') or '').strip().lower()
+        hydrate_mode = raw_mode if raw_mode in _VALID_HYDRATE_MODES else 'chunks'
         selections.append({
             'path': path,
             'confidence': _normalize_confidence(item.get('confidence')),
+            'hydrate_mode': hydrate_mode,
         })
     return selections
 
@@ -220,8 +238,8 @@ def _format_items_for_llm(
     Always shows ALL items (L1 + L2).  Overflow controls whether
     summaries are included — not which levels are shown.
 
-    Normal:   path + title + chunk_count + assets_count + summary
-    Overflow: path + title + chunk_count + assets_count  (no summary)
+    Normal:   path + title + text=N image=I table=T + summary
+    Overflow: path + title + text=N image=I table=T  (no summary)
 
     Returns (text, overflowed).
     """
@@ -238,10 +256,13 @@ def _render_line(item: dict, include_summary: bool) -> str:
         line = f'{indent}- path="{item["path"]}"  title="{item["title"]}"'
         chunk_count = item.get('chunk_count', 0)
         if chunk_count > 0:
-            line += f'  chunks={chunk_count}'
-        assets = item.get('assets_count', 0)
-        if assets > 0:
-            line += f'  assets={assets}'
+            line += f'  text={chunk_count}'
+        image_count = item.get('image_count', 0)
+        if image_count > 0:
+            line += f'  image={image_count}'
+        table_count = item.get('table_count', 0)
+        if table_count > 0:
+            line += f'  table={table_count}'
         if include_summary:
             summary = item.get('summary') or item.get('title', '')
             if summary:
@@ -410,16 +431,20 @@ async def _load_child_sections(
     document_id: str,
     job_result_id: str,
     scope_path: str | None = None,
+    exclude_paths: set[str] | None = None,
 ) -> list[dict]:
     """Load the next 2 available section depth bands under *scope_path*.
 
     Returns a flat list sorted by sort_order, each item:
-        {path, title, summary, chunk_count, assets_count, level}
+        {path, title, summary, chunk_count, image_count, table_count, level}
 
     - level=1: nearest available descendant depth under scope
     - level=2: second nearest available descendant depth under scope
     - chunk_count: text chunks under this section (excluding image/table)
-    - assets_count: image + table chunks under this section
+    - image_count: image chunks under this section
+    - table_count: table chunks under this section
+    - exclude_paths: paths already seen in prior revision rounds;
+      any path matching (exact or subtree) is skipped
     """
     # ── Fetch all sections for this document revision ────────────────────
     stmt = (
@@ -465,6 +490,7 @@ async def _load_child_sections(
     # round's relative L1/L2 instead of synthesizing missing ancestors.
     visible_sections: list[tuple[str, dict, int]] = []
     visible_depths: set[int] = set()
+    _excl = exclude_paths or set()
     for path, meta in all_sections.items():
         parts = meta['parts']
         if scope_parts and (
@@ -474,6 +500,12 @@ async def _load_child_sections(
         relative_depth = len(parts) - scope_depth
         if relative_depth < 1:
             continue
+        # Skip paths already seen in prior revision rounds
+        if _excl and any(
+            path == ep or path.startswith(ep + ' / ') or ep.startswith(path + ' / ')
+            for ep in _excl
+        ):
+            continue
         visible_sections.append((path, meta, relative_depth))
         visible_depths.add(relative_depth)
 
@@ -497,14 +529,15 @@ async def _load_child_sections(
                 'level': level,
                 'sort_order': meta['sort_order'],
                 'chunk_count': 0,
-                'assets_count': 0,
+                'image_count': 0,
+                'table_count': 0,
                 'section_id': meta['section_id'],
             }
 
     if not items_by_path:
         return []
 
-    # ── Count chunks per section (text vs assets) ───────────────────────
+    # ── Count chunks per section (text / image / table) ──────────────────
     section_ids = [meta['section_id'] for meta in all_sections.values()]
     if section_ids:
         from sqlalchemy import case, literal_column
@@ -518,18 +551,23 @@ async def _load_child_sections(
                 ).label('text_count'),
                 func.count(
                     case(
-                        (DocumentChunk.chunk_type.in_(['image', 'table']), literal_column('1')),
+                        (DocumentChunk.chunk_type == 'image', literal_column('1')),
+                    )
+                ).label('image_count'),
+                func.count(
+                    case(
+                        (DocumentChunk.chunk_type == 'table', literal_column('1')),
                     )
-                ).label('asset_count'),
+                ).label('table_count'),
             )
             .where(DocumentChunk.document_id == document_id)
             .where(DocumentChunk.job_result_id == job_result_id)
             .where(DocumentChunk.section_id.in_(section_ids))
             .group_by(DocumentChunk.section_id)
         )
         chunk_rows = (await db.execute(chunk_stmt)).all()
-        section_id_counts: dict[str, tuple[int, int]] = {
-            sid: (int(tc), int(ac)) for sid, tc, ac in chunk_rows
+        section_id_counts: dict[str, tuple[int, int, int]] = {
+            sid: (int(tc), int(ic), int(tbc)) for sid, tc, ic, tbc in chunk_rows
         }
     else:
         section_id_counts = {}
@@ -538,15 +576,16 @@ async def _load_child_sections(
     sid_to_path = {meta['section_id']: path for path, meta in all_sections.items()}
 
     # Aggregate chunk counts upward: each item gets counts from itself + descendants
-    for sid, (text_c, asset_c) in section_id_counts.items():
+    for sid, (text_c, img_c, tbl_c) in section_id_counts.items():
         chunk_path = sid_to_path.get(sid, '')
         if not chunk_path:
             continue
         # Add to every ancestor item that is in our items_by_path
         for item_path, item in items_by_path.items():
             if chunk_path == item_path or chunk_path.startswith(item_path + ' / '):
                 item['chunk_count'] += text_c
-                item['assets_count'] += asset_c
+                item['image_count'] += img_c
+                item['table_count'] += tbl_c
 
     # ── Sort: interleave L2 under their L1 parent ────────────────────────
     #