feat: TOC physical rehome + page_memory render scope isolation

apps/worker/app/services/document_agent/calibration/scan.py

@@ -5,8 +5,10 @@
 window, feeding each round's cursor into the next one, so a miss never re-opens
 pages that were already inspected.
 
-Each round still covers ``window_schedule[i]`` pages, but pages are inspected
-one-at-a-time concurrently (never batched into a single VLM call).
+Each round still covers ``window_schedule[i]`` pages. PDF→PNG for the whole
+window is rendered in one serial child-process call (same discipline as TOC
+extract: never drive the gevent PyMuPDF pool from a ThreadPool). VLM inspect
+then runs one page per call, concurrently.
 """
 
 from __future__ import annotations
@@ -25,6 +27,11 @@
 )
 from app.services.document_agent.manifest import ToolContext
 from app.services.document_agent.tools.inspect_pages import inspect_pages
+from app.services.document_agent.visual import render_pages
+from shared.services.ai.token_tracking import (
+    bind_token_tracker,
+    get_current_token_tracker_root_id,
+)
 
 DEFAULT_WINDOW_SCHEDULE: tuple[int, ...] = (2, 4, 6, 10)
 
@@ -111,6 +118,7 @@ def _inspect_one_page(
     ctx: ToolContext,
     title: str,
     page: int,
+    rendered_page: dict[str, Any],
 ) -> PageInspectResult:
     result = inspect_pages(
         ctx,
@@ -122,6 +130,7 @@ def _inspect_one_page(
             "folder_name": "calibration_scan",
             "prefix": "scan",
             "usage_task": "calibration.scan_title_forward",
+            "rendered_pages": [rendered_page],
         },
     )
     if result.status != "ok":
@@ -142,14 +151,63 @@ def _inspect_pages_concurrent(
     title: str,
     pages: list[int],
 ) -> list[PageInspectResult]:
-    """Inspect each page alone; keep page order in the returned list."""
+    """Serial window render, then concurrent single-page VLM inspect."""
+    rendered = render_pages(
+        ctx,
+        pages,
+        folder_name="calibration_scan",
+        prefix="scan",
+        timeout=120,
+    )
+    rendered_by_page = {
+        int(item["page"]): {
+            "page": int(item["page"]),
+            "png_path": str(item["png_path"]),
+        }
+        for item in rendered
+        if item.get("page") is not None and item.get("png_path")
+    }
+    missing = [page for page in pages if page not in rendered_by_page]
+    if missing:
+        return [
+            PageInspectResult(
+                page=page,
+                found=False,
+                error=(
+                    f"render failed for pages={missing}"
+                    if page in missing
+                    else "render incomplete"
+                ),
+            )
+            for page in pages
+        ]
+
     if len(pages) == 1:
-        return [_inspect_one_page(ctx=ctx, title=title, page=pages[0])]
+        page = pages[0]
+        return [
+            _inspect_one_page(
+                ctx=ctx,
+                title=title,
+                page=page,
+                rendered_page=rendered_by_page[page],
+            )
+        ]
+
+    token_tracker_root_id = get_current_token_tracker_root_id()
+
+    def _inspect_one_page_with_tracking(page: int) -> PageInspectResult:
+        with bind_token_tracker(token_tracker_root_id):
+            return _inspect_one_page(
+                ctx=ctx,
+                title=title,
+                page=page,
+                rendered_page=rendered_by_page[page],
+            )
 
     by_page: dict[int, PageInspectResult] = {}
     with ThreadPoolExecutor(max_workers=len(pages)) as pool:
         futures = {
-            pool.submit(_inspect_one_page, ctx=ctx, title=title, page=page): page
+            pool.submit(_inspect_one_page_with_tracking, page): page
             for page in pages
         }
         for future in as_completed(futures):
@@ -174,9 +232,10 @@ def scan_title_forward(
     """Scan forward from ``start_page`` until the title is found or rounds run out.
 
     Each round covers ``window_schedule[i]`` consecutive pages starting at the
-    cursor left by the previous round. Pages inside a round are inspected
-    concurrently, one page per VLM call; the earliest true page wins. A page
-    error without a hit is logged and the scan continues to the next window.
+    cursor left by the previous round. The window is rendered once serially,
+    then each page is inspected via VLM concurrently; the earliest true page
+    wins. A page error without a hit is logged and the scan continues to the
+    next window.
     """
     scanned: list[int] = []
     rounds: list[ScanRound] = []

apps/worker/app/services/document_agent/structure/hierarchy_locator.py

@@ -136,11 +136,11 @@ def first_leaf_start_under(
     parent_titles: tuple[str, ...],
     match_overrides: dict[tuple[str, ...], TitleMatch],
 ) -> int | None:
-    """Min start page among located leaves under *node*; None if none located."""
+    """Min start page among structural leaves; rehome attachments do not bound scope."""
     min_page: int | None = None
     for leaf_path, _leaf in iter_leaf_title_nodes([node], parent_titles=parent_titles):
         match = match_overrides.get(leaf_path)
-        if match is None:
+        if match is None or _is_rehome_match(match):
             continue
         if min_page is None or match.page < min_page:
             min_page = match.page
@@ -179,7 +179,12 @@ def resolve_hierarchy_page_ranges(
         match_overrides=match_overrides or {},
         resolved=resolved,
     )
-    return resolved
+    rehome_ranges = _attach_rehome_ranges(
+        nodes,
+        match_overrides=match_overrides or {},
+        structural_ranges=resolved,
+    )
+    return _ranges_in_tree_order(nodes, [*resolved, *rehome_ranges])
 
 
 def coverage_by_path(
@@ -240,10 +245,19 @@ def _resolve_siblings(
     match_overrides: dict[tuple[str, ...], TitleMatch],
     resolved: list[ResolvedHierarchyRange],
 ) -> None:
+    boundary_nodes = [
+        node
+        for node in nodes
+        if not _is_rehome_leaf(
+            node,
+            (*parent_titles, node.title),
+            match_overrides,
+        )
+    ]
     located: list[tuple[TitleNode, int, TitleMatch | None]] = []
     lower_bound = parent_scope.start
 
-    for index, node in enumerate(nodes):
+    for index, node in enumerate(boundary_nodes):
         path_titles = (*parent_titles, node.title)
         pages = _allowed_pages_between(lower_bound, parent_scope.end, allowed_pages)
         match = _locate_match_for_node(
@@ -259,9 +273,9 @@ def _resolve_siblings(
         located.append((node, start_page, match))
         if match is not None:
             lower_bound = start_page
-        elif index + 1 < len(nodes):
+        elif index + 1 < len(boundary_nodes):
             next_match = _find_next_located_sibling(
-                nodes=nodes,
+                nodes=boundary_nodes,
                 start_index=index + 1,
                 lower_bound=lower_bound,
                 parent_end=parent_scope.end,
@@ -334,6 +348,114 @@ def _resolve_siblings(
         )
 
 
+def _is_rehome_match(match: TitleMatch | None) -> bool:
+    if match is None:
+        return False
+    return bool((match.evidence or {}).get("toc_rehome"))
+
+
+def _is_rehome_leaf(
+    node: TitleNode,
+    path_titles: tuple[str, ...],
+    match_overrides: dict[tuple[str, ...], TitleMatch],
+) -> bool:
+    return not node.children and _is_rehome_match(match_overrides.get(path_titles))
+
+
+def _rehome_host_range(
+    *,
+    structural_ranges: list[ResolvedHierarchyRange],
+    physical_page: int,
+) -> ResolvedHierarchyRange:
+    candidates = [
+        item
+        for item in structural_ranges
+        if item.start_page <= physical_page <= item.end_page
+    ]
+    if not candidates:
+        raise ValueError(
+            "rehome leaf has no resolved host scope "
+            f"page={physical_page}"
+        )
+    latest_start = max(item.start_page for item in candidates)
+    return next(
+        item
+        for item in reversed(candidates)
+        if item.start_page == latest_start
+    )
+
+
+def _attach_rehome_ranges(
+    nodes: list[TitleNode],
+    *,
+    match_overrides: dict[tuple[str, ...], TitleMatch],
+    structural_ranges: list[ResolvedHierarchyRange],
+    parent_titles: tuple[str, ...] = (),
+) -> list[ResolvedHierarchyRange]:
+    attached: list[ResolvedHierarchyRange] = []
+    for node in nodes:
+        path_titles = (*parent_titles, node.title)
+        if _is_rehome_leaf(node, path_titles, match_overrides):
+            match = match_overrides.get(path_titles)
+            if match is None:
+                raise ValueError(f"rehome leaf override missing path={path_titles!r}")
+            host_range = _rehome_host_range(
+                structural_ranges=structural_ranges,
+                physical_page=match.page,
+            )
+            attached.append(
+                ResolvedHierarchyRange(
+                    title=node.title,
+                    level=node.level,
+                    start_page=host_range.start_page,
+                    end_page=host_range.end_page,
+                    path_titles=path_titles,
+                    match=match,
+                    evidence={
+                        **_range_evidence(match),
+                        "status": "rehome_attached",
+                        "skeleton_kind": "rehome_attachment",
+                        "scope_host_path": list(host_range.path_titles),
+                    },
+                )
+            )
+            continue
+        if node.children:
+            attached.extend(
+                _attach_rehome_ranges(
+                    node.children,
+                    match_overrides=match_overrides,
+                    structural_ranges=structural_ranges,
+                    parent_titles=path_titles,
+                )
+            )
+    return attached
+
+
+def _ranges_in_tree_order(
+    nodes: list[TitleNode],
+    ranges: list[ResolvedHierarchyRange],
+) -> list[ResolvedHierarchyRange]:
+    order = {
+        path: index
+        for index, (path, _node) in enumerate(_iter_title_nodes(nodes))
+    }
+    return sorted(ranges, key=lambda item: order[item.path_titles])
+
+
+def _iter_title_nodes(
+    nodes: list[TitleNode],
+    *,
+    parent_titles: tuple[str, ...] = (),
+) -> list[tuple[tuple[str, ...], TitleNode]]:
+    items: list[tuple[tuple[str, ...], TitleNode]] = []
+    for node in nodes:
+        path_titles = (*parent_titles, node.title)
+        items.append((path_titles, node))
+        items.extend(_iter_title_nodes(node.children, parent_titles=path_titles))
+    return items
+
+
 def _locate_match_for_node(
     node: TitleNode,
     *,
@@ -391,7 +513,7 @@ def _infer_start_from_descendant_overrides(
     min_match: TitleMatch | None = None
     for leaf_path, _leaf_node in leaves:
         m = match_overrides.get(leaf_path)
-        if m is None:
+        if m is None or _is_rehome_match(m):
             continue
         if m.page not in scope_pages:
             continue