Release: demo materialization, agentic retrieval refactor & hardening fixes

.github/workflows/codeql.yml

@@ -28,7 +28,7 @@ jobs:
           persist-credentials: false
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4
+        uses: github/codeql-action/init@v3
         with:
           languages: python
           queries: security-extended,security-and-quality
@@ -40,7 +40,7 @@ jobs:
           python-version: "3.11"
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4
+        uses: github/codeql-action/autobuild@v3
 
       - name: Perform CodeQL analysis
-        uses: github/codeql-action/analyze@v4
+        uses: github/codeql-action/analyze@v3

AGENTS.md

@@ -565,7 +565,9 @@ For each selected document, the agent performs a constrained Breadth-First Searc
 1. **Scope Navigation**: The document's section tree is dynamically rendered to the LLM. 
    - *Path-Based Hierarchy*: Child nodes are strictly filtered using structural path prefixes (e.g., `child_path.startswith(parent_path + ' / ')`) to maintain structural integrity and eliminate L2 duplicate rendering.
    - *Visual Constraints*: Actionable drill-down paths are explicitly prefixed with `[SELECT]` tags. The LLM system prompt tightly constrains the model to only pick paths with this tag, preventing redundant re-selection of the current scope.
-2. **Discovery Select**: The LLM reviews the specific paths flagged by Phase 1's Bottom Discovery for the current document. Selected discovery paths have their leaf chunks merged directly into the BFS document tree.
+2. **Discovery Select**: The LLM reviews the specific paths flagged by Phase 1's Bottom Discovery for the current document. Selected discovery paths are hydrated into leaf chunks (with `job_result_id` dynamically extracted from the chunks) and merged directly into the BFS document tree. 
+   - *Reparenting*: The `DocTreeNode.merge()` process reparents these discovered leaf chunks into the closest matching navigated child node. 
+   - *Orphan Leaves*: Discovered chunks whose paths are not explicitly covered by the BFS `outline_items` are rendered cleanly as `[Leaf]` items (orphans) beneath their appropriate parent, ensuring no relevant data is lost even if the BFS did not explicitly drill into that path.
 
 **Phase 3: Verdict & Revision**
 The combined document tree (BFS Navigation + Discovery) is rendered as unified evidence. The tree naturally displays structural context (outlines) alongside hydrated chunk rows (for selected leaf paths). The LLM attempts to answer the user's query:

apps/api/alembic/versions/f6a7b8c9d0e1_add_demo_materializations.py

@@ -0,0 +1,61 @@
+"""add demo materializations
+
+Revision ID: f6a7b8c9d0e1
+Revises: e5f6a7b8c9d0
+Create Date: 2026-05-12 08:25:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "f6a7b8c9d0e1"
+down_revision: Union[str, Sequence[str], None] = "e5f6a7b8c9d0"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    op.create_table(
+        "demo_materializations",
+        sa.Column("id", sa.String(length=36), nullable=False),
+        sa.Column("user_id", sa.Text(), nullable=False),
+        sa.Column("namespace", sa.String(length=255), nullable=False),
+        sa.Column("demo_source_id", sa.String(length=128), nullable=False),
+        sa.Column("document_id", sa.String(length=36), nullable=False),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"],
+            ["documents.document_id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="RESTRICT"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "user_id",
+            "namespace",
+            "demo_source_id",
+            name="uq_demo_materializations_scope_source",
+        ),
+    )
+    op.create_index(
+        "idx_demo_materializations_document",
+        "demo_materializations",
+        ["document_id"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    op.drop_index(
+        "idx_demo_materializations_document",
+        table_name="demo_materializations",
+    )
+    op.drop_table("demo_materializations")

apps/api/app/api/v1/api_v1.py

@@ -4,6 +4,7 @@
 
 from app.api.v1.routes import (
     api_key,
+    demo,
     documents,
     guest,
     jobs,
@@ -36,6 +37,9 @@
 # Unified Jobs routes
 api_router.include_router(jobs.router, prefix="/jobs", tags=["Jobs"])
 
+# Demo documents
+api_router.include_router(demo.router, prefix="/demo", tags=["Demo Documents"])
+
 # Retrieval
 api_router.include_router(retrieval.router, prefix="/retrieval", tags=["Retrieval"])
 

apps/api/app/api/v1/routes/demo.py

@@ -0,0 +1,158 @@
+"""Notebook demo document catalog routes."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from app.services.demo_document_service import DemoDocumentService
+from app.services.rate_limit.dependencies import CurrentUser, with_current_user
+from fastapi import APIRouter, Depends, Query
+from fastapi.responses import FileResponse
+from pydantic import BaseModel, Field
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from shared.core.database import get_db
+from shared.core.exceptions.domain_exceptions import NotFoundException
+
+router = APIRouter(tags=["Demo Documents"])
+
+_demo_document_service = DemoDocumentService()
+
+
+class DemoMaterializeRequest(BaseModel):
+    """Request to copy selected canonical demo sources into a namespace."""
+
+    namespace: str | None = Field(None, description="Target retrieval namespace")
+    demo_source_ids: list[str] = Field(
+        default_factory=list,
+        min_length=1,
+        description="Canonical demo source IDs to materialize",
+    )
+
+
+@router.get("/catalog")
+async def get_demo_catalog() -> dict[str, Any]:
+    """Return API-owned canonical demo source metadata and curated Q/A."""
+    return _demo_document_service.get_catalog()
+
+
+@router.get("/sources/{demo_source_id}/chunks")
+async def list_demo_source_chunks(
+    demo_source_id: str,
+    page: int = Query(1, ge=1, description="Page number"),
+    page_size: int = Query(50, ge=1, le=200, description="Items per page"),
+) -> dict[str, Any]:
+    """Return paginated canonical chunks for a demo source."""
+    response = _demo_document_service.list_chunks(
+        demo_source_id=demo_source_id,
+        page=page,
+        page_size=page_size,
+    )
+    if response is None:
+        raise _demo_source_not_found(demo_source_id)
+    return response
+
+
+@router.get("/sources/{demo_source_id}/chunks/{demo_chunk_id}")
+async def get_demo_source_chunk(
+    demo_source_id: str,
+    demo_chunk_id: str,
+) -> dict[str, Any]:
+    """Return one canonical demo chunk for citation focusing."""
+    response = _demo_document_service.get_chunk(
+        demo_source_id=demo_source_id,
+        demo_chunk_id=demo_chunk_id,
+    )
+    if response is None:
+        raise NotFoundException(
+            resource="Demo document chunk",
+            resource_id=demo_chunk_id,
+            internal_message="Demo document chunk not found",
+        )
+    return response
+
+
+@router.get("/sources/{demo_source_id}/original")
+async def get_demo_source_original(demo_source_id: str) -> FileResponse:
+    """Return the canonical original file for preview."""
+    file_path = _demo_document_service.get_original_file_path(
+        demo_source_id=demo_source_id,
+    )
+    if file_path is None:
+        raise _demo_source_not_found(demo_source_id)
+
+    return FileResponse(
+        path=file_path,
+        media_type="application/pdf",
+        filename=file_path.name,
+        content_disposition_type="inline",
+    )
+
+
+@router.get("/sources/{demo_source_id}/assets/{asset_path:path}")
+async def get_demo_source_asset(
+    demo_source_id: str,
+    asset_path: str,
+) -> FileResponse:
+    """Return a canonical parsed media or table asset for preview."""
+    file_path = _demo_document_service.get_asset_file_path(
+        demo_source_id=demo_source_id,
+        asset_path=asset_path,
+    )
+    if file_path is None:
+        raise _demo_source_not_found(demo_source_id)
+
+    return FileResponse(
+        path=file_path,
+        filename=file_path.name,
+        content_disposition_type="inline",
+    )
+
+
+@router.post("/materializations")
+async def materialize_demo_sources(
+    payload: DemoMaterializeRequest,
+    current_user: CurrentUser = Depends(with_current_user),
+    db: AsyncSession = Depends(get_db),
+) -> dict[str, Any]:
+    """Copy canonical demo sources into the authenticated user's namespace."""
+    namespace = (payload.namespace or "default").strip() or "default"
+    try:
+        materialized_sources = await _demo_document_service.materialize_sources(
+            db,
+            user_id=current_user.user_id,
+            namespace=namespace,
+            demo_source_ids=payload.demo_source_ids,
+        )
+    except KeyError as error:
+        raise _demo_source_not_found(str(error.args[0])) from error
+
+    return {
+        "namespace": namespace,
+        "sources": [
+            {
+                "demo_source_id": source.demo_source_id,
+                "document_id": source.document_id,
+                "status": source.status,
+                "title": source.title,
+                "mime_type": source.mime_type,
+                "size_bytes": source.size_bytes,
+                "chunk_count": source.chunk_count,
+                "original_file": {
+                    "url": f"/api/v1/demo/sources/{source.demo_source_id}/original",
+                    "mime_type": source.mime_type,
+                    "size_bytes": source.size_bytes,
+                    "can_download": False,
+                },
+            }
+            for source in materialized_sources
+        ],
+    }
+
+
+def _demo_source_not_found(demo_source_id: str) -> NotFoundException:
+    return NotFoundException(
+        resource="Demo document source",
+        resource_id=demo_source_id,
+        internal_message="Demo document source not found",
+    )

apps/api/app/api/v1/routes/jobs.py

@@ -6,7 +6,7 @@
 
 import os
 import uuid
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, Literal, Optional, cast
 from urllib.parse import urlparse
 
@@ -53,6 +53,7 @@
     StandardErrorObject,
 )
 from shared.services.storage.file_upload_service import FileUploadService
+from shared.utils.utc_now import utc_now_naive
 from shared.utils.url_security import (
     validate_http_url_and_resolve_ip_async,
 )
@@ -268,6 +269,15 @@ def ensure_utc(dt: Optional[datetime]) -> Optional[datetime]:
     return dt.replace(tzinfo=timezone.utc)
 
 
+def normalize_naive_utc_filter_datetime(dt: Optional[datetime]) -> Optional[datetime]:
+    """Convert a query datetime into the naive UTC form used by database columns."""
+    if dt is None:
+        return None
+    if dt.tzinfo is None or dt.utcoffset() is None:
+        return dt
+    return dt.astimezone(timezone.utc).replace(tzinfo=None)
+
+
 def require_utc(dt: Optional[datetime], *, field_name: str) -> datetime:
     """Normalize a required datetime to UTC."""
     normalized_dt = ensure_utc(dt)
@@ -678,23 +688,28 @@ async def list_jobs(
                 user_message="recent_days only supports 1, 7, or 30",
                 violations=[{"field": "recent_days", "description": "Invalid value"}],
             )
-        created_after = None
+        created_after: Optional[datetime] = None
         if recent_days:
-            from datetime import datetime, timedelta
+            created_after = utc_now_naive() - timedelta(days=recent_days)
 
-            created_after = datetime.now() - timedelta(days=recent_days)
+        normalized_start_time = normalize_naive_utc_filter_datetime(start_time)
+        normalized_end_time = normalize_naive_utc_filter_datetime(end_time)
 
-        if start_time and end_time and start_time > end_time:
+        if (
+            normalized_start_time
+            and normalized_end_time
+            and normalized_start_time > normalized_end_time
+        ):
             raise ValidationException(
                 user_message="start_time cannot be later than end_time",
                 violations=[
                     {"field": "start_time", "description": "Must be before end_time"}
                 ],
             )
         # start_time / end_time take priority over recent_days.
-        if start_time:
-            created_after = start_time
-        created_before = end_time
+        if normalized_start_time:
+            created_after = normalized_start_time
+        created_before = normalized_end_time
 
         # Count matching rows.
         total_count = await job_repo.count_jobs_by_user(
@@ -752,10 +767,8 @@ async def list_jobs(
 
                 # Compute result_url_expires_at when a download URL was issued.
                 if result_url:
-                    from datetime import datetime, timedelta
-
                     expires_in = int(result_url_info.get("expires_in", 3600))
-                    result_url_expires_at = datetime.now() + timedelta(
+                    result_url_expires_at = utc_now_naive() + timedelta(
                         seconds=expires_in
                     )