Release staging to main - 2026-05-19

apps/api/.env.example

@@ -90,6 +90,7 @@ ARK_API_KEY=
 # File handling defaults
 SUPPORTED_EXTENSIONS=.doc,.docx,.pdf,.txt,.xls,.xlsx,.csv,.pptx,.jpg,.jpeg,.png,.md
 MAX_FILE_SIZE=104857600
+MAX_PDF_PAGE_LIMIT=600
 
 # Required for specific features: webhooks and callbacks
 WEBHOOK_MASTER_KEY=
@@ -115,7 +116,9 @@ ILOVEAPI_SECRET_KEY=
 # MINERU_TOKEN_RPM_LIMIT=300
 # MINERU_TOKEN_DAILY_LIMIT=10000
 # MINERU_TOKEN_COOLDOWN_SECONDS=60
-# FORCE_MINERU_UPLOAD_ENABLED=false
+# False prefers S3 URL mode when the source object is reusable. Set true only
+# when MinerU cannot fetch your storage URLs and direct upload is required.
+# MINERU_UPLOAD_MODE_ENABLED=false
 # ILOVEAPI_BASE_URL=https://api.ilovepdf.com/v1
 # ILOVEAPI_TIMEOUT=120
 

apps/api/app/api/v1/routes/retrieval.py

@@ -17,6 +17,10 @@
 router = APIRouter(tags=["Retrieval"])
 
 
+def _is_none(value: object) -> bool:
+    return value is None
+
+
 class ExcludeSection(BaseModel):
     document_id: str
     section_path: str
@@ -83,6 +87,9 @@ class RetrievalQueryResponse(BaseModel):
     answer_text: str | None = None
     referenced_chunks: list[dict] = Field(default_factory=list)
     results: list[dict] = Field(default_factory=list)
+    evidence_text: str | None = Field(default=None, exclude_if=_is_none)
+    stop_reason: str | None = Field(default=None, exclude_if=_is_none)
+    failure_reason: str | None = Field(default=None, exclude_if=_is_none)
 
 
 @router.post("/query", response_model=RetrievalQueryResponse)

apps/api/pyproject.toml

@@ -2,7 +2,7 @@
 name = "knowhere-api-app"
 version = "0.1.0"
 description = "Knowhere API service"
-requires-python = ">=3.11,<3.14"
+requires-python = ">=3.11"
 dependencies = [
     "knowhere-shared",
     "fastapi==0.135.1",
@@ -13,7 +13,7 @@ dependencies = [
     "redis==5.3.1",
     "celery==5.4.0",
     "stripe==13.0.1",
-    "pydantic==2.11.7",
+    "pydantic==2.13.4",
     "PyJWT==2.12.0",
     "qstash==3.2.0",
     "httpx==0.28.1",

apps/api/tests/contract/test_retrieval_contract.py

@@ -1,7 +1,7 @@
-from collections.abc import Callable
+from collections.abc import Callable, Coroutine, Sequence
 from contextlib import AbstractAsyncContextManager
 from datetime import datetime, timezone
-from typing import cast
+from typing import Any, cast
 from uuid import uuid4
 
 import pytest
@@ -14,6 +14,9 @@
 from shared.services.retrieval.workflow.run_request import WorkflowRunRequest
 from shared.services.retrieval.workflow.types import PlannedStep, QueryPlan, WorkflowResult
 
+LLMFnInput = str | Sequence[dict[str, Any]]
+LLMFn = Callable[[LLMFnInput], Coroutine[Any, Any, str]]
+
 
 async def _seed_retrieval_document(
     *,
@@ -23,6 +26,9 @@ async def _seed_retrieval_document(
     section_path: str,
     content: str,
     chunk_id: str | None = None,
+    chunk_type: str = "text",
+    file_path: str | None = None,
+    chunk_metadata: dict[str, Any] | None = None,
 ) -> dict[str, str]:
     document_id = f"doc_{uuid4().hex[:12]}"
     job_id = f"job_{uuid4().hex[:12]}"
@@ -80,9 +86,11 @@ async def _seed_retrieval_document(
         document_id=document_id,
         job_result_id=job_result_id,
         section_id=section_id,
-        chunk_type="text",
+        chunk_type=chunk_type,
         content=content,
         section_path=section_path,
+        file_path=file_path,
+        chunk_metadata=chunk_metadata,
     )
 
     return {
@@ -600,6 +608,121 @@ async def test_agentic_retrieval_should_reference_discovery_content_when_navigat
     }
 
 
+@pytest.mark.asyncio
+async def test_agentic_retrieval_should_not_send_table_artifacts_to_vlm(
+    developer_api_client_factory: Callable[
+        [], AbstractAsyncContextManager[AsyncClient]
+    ],
+    monkeypatch: MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("LLM_MOCK_ENABLED", "true")
+    vlm_calls: list[LLMFnInput] = []
+
+    async def fake_vlm(prompt: LLMFnInput) -> str:
+        vlm_calls.append(prompt)
+        return '{"status":"DONE","answer":"unexpected table VLM answer"}'
+
+    def fake_create_retrieval_vlm_fn(**_kwargs: object) -> LLMFn:
+        return fake_vlm
+
+    class FakeResultStorage:
+        def generate_artifact_url(
+            self,
+            *,
+            job_id: str,
+            artifact_ref: str,
+            expires_in: int = 3600,
+        ) -> str | None:
+            del expires_in
+            return f"https://assets.example.com/{job_id}/{artifact_ref}"
+
+        def normalize_artifact_ref(self, artifact_ref: str | None) -> str | None:
+            if not artifact_ref:
+                return None
+            normalized = artifact_ref.strip().replace("\\", "/").lstrip("/")
+            if not normalized:
+                return None
+            root_dir = normalized.split("/", 1)[0]
+            if root_dir not in {"images", "tables"}:
+                return None
+            return normalized
+
+    def fake_get_result_storage() -> FakeResultStorage:
+        return FakeResultStorage()
+
+    async with developer_api_client_factory() as api_client:
+        monkeypatch.setattr(
+            "shared.services.retrieval.llm_adapter.create_retrieval_vlm_fn",
+            fake_create_retrieval_vlm_fn,
+        )
+        monkeypatch.setattr(
+            "shared.services.retrieval.hydration.assets.get_result_storage",
+            fake_get_result_storage,
+        )
+        table_document = await _seed_retrieval_document(
+            user_id="local-dev-user",
+            namespace="contract-agentic-table-vlm-filter",
+            source_file_name="table-report.md",
+            section_path="Realdata Results Summary / Main Metrics",
+            content=(
+                "<table><tr><th>budget</th><th>metric</th><th>value</th></tr>"
+                "<tr><td>1000</td><td>Flat inspect_evidence_score_mean</td>"
+                "<td>0.5674</td></tr></table>"
+            ),
+            chunk_type="table",
+            file_path="tables/table-0-main-metrics.html",
+        )
+        await _seed_retrieval_document(
+            user_id="local-dev-user",
+            namespace="contract-agentic-table-vlm-filter",
+            source_file_name="filler-table-report.md",
+            section_path="Appendix / Filler Metrics",
+            content=(
+                "<table><tr><th>metric</th><th>value</th></tr>"
+                "<tr><td>unrelated filler metric</td><td>999</td></tr></table>"
+            ),
+            chunk_type="table",
+            file_path="tables/table-1-filler-metrics.html",
+        )
+
+        response = await api_client.post(
+            "/api/v1/retrieval/query",
+            json={
+                "namespace": "contract-agentic-table-vlm-filter",
+                "query": "budget 1000 Flat inspect_evidence_score_mean",
+                "top_k": 1,
+                "data_type": 4,
+                "use_agentic": True,
+            },
+        )
+
+    assert response.status_code == 200
+
+    response_json = cast(dict[str, object], response.json())
+    referenced_chunks = cast(list[dict[str, object]], response_json["referenced_chunks"])
+    results = cast(list[dict[str, object]], response_json["results"])
+
+    assert response_json["router_used"] == "workflow_single_step"
+    assert vlm_calls == []
+    matching_references = [
+        reference
+        for reference in referenced_chunks
+        if reference["chunk_id"] == table_document["chunk_id"]
+    ]
+    assert len(matching_references) == 1
+    assert matching_references[0]["document_id"] == table_document["document_id"]
+    assert matching_references[0]["chunk_type"] == "table"
+    assert matching_references[0]["section_path"] == table_document["section_path"]
+    assert matching_references[0]["file_path"] == "tables/table-0-main-metrics.html"
+    assert matching_references[0]["job_id"] == table_document["job_id"]
+    assert str(matching_references[0]["asset_url"]).startswith(
+        "https://assets.example.com/"
+    )
+    assert len(results) == 1
+    assert results[0]["chunk_type"] == "table"
+    assert _result_source(results[0])["document_id"] == table_document["document_id"]
+
+
 @pytest.mark.asyncio
 async def test_agentic_retrieval_should_not_hydrate_references_outside_request_scope(
     developer_api_client_factory: Callable[

apps/worker/.env.example

@@ -106,15 +106,17 @@ ILOVEAPI_SECRET_KEY=
 # MINERU_TOKEN_RPM_LIMIT=300
 # MINERU_TOKEN_DAILY_LIMIT=10000
 # MINERU_TOKEN_COOLDOWN_SECONDS=60
-# FORCE_MINERU_UPLOAD_ENABLED=false
+# False prefers S3 URL mode when the source object is reusable. Set true only
+# when MinerU cannot fetch your storage URLs and direct upload is required.
+# MINERU_UPLOAD_MODE_ENABLED=false
 # ILOVEAPI_BASE_URL=https://api.ilovepdf.com/v1
 # ILOVEAPI_TIMEOUT=120
 
 # File handling defaults
 SUPPORTED_EXTENSIONS=.doc,.docx,.pdf,.txt,.xls,.xlsx,.csv,.pptx,.jpg,.jpeg,.png,.md
 MAX_FILE_SIZE=104857600
+MAX_PDF_PAGE_LIMIT=600
 
 # Legacy parser compatibility fields.
 # ALL_DF_COLS=content,path,type,length,keywords,summary,know_id,tokens,connectto,addtime,page_nums
 # SPLIT_CHAR=/
-

apps/worker/app/services/document_parser/orchestration/parse_session.py

@@ -16,8 +16,6 @@
 from shared.core.config import settings
 from shared.core.exceptions.domain_exceptions import ValidationException
 
-PDF_PAGE_LIMIT = 600
-
 
 @dataclass(frozen=True)
 class ParseSession:
@@ -111,16 +109,17 @@ def build_parse_session(parse_input: ParseInput) -> ParseSession:
                 f"ℹ️ VLM rejected atlas for {parse_input.filename}, routing as generic"
             )
 
-    if profile.file_type == "pdf" and profile.page_count > PDF_PAGE_LIMIT:
+    pdf_page_limit = settings.MAX_PDF_PAGE_LIMIT
+    if profile.file_type == "pdf" and profile.page_count > pdf_page_limit:
         raise ValidationException(
             user_message=(
-                f"Document too large: {profile.page_count} pages exceeds the {PDF_PAGE_LIMIT}-page limit. "
+                f"Document too large: {profile.page_count} pages exceeds the {pdf_page_limit}-page limit. "
                 "Please split the document and upload in smaller batches."
             ),
             violations=[
                 {
                     "field": "page_count",
-                    "description": f"PDF has {profile.page_count} pages, limit is {PDF_PAGE_LIMIT}",
+                    "description": f"PDF has {profile.page_count} pages, limit is {pdf_page_limit}",
                 }
             ],
         )

apps/worker/app/services/document_parser/providers/mineru/pdf_service.py

@@ -32,10 +32,7 @@
 
 
 def _should_use_mineru_s3_url_mode(s3_key: Optional[str]) -> bool:
-    if settings.FORCE_MINERU_UPLOAD_ENABLED:
-        return False
-
-    return settings.ENVIRONMENT != "development" and s3_key is not None
+    return not settings.MINERU_UPLOAD_MODE_ENABLED and s3_key is not None
 
 
 def _log_mineru_url_mode_storage_fallback(

apps/worker/pyproject.toml

@@ -2,7 +2,7 @@
 name = "knowhere-worker-app"
 version = "0.1.0"
 description = "Knowhere Worker service"
-requires-python = ">=3.11,<3.14"
+requires-python = ">=3.11"
 dependencies = [
     "knowhere-shared",
     "numpy==2.2.6",

apps/worker/requirements.txt

@@ -33,11 +33,9 @@ argon2-cffi-bindings==25.1.0
     # via argon2-cffi
 asgiref==3.11.1
     # via opentelemetry-instrumentation-asgi
-async-timeout==5.0.1 ; python_full_version < '3.12'
-    # via
-    #   asyncpg
-    #   redis
-asyncpg==0.29.0
+async-timeout==5.0.1 ; python_full_version < '3.11.3'
+    # via redis
+asyncpg==0.31.0
     # via knowhere-shared
 attrs==26.1.0
     # via aiohttp
@@ -333,24 +331,24 @@ psycogreen==1.0.2
     # via
     #   knowhere-shared
     #   knowhere-worker-app
-psycopg2-binary==2.9.9
+psycopg2-binary==2.9.12
     # via knowhere-shared
 pwdlib==0.3.0
     # via fastapi-users
 pycparser==3.0 ; implementation_name != 'PyPy'
     # via cffi
 pycryptodome==3.23.0
     # via oss2
-pydantic==2.11.7
+pydantic==2.13.4
     # via
     #   fastapi
     #   knowhere-shared
     #   openai
     #   pptx2md
     #   pydantic-settings
-pydantic-core==2.33.2
+pydantic-core==2.46.4
     # via pydantic
-pydantic-settings==2.10.1
+pydantic-settings==2.14.1
     # via knowhere-shared
 pygments==2.20.0
     # via