diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 366d9fdf9..8de503391 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -28,7 +28,7 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@v3 with: languages: python queries: security-extended,security-and-quality @@ -40,7 +40,7 @@ jobs: python-version: "3.11" - name: Autobuild - uses: github/codeql-action/autobuild@v4 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@v3 diff --git a/AGENTS.md b/AGENTS.md index 48a62326b..1bec3165d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -565,7 +565,9 @@ For each selected document, the agent performs a constrained Breadth-First Searc 1. **Scope Navigation**: The document's section tree is dynamically rendered to the LLM. - *Path-Based Hierarchy*: Child nodes are strictly filtered using structural path prefixes (e.g., `child_path.startswith(parent_path + ' / ')`) to maintain structural integrity and eliminate L2 duplicate rendering. - *Visual Constraints*: Actionable drill-down paths are explicitly prefixed with `[SELECT]` tags. The LLM system prompt tightly constrains the model to only pick paths with this tag, preventing redundant re-selection of the current scope. -2. **Discovery Select**: The LLM reviews the specific paths flagged by Phase 1's Bottom Discovery for the current document. Selected discovery paths have their leaf chunks merged directly into the BFS document tree. +2. **Discovery Select**: The LLM reviews the specific paths flagged by Phase 1's Bottom Discovery for the current document. Selected discovery paths are hydrated into leaf chunks (with `job_result_id` dynamically extracted from the chunks) and merged directly into the BFS document tree. + - *Reparenting*: The `DocTreeNode.merge()` process reparents these discovered leaf chunks into the closest matching navigated child node. + - *Orphan Leaves*: Discovered chunks whose paths are not explicitly covered by the BFS `outline_items` are rendered cleanly as `[Leaf]` items (orphans) beneath their appropriate parent, ensuring no relevant data is lost even if the BFS did not explicitly drill into that path. **Phase 3: Verdict & Revision** The combined document tree (BFS Navigation + Discovery) is rendered as unified evidence. The tree naturally displays structural context (outlines) alongside hydrated chunk rows (for selected leaf paths). The LLM attempts to answer the user's query: diff --git a/apps/api/alembic/versions/f6a7b8c9d0e1_add_demo_materializations.py b/apps/api/alembic/versions/f6a7b8c9d0e1_add_demo_materializations.py new file mode 100644 index 000000000..f6277c983 --- /dev/null +++ b/apps/api/alembic/versions/f6a7b8c9d0e1_add_demo_materializations.py @@ -0,0 +1,61 @@ +"""add demo materializations + +Revision ID: f6a7b8c9d0e1 +Revises: e5f6a7b8c9d0 +Create Date: 2026-05-12 08:25:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "f6a7b8c9d0e1" +down_revision: Union[str, Sequence[str], None] = "e5f6a7b8c9d0" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "demo_materializations", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("user_id", sa.Text(), nullable=False), + sa.Column("namespace", sa.String(length=255), nullable=False), + sa.Column("demo_source_id", sa.String(length=128), nullable=False), + sa.Column("document_id", sa.String(length=36), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["document_id"], + ["documents.document_id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="RESTRICT"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "user_id", + "namespace", + "demo_source_id", + name="uq_demo_materializations_scope_source", + ), + ) + op.create_index( + "idx_demo_materializations_document", + "demo_materializations", + ["document_id"], + unique=False, + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index( + "idx_demo_materializations_document", + table_name="demo_materializations", + ) + op.drop_table("demo_materializations") diff --git a/apps/api/app/api/v1/api_v1.py b/apps/api/app/api/v1/api_v1.py index 023fb7621..dab7d14cb 100644 --- a/apps/api/app/api/v1/api_v1.py +++ b/apps/api/app/api/v1/api_v1.py @@ -4,6 +4,7 @@ from app.api.v1.routes import ( api_key, + demo, documents, guest, jobs, @@ -36,6 +37,9 @@ # Unified Jobs routes api_router.include_router(jobs.router, prefix="/jobs", tags=["Jobs"]) +# Demo documents +api_router.include_router(demo.router, prefix="/demo", tags=["Demo Documents"]) + # Retrieval api_router.include_router(retrieval.router, prefix="/retrieval", tags=["Retrieval"]) diff --git a/apps/api/app/api/v1/routes/demo.py b/apps/api/app/api/v1/routes/demo.py new file mode 100644 index 000000000..330a5167c --- /dev/null +++ b/apps/api/app/api/v1/routes/demo.py @@ -0,0 +1,158 @@ +"""Notebook demo document catalog routes.""" + +from __future__ import annotations + +from typing import Any + +from app.services.demo_document_service import DemoDocumentService +from app.services.rate_limit.dependencies import CurrentUser, with_current_user +from fastapi import APIRouter, Depends, Query +from fastapi.responses import FileResponse +from pydantic import BaseModel, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from shared.core.database import get_db +from shared.core.exceptions.domain_exceptions import NotFoundException + +router = APIRouter(tags=["Demo Documents"]) + +_demo_document_service = DemoDocumentService() + + +class DemoMaterializeRequest(BaseModel): + """Request to copy selected canonical demo sources into a namespace.""" + + namespace: str | None = Field(None, description="Target retrieval namespace") + demo_source_ids: list[str] = Field( + default_factory=list, + min_length=1, + description="Canonical demo source IDs to materialize", + ) + + +@router.get("/catalog") +async def get_demo_catalog() -> dict[str, Any]: + """Return API-owned canonical demo source metadata and curated Q/A.""" + return _demo_document_service.get_catalog() + + +@router.get("/sources/{demo_source_id}/chunks") +async def list_demo_source_chunks( + demo_source_id: str, + page: int = Query(1, ge=1, description="Page number"), + page_size: int = Query(50, ge=1, le=200, description="Items per page"), +) -> dict[str, Any]: + """Return paginated canonical chunks for a demo source.""" + response = _demo_document_service.list_chunks( + demo_source_id=demo_source_id, + page=page, + page_size=page_size, + ) + if response is None: + raise _demo_source_not_found(demo_source_id) + return response + + +@router.get("/sources/{demo_source_id}/chunks/{demo_chunk_id}") +async def get_demo_source_chunk( + demo_source_id: str, + demo_chunk_id: str, +) -> dict[str, Any]: + """Return one canonical demo chunk for citation focusing.""" + response = _demo_document_service.get_chunk( + demo_source_id=demo_source_id, + demo_chunk_id=demo_chunk_id, + ) + if response is None: + raise NotFoundException( + resource="Demo document chunk", + resource_id=demo_chunk_id, + internal_message="Demo document chunk not found", + ) + return response + + +@router.get("/sources/{demo_source_id}/original") +async def get_demo_source_original(demo_source_id: str) -> FileResponse: + """Return the canonical original file for preview.""" + file_path = _demo_document_service.get_original_file_path( + demo_source_id=demo_source_id, + ) + if file_path is None: + raise _demo_source_not_found(demo_source_id) + + return FileResponse( + path=file_path, + media_type="application/pdf", + filename=file_path.name, + content_disposition_type="inline", + ) + + +@router.get("/sources/{demo_source_id}/assets/{asset_path:path}") +async def get_demo_source_asset( + demo_source_id: str, + asset_path: str, +) -> FileResponse: + """Return a canonical parsed media or table asset for preview.""" + file_path = _demo_document_service.get_asset_file_path( + demo_source_id=demo_source_id, + asset_path=asset_path, + ) + if file_path is None: + raise _demo_source_not_found(demo_source_id) + + return FileResponse( + path=file_path, + filename=file_path.name, + content_disposition_type="inline", + ) + + +@router.post("/materializations") +async def materialize_demo_sources( + payload: DemoMaterializeRequest, + current_user: CurrentUser = Depends(with_current_user), + db: AsyncSession = Depends(get_db), +) -> dict[str, Any]: + """Copy canonical demo sources into the authenticated user's namespace.""" + namespace = (payload.namespace or "default").strip() or "default" + try: + materialized_sources = await _demo_document_service.materialize_sources( + db, + user_id=current_user.user_id, + namespace=namespace, + demo_source_ids=payload.demo_source_ids, + ) + except KeyError as error: + raise _demo_source_not_found(str(error.args[0])) from error + + return { + "namespace": namespace, + "sources": [ + { + "demo_source_id": source.demo_source_id, + "document_id": source.document_id, + "status": source.status, + "title": source.title, + "mime_type": source.mime_type, + "size_bytes": source.size_bytes, + "chunk_count": source.chunk_count, + "original_file": { + "url": f"/api/v1/demo/sources/{source.demo_source_id}/original", + "mime_type": source.mime_type, + "size_bytes": source.size_bytes, + "can_download": False, + }, + } + for source in materialized_sources + ], + } + + +def _demo_source_not_found(demo_source_id: str) -> NotFoundException: + return NotFoundException( + resource="Demo document source", + resource_id=demo_source_id, + internal_message="Demo document source not found", + ) diff --git a/apps/api/app/api/v1/routes/jobs.py b/apps/api/app/api/v1/routes/jobs.py index 31bc5268b..d7c6972a2 100644 --- a/apps/api/app/api/v1/routes/jobs.py +++ b/apps/api/app/api/v1/routes/jobs.py @@ -6,7 +6,7 @@ import os import uuid -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any, Dict, Literal, Optional, cast from urllib.parse import urlparse @@ -53,6 +53,7 @@ StandardErrorObject, ) from shared.services.storage.file_upload_service import FileUploadService +from shared.utils.utc_now import utc_now_naive from shared.utils.url_security import ( validate_http_url_and_resolve_ip_async, ) @@ -268,6 +269,15 @@ def ensure_utc(dt: Optional[datetime]) -> Optional[datetime]: return dt.replace(tzinfo=timezone.utc) +def normalize_naive_utc_filter_datetime(dt: Optional[datetime]) -> Optional[datetime]: + """Convert a query datetime into the naive UTC form used by database columns.""" + if dt is None: + return None + if dt.tzinfo is None or dt.utcoffset() is None: + return dt + return dt.astimezone(timezone.utc).replace(tzinfo=None) + + def require_utc(dt: Optional[datetime], *, field_name: str) -> datetime: """Normalize a required datetime to UTC.""" normalized_dt = ensure_utc(dt) @@ -678,13 +688,18 @@ async def list_jobs( user_message="recent_days only supports 1, 7, or 30", violations=[{"field": "recent_days", "description": "Invalid value"}], ) - created_after = None + created_after: Optional[datetime] = None if recent_days: - from datetime import datetime, timedelta + created_after = utc_now_naive() - timedelta(days=recent_days) - created_after = datetime.now() - timedelta(days=recent_days) + normalized_start_time = normalize_naive_utc_filter_datetime(start_time) + normalized_end_time = normalize_naive_utc_filter_datetime(end_time) - if start_time and end_time and start_time > end_time: + if ( + normalized_start_time + and normalized_end_time + and normalized_start_time > normalized_end_time + ): raise ValidationException( user_message="start_time cannot be later than end_time", violations=[ @@ -692,9 +707,9 @@ async def list_jobs( ], ) # start_time / end_time take priority over recent_days. - if start_time: - created_after = start_time - created_before = end_time + if normalized_start_time: + created_after = normalized_start_time + created_before = normalized_end_time # Count matching rows. total_count = await job_repo.count_jobs_by_user( @@ -752,10 +767,8 @@ async def list_jobs( # Compute result_url_expires_at when a download URL was issued. if result_url: - from datetime import datetime, timedelta - expires_in = int(result_url_info.get("expires_in", 3600)) - result_url_expires_at = datetime.now() + timedelta( + result_url_expires_at = utc_now_naive() + timedelta( seconds=expires_in ) diff --git a/apps/api/app/api/v1/routes/qstash_callbacks.py b/apps/api/app/api/v1/routes/qstash_callbacks.py index cfa2b133d..84cae94f2 100644 --- a/apps/api/app/api/v1/routes/qstash_callbacks.py +++ b/apps/api/app/api/v1/routes/qstash_callbacks.py @@ -128,25 +128,67 @@ def _build_callback_log_idempotency_key( return event_id +def _get_response_status_code(value: Any) -> Optional[int]: + """Return the destination response status reported by QStash.""" + if value is None: + return None + + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _is_success_response_status(status_code: Optional[int]) -> bool: + """Return whether a destination response status is successful.""" + return status_code is not None and 200 <= status_code < 300 + + +def _get_callback_event_status(data: Dict[str, Any]) -> str: + """Map a normal QStash callback to the current webhook event status.""" + response_status = _get_response_status_code(data.get("status")) + if _is_success_response_status(response_status): + return WebhookEventStatus.DELIVERED + + return WebhookEventStatus.DELIVERING + + +def _resolve_event_status(current_status: str, callback_status: str) -> str: + """Apply callback status without downgrading terminal delivery state.""" + if current_status in ( + WebhookEventStatus.DELIVERED, + WebhookEventStatus.FAILED, + WebhookEventStatus.CANCELED, + ): + return current_status + + return callback_status + + def _process_qstash_callback( data: Dict[str, Any], event_id: str, - terminal_status: str, + callback_status: str, log_label: str, ) -> Response: """Shared logic for both success and failure QStash callbacks. Fetches the WebhookEvent, updates its status, and writes a WebhookLog entry. """ - response_status = data.get("status") + response_status_code = _get_response_status_code(data.get("status")) response_body = data.get("body", "") qstash_message_id = data.get("sourceMessageId") retried = data.get("retried", 0) - error_message = ( - data.get("error", response_body) - if terminal_status == WebhookEventStatus.FAILED - else None + is_failed_delivery_attempt = ( + callback_status == WebhookEventStatus.FAILED + or ( + callback_status == WebhookEventStatus.DELIVERING + and not _is_success_response_status(response_status_code) + ) ) + error_message = None + if is_failed_delivery_attempt: + error_message = data.get("error") or response_body with get_sync_db_context() as db: event = db.execute( @@ -158,21 +200,23 @@ def _process_qstash_callback( return Response(status_code=200, content="OK (event not found)") now = datetime.now(timezone.utc).replace(tzinfo=None) - event.status = terminal_status - event.attempts = retried + 1 + event_status = _resolve_event_status(event.status, callback_status) + attempt_number = retried + 1 + event.status = event_status + event.attempts = max(event.attempts, attempt_number) event.updated_at = now log = WebhookLog( job_id=event.job_id, event_id=event.id, webhook_url=event.target_url, - attempt_number=retried + 1, + attempt_number=attempt_number, request_payload=event.payload, signature="", idempotency_key=_build_callback_log_idempotency_key( qstash_message_id, event.id ), - response_status_code=int(response_status) if response_status else None, + response_status_code=response_status_code, response_body=response_body[:4096] if response_body else None, error_message=str(error_message)[:4096] if error_message else None, duration_ms=0, @@ -210,8 +254,9 @@ async def handle_qstash_callback(request: Request) -> Response: f"retried={retried}, qstash_message_id={data.get('sourceMessageId')}" ) + event_status = _get_callback_event_status(data) return _process_qstash_callback( - data, event_id, WebhookEventStatus.DELIVERED, "callback" + data, event_id, event_status, "callback" ) diff --git a/apps/api/app/core/dependencies.py b/apps/api/app/core/dependencies.py index de7b3a636..bf56d39c8 100644 --- a/apps/api/app/core/dependencies.py +++ b/apps/api/app/core/dependencies.py @@ -7,6 +7,7 @@ from fastapi import Depends, Header, Request from jwt import PyJWKClient from loguru import logger +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from shared.core.config import settings @@ -14,6 +15,7 @@ from shared.core.exceptions.domain_exceptions import ( AuthException, ) +from shared.models.database.user import User from shared.utils.api_keys import is_api_key_token # Standard JWKS endpoint path (fixed, following OpenID Connect convention) @@ -106,6 +108,23 @@ def decode_jwt_token(token: str) -> str: raise AuthException(user_message="Invalid token") +async def _ensure_authenticated_user_exists( + db: AsyncSession, + user_id: str, +) -> None: + result = await db.execute(select(User.id).where(User.id == user_id).limit(1)) + if result.scalar_one_or_none() is not None: + return + + raise AuthException( + user_message="Invalid authentication credentials", + internal_message=( + "Authenticated user id is not present in the user table: " + f"user_id={user_id}" + ), + ) + + async def get_current_user_id( request: Request, authorization: str | None = Header( @@ -134,4 +153,6 @@ async def get_current_user_id( raise AuthException(user_message="Invalid API Key") # Mode 2: JWT verification (for Dashboard/Internal) - return decode_jwt_token(token) + user_id = decode_jwt_token(token) + await _ensure_authenticated_user_exists(db, user_id) + return user_id diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/chunks.json b/apps/api/app/data/demo_documents/tsla-q4-2025/chunks.json new file mode 100644 index 000000000..7e7147254 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/chunks.json @@ -0,0 +1,3313 @@ +{ + "chunks": [ + { + "chunk_id": "15bcc860-b8d0-50c6-a627-66dbae67acd4", + "type": "table", + "content": "
Profitability$4.4B GAAP operating income in 2025; $1.4B in Q42025 marked a critical year for Tesla as we further expanded our mission and continued our transition from a hardware-centric business to a physical AI company. We laid the foundation for the future of Tesla as we further advanced FSD (Supervised) $^{4}$ , launched our Robotaxi service, began installing production lines for Cybercab and fine-tuned our production-primed Optimus design while expanding our AI training infrastructure.
$3.8B GAAP net income in 2025; $0.8B in Q4
$5.9B non-GAAP net income $^{1}$ in 2025; $1.8B in Q4
CashOperating cash flow of $14.7B in 2025; $3.8B in Q4Our approach to autonomous vehicles and humanoid robots mirrors the way we approached electric vehicles and energy storage – at the system level where we identify the limiting factor and develop bespoke and scalable solutions (batteries, power electronics, inverters, software, AI silicon, etc.) to optimize for cost, functionality, efficiency and safety. Our vertical integration has enabled us to achieve economies of scale in a profitable manner, quickly troubleshoot bottlenecks in production and iteratively optimize our technologies more rapidly than others.
Free cash flow $^{2}$ of $6.2B in 2025; $1.4B in Q4In 2025, we completed the refresh of our vehicle lineup with the launch of the new Model Y, including additional variants. We believe that maintaining an optimized and efficient product portfolio, with a continued focus on high-value features such as long range, best-in-class software and autonomy, is the correct strategy to win the autos market of the future. Similarly, we continued to evolve our energy offerings for commercial, utility and retail customers, as we position ourselves as a supplier of choice for clean, affordable and rapidly deployable energy capacity ahead of expected sustained demand growth for electricity.
$7.5B increase in our cash and investments $^{3}$ in 2025 to $44.1B
OperationsBegan removing safety monitor from our Robotaxis in Austin in JanuaryIn 2026, we will further invest in the infrastructure needed to support clean energy and transport and autonomous robots, including the ramp of six new production lines across vehicle, robots, energy storage and battery manufacturing, while further leveraging our existing factory, charging and service center footprint to support future growth.
Record Q4 & FY'25 energy storage deployments
Record vehicle deliveries in APAC
", + "path": "tables/table-0 Tesla 2025 Results.html", + "metadata": { + "length": 2783, + "summary": "table-1\nTesla reported strong 2025 financials with $4.4B operating income and expanded AI initiatives including Robotaxi and Optimus.", + "page_nums": [ + 11 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-0 Tesla 2025 Results.html", + "keywords": [ + "Profitability", + "Cash", + "Operations" + ], + "tokens": [] + } + }, + { + "chunk_id": "68a6be7d-c587-5c73-abf2-56f4686e28e6", + "type": "text", + "content": "[tables/table-0 Tesla 2025 Results.html]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->HIGHLIGHTS", + "metadata": { + "length": 83, + "summary": "", + "page_nums": [ + 1, + 3, + 11 + ], + "document_top_summary": "This document includes:", + "tokens": [], + "keywords": [], + "connect_to": [ + { + "target": "15bcc860-b8d0-50c6-a627-66dbae67acd4", + "relation": "embeds", + "ref": "[tables/table-0 Tesla 2025 Results.html]", + "position": { + "start": 0, + "end": 40 + } + } + ] + } + }, + { + "chunk_id": "481804a6-0fb5-52fc-bc47-ef5728621f6b", + "type": "table", + "content": "
($ in millions, except percentages and per share data)Q4-2024Q1-2025Q2-2025Q3-2025Q4-2025YoY
Total automotive revenues19,79813,96716,66121,20517,693-11%
Energy generation and storage revenue3,0612,7302,7893,4153,83725%
Services and other revenue2,8482,6383,0463,4753,37118%
Total revenues25,70719,33522,49628,09524,901-3%
Total gross profit4,1793,1533,8785,0545,00920%
Total GAAP gross margin16.3%16.3%17.2%18.0%20.1%386 bp
Operating expenses2,5962,7542,9553,4303,60039%
Income from operations1,5833999231,6241,409-11%
Operating margin6.2%2.1%4.1%5.8%5.7%-50 bp
Adjusted EBITDA (1) (2)4,3332,8143,4014,2274,154-4%
Adjusted EBITDA margin (1) (2)16.9%14.6%15.1%15.0%16.7%-17 bp
Net income attributable to common stockholders (GAAP) (1)2,1284091,1721,373840-61%
Net income attributable to common stockholders (non-GAAP) (1) (3)2,1079341,3931,7701,761-16%
EPS attributable to common stockholders, diluted (GAAP) (1)0.600.120.330.390.24-60%
EPS attributable to common stockholders, diluted (non-GAAP) (1) (3)0.600.270.400.500.50-17%
Net cash provided by operating activities4,8142,1562,5406,2383,813-21%
Capital expenditures (4)(2,780)(1,492)(2,394)(2,248)(2,393)-14%
Free cash flow (4)2,0346641463,9901,420-30%
Cash, cash equivalents and investments36,56336,99636,78241,64744,05921%
", + "path": "tables/table-1 Q4 2025 Financials.html", + "metadata": { + "length": 2894, + "summary": "table-2\nTable shows Tesla's quarterly financials through Q4 2025, including revenues, gross profit, operating income, and free cash flow in millions of dollars.", + "page_nums": [ + 11 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-1 Q4 2025 Financials.html", + "keywords": [ + "revenue", + "profit", + "cash flow" + ], + "tokens": [] + } + }, + { + "chunk_id": "31077385-2188-5cc5-bbfb-85e07e94c5c8", + "type": "table", + "content": "
($ in millions, except percentages and per share data)20212022202320242025YoY
Total automotive revenues47,23271,46282,41977,07069,526-10%
Energy generation and storage revenue2,7893,9096,03510,08612,77127%
Services and other revenue3,8026,0918,31910,53412,53019%
Total revenues53,82381,46296,77397,69094,827-3%
Total gross profit13,60620,85317,66017,45017,094-2%
Total GAAP gross margin25.3%25.6%18.2%17.9%18.0%16 bp
Operating expenses7,0837,1978,76910,37412,73923%
Income from operations6,52313,6568,8917,0764,355-38%
Operating margin12.1%16.8%9.2%7.2%4.6%-265 bp
Adjusted EBITDA (1)11,72219,39016,63116,05614,596-9%
Adjusted EBITDA margin (1)21.8%23.8%17.2%16.4%15.4%-104 bp
Net income attributable to common stockholders (GAAP)5,51912,55614,9977,0913,794-46%
Net income attributable to common stockholders (non-GAAP) (2)7,71914,27610,8827,9605,858-26%
EPS attributable to common stockholders, diluted (GAAP)1.633.624.302.041.08-47%
EPS attributable to common stockholders, diluted (non-GAAP) (2)2.284.123.122.291.66-28%
Net cash provided by operating activities11,49714,72413,25614,92314,747-1%
Capital expenditures (3)(6,514)(7,163)(8,899)(11,342)(8,527)-25%
Free cash flow (3)4,9837,5614,3573,5816,22074%
Cash, cash equivalents and investments17,70722,18529,09436,56344,05921%
", + "path": "tables/table-2 Financial Data 2021-25.html", + "metadata": { + "length": 2897, + "summary": "table-3\nTable shows financial metrics from 2021 to 2025, including revenues, gross profit, operating income, and free cash flow in millions of dollars.", + "page_nums": [ + 11 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-2 Financial Data 2021-25.html", + "keywords": [ + "revenue", + "profit", + "cash flow" + ], + "tokens": [] + } + }, + { + "chunk_id": "8ef4ea32-6ed5-5aec-a413-6af0d752355b", + "type": "table", + "content": "
Q4-2024Q1-2025Q2-2025Q3-2025Q4-2025YoY
Model 3/Y production436,718345,454396,835435,826422,652-3%
Other models production22,72717,16113,40911,62411,706-48%
Total production459,445362,615410,244447,450434,358-5%
Model 3/Y deliveries471,930323,800373,728481,166406,585-14%
Other models deliveries23,64012,88110,39415,93311,642-51%
Total deliveries495,570336,681384,122497,099418,227-16%
of which subject to operating lease accounting26,96213,7216,67010,23010,996-59%
Cumulative $deliveries^{(1)}$ (all-time; mil)7.37.68.08.58.922%
Active FSD $Subscriptions^{(2)}$ (mil)0.80.80.91.01.138%
Total end of quarter operating lease (new vehicle) $count^{(3)}$ 180,523179,930172,882167,163163,075-10%
Global vehicle inventory (days of supply) $^{(4)}$ 122224101525%
Storage deployed (GWh)11.010.49.612.514.229%
Tesla locations1,3591,3901,4541,4981,55314%
Supercharger stations6,9757,1317,3777,7538,18217%
Supercharger connectors65,49567,31670,22873,81777,68219%
", + "path": "tables/table-3 Tesla Q4-2025 Data.html", + "metadata": { + "length": 2288, + "summary": "table-4\nTable shows Tesla's quarterly production, deliveries, and inventory from Q4 2024 to Q4 2025. Total deliveries dropped 16% YoY in Q4 2025.", + "page_nums": [ + 11 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-3 Tesla Q4-2025 Data.html", + "keywords": [ + "production", + "deliveries", + "inventory" + ], + "tokens": [] + } + }, + { + "chunk_id": "4c82227e-3ba3-5bb2-9d00-4397c1dc38f6", + "type": "table", + "content": "
20212022202320242025YoY
Model 3/Y production906,0321,298,4341,775,1591,679,3381,600,767-5%
Other models production24,39071,17770,82694,10553,900-43%
Total production930,4221,369,6111,845,9851,773,4431,654,667-7%
Model 3/Y deliveries911,2421,247,1461,739,7071,704,0931,585,279-7%
Other models deliveries24,98066,70568,87485,13350,850-40%
Total deliveries936,2221,313,8511,808,5811,789,2261,636,129-9%
of which subject to operating lease accounting60,91247,58272,22660,00341,617-31%
Cumulative $deliveries^{(1)}$ (all-time; mil)2.33.75.57.38.922%
Active FSD $Subscriptions^{(2)}$ (mil)0.40.50.60.81.138%
Total end of year operating lease (new vehicle) count120,342140,667176,564180,523163,075-10%
Global vehicle inventory (days of supply) $^{(3)}$ 61616131515%
Storage deployed (GWh)4.06.514.731.446.749%
Tesla locations6449631,2081,3591,55314%
Supercharger stations3,4764,6785,9526,9758,18217%
Supercharger connectors31,49842,41954,89265,49577,68219%
", + "path": "tables/table-4 Tesla 2021-2025 Data.html", + "metadata": { + "length": 2285, + "summary": "table-5\nTable shows Tesla's production, deliveries, and infrastructure metrics from 2021 to 2025. Total production and deliveries peaked in 2023 then declined by 2025.", + "page_nums": [ + 11 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-4 Tesla 2021-2025 Data.html", + "keywords": [ + "production", + "deliveries", + "growth" + ], + "tokens": [] + } + }, + { + "chunk_id": "60109008-6261-51e6-b202-093d904eb881", + "type": "text", + "content": "FINANCIAL SUMMARY\n(Unaudited)\n\n[tables/table-1 Q4 2025 Financials.html]\n\n(1) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.\n(2) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(3) Beginning in Q1'25, Net income attributable to common stockholders (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(4) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted.\nFINANCIAL SUMMARY\n(Unaudited)\n\n[tables/table-2 Financial Data 2021-25.html]\n\n(1) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(2) Beginning in Q1'25, Net income attributable to common stockholders (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(3) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted.\nOPERATIONAL SUMMARY\n(Unaudited)\n\n[tables/table-3 Tesla Q4-2025 Data.html]\n\nOPERATIONAL SUMMARY\n(Unaudited)\n\n[tables/table-4 Tesla 2021-2025 Data.html]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY", + "metadata": { + "length": 1517, + "summary": "The document presents unaudited financial and operational summaries for a company, covering quarterly data from Q4 2024 through Q4 2025 and annual data from 2021 to 2025. Key notes indicate significant accounting changes effective Q1 2025: Adjusted EBITDA and Net income attributable to common stockholders are now presented net of digital assets gains and losses, with all prior periods adjusted accordingly. Additionally, Capital expenditures now include purchases of energy generation and storage systems, requiring restatement of previous periods. The content references multiple tables detailing these metrics but does not display the specific numerical values.", + "page_nums": [ + 8, + 11 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "FINANCIAL", + "SUMMARY", + "Unaudited", + "As", + "result", + "adoption", + "crypto", + "assets", + "standard", + "previously", + "reported", + "quarterly", + "periods", + "2024", + "recast", + "Beginning", + "Q1", + "25", + "Adjusted", + "EBITDA", + "GAAP", + "presented", + "net", + "digital", + "gains", + "losses", + "prior", + "adjusted", + "Net", + "income", + "attributable", + "common", + "stockholders", + "Capital", + "expenditures", + "inclusive", + "purchases", + "energy", + "generation", + "storage", + "systems", + "OPERATIONAL" + ], + "keywords": [ + "financials", + "crypto assets", + "adjusted metrics" + ], + "connect_to": [ + { + "target": "481804a6-0fb5-52fc-bc47-ef5728621f6b", + "relation": "embeds", + "ref": "[tables/table-1 Q4 2025 Financials.html]", + "position": { + "start": 31, + "end": 71 + } + }, + { + "target": "31077385-2188-5cc5-bbfb-85e07e94c5c8", + "relation": "embeds", + "ref": "[tables/table-2 Financial Data 2021-25.html]", + "position": { + "start": 724, + "end": 768 + } + }, + { + "target": "8ef4ea32-6ed5-5aec-a413-6af0d752355b", + "relation": "embeds", + "ref": "[tables/table-3 Tesla Q4-2025 Data.html]", + "position": { + "start": 1288, + "end": 1328 + } + }, + { + "target": "4c82227e-3ba3-5bb2-9d00-4397c1dc38f6", + "relation": "embeds", + "ref": "[tables/table-4 Tesla 2021-2025 Data.html]", + "position": { + "start": 1363, + "end": 1405 + } + } + ] + } + }, + { + "chunk_id": "60339310-5480-5ae2-8791-e6017bafb730", + "type": "text", + "content": "While automotive sales declined sequentially, gross margin (even when excluding the impact of regulatory credits) improved. The APAC region continued to show strength across multiple markets and set a record for deliveries in the quarter. We continued the rollout of Model Y variants across markets in Q4, including the standard and performance versions.\nPreparations continue in North America for the production ramps of Tesla Semi and Cybercab, both commencing 1H26, and production of the next-generation Roadster.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->Automotive", + "metadata": { + "length": 516, + "summary": "", + "page_nums": [ + 8 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "While", + "automotive", + "sales", + "declined", + "sequentially", + "gross", + "margin", + "excluding", + "impact", + "regulatory", + "credits", + "improved", + "The", + "APAC", + "region", + "continued", + "show", + "strength", + "multiple", + "markets", + "set", + "record", + "deliveries", + "quarter", + "We", + "rollout", + "Model", + "variants", + "Q4", + "including", + "standard", + "performance", + "versions", + "Preparations", + "continue", + "North", + "America", + "production", + "ramps", + "Tesla", + "Semi", + "Cybercab", + "commencing", + "1H26", + "generation", + "Roadster" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "eb18e33e-b093-532e-b4df-bd05d63b0294", + "type": "text", + "content": "We achieved our highest quarterly energy storage deployments, driven by record Megapack deployments. Total gross profit rose, both sequentially and year-over-year, to a record \\$1.1 billion, marking the fifth consecutive record quarter. We plan to begin Megapack 3 and Megablock production at Megafactory Houston in 2026. In 2025, our global Powerwall network supported more than 89,000 Virtual Power Plant events across over 1 million installed units, allowing homeowners to save over \\$1 billion in electricity bills as Virtual Power Plant participation continues to scale rapidly.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->Energy generation and storage", + "metadata": { + "length": 583, + "summary": "", + "page_nums": [ + 8 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "achieved", + "highest", + "quarterly", + "energy", + "storage", + "deployments", + "driven", + "record", + "Megapack", + "Total", + "gross", + "profit", + "rose", + "sequentially", + "year", + "1.1", + "billion", + "marking", + "consecutive", + "quarter", + "plan", + "begin", + "Megablock", + "production", + "Megafactory", + "Houston", + "2026", + "In", + "2025", + "global", + "Powerwall", + "network", + "supported", + "89", + "000", + "Virtual", + "Power", + "Plant", + "events", + "million", + "installed", + "units", + "allowing", + "homeowners", + "save", + "electricity", + "bills", + "participation", + "continues", + "scale", + "rapidly" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "cdae270e-2dc4-5edc-9876-26a9454fbfff", + "type": "table", + "content": "
RegionProductCapacityStatus
Automotive
CaliforniaModel 3 / Model Y>550,000Production
Model S / Model X100,000Production
ShanghaiModel 3 / Model Y>950,000Production
BerlinModel Y>375,000Production
TexasModel Y>250,000Production
Cybertruck>125,000Production
Cybercab-Tooling
NevadaTesla Semi-Tooling
TBDRoadster-Design development
Energy Generation and Storage
CaliforniaMegapack40 GWhProduction
NevadaPowerwall>6 GWhProduction
ShanghaiMegapack40 GWhProduction
TexasMegapack-Construction
Robotics
CaliforniaOptimus-Construction
", + "path": "tables/table-5 Tesla Production.html", + "metadata": { + "length": 1306, + "summary": "table-6\nTesla operates global facilities in California, Shanghai, Berlin, Texas, and Nevada for Model 3/Y, S/X, Cybertruck, Semi, Megapack, Powerwall, and Optimus.", + "page_nums": [ + 8 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-5 Tesla Production.html", + "keywords": [ + "Tesla", + "Manufacturing", + "Capacity" + ], + "tokens": [] + } + }, + { + "chunk_id": "60518074-bcbe-5b48-89a9-dfeb5d1b531b", + "type": "text", + "content": "We made further progress on the Optimus program in 2025. In Q1 of this year, we plan to unveil the Gen 3 version of Optimus, which will include major upgrades from version 2.5, including our latest hand design. The Gen 3 is our first design meant for mass production. Preparations are underway for the first production line, including supply chain readiness, with start of production planned before the end of 2026 and eventual planned capacity of 1 million robots per year.\nInstalled Annual Manufacturing Capacity\n\n[tables/table-5 Tesla Production.html]\n\nInstalled capacity ≠ current production rate and there may be limitations discovered as production rates approach capacity. Production rates depend on a variety of factors, including equipment uptime, component supply, downtime related to factory upgrades, regulatory considerations and other factors. Construction includes factory and infrastructure buildout as well as tool installation.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->Robotics", + "metadata": { + "length": 959, + "summary": "", + "page_nums": [ + 8, + 9 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "made", + "progress", + "Optimus", + "program", + "2025", + "In", + "Q1", + "year", + "plan", + "unveil", + "Gen", + "version", + "include", + "major", + "upgrades", + "2.5", + "including", + "latest", + "hand", + "design", + "The", + "meant", + "mass", + "production", + "Preparations", + "underway", + "line", + "supply", + "chain", + "readiness", + "start", + "planned", + "end", + "2026", + "eventual", + "capacity", + "million", + "robots", + "Installed", + "Annual", + "Manufacturing", + "Capacity", + "current", + "rate", + "limitations", + "discovered", + "rates", + "approach", + "Production", + "depend", + "variety", + "factors", + "equipment", + "uptime", + "component", + "downtime", + "related", + "factory", + "regulatory", + "considerations", + "Construction", + "includes", + "infrastructure", + "buildout", + "tool", + "installation" + ], + "keywords": [], + "connect_to": [ + { + "target": "cdae270e-2dc4-5edc-9876-26a9454fbfff", + "relation": "embeds", + "ref": "[tables/table-5 Tesla Production.html]", + "position": { + "start": 516, + "end": 554 + } + } + ] + } + }, + { + "chunk_id": "c49883a3-5834-5537-be53-35e58da70bf7", + "type": "text", + "content": "We are currently building Cortex 2 at Gigafactory Texas to further increase our AI training compute capacity. In the first half of 2026, we plan to more than double the size of onsite compute in Texas (in terms of H100 equivalents). We aim to maximize capital efficiency by scaling training compute judiciously, including when the training backlog gets too long or in anticipation of greater demand from our engineers to support our AI-related offerings.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->AI Training Compute", + "metadata": { + "length": 454, + "summary": "", + "page_nums": [ + 9 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "building", + "Cortex", + "Gigafactory", + "Texas", + "increase", + "AI", + "training", + "compute", + "capacity", + "In", + "half", + "2026", + "plan", + "double", + "size", + "onsite", + "terms", + "H100", + "equivalents", + "aim", + "maximize", + "capital", + "efficiency", + "scaling", + "judiciously", + "including", + "backlog", + "long", + "anticipation", + "greater", + "demand", + "engineers", + "support", + "related", + "offerings" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "fcbc3b02-dec6-5e91-849e-fccd3f22320e", + "type": "text", + "content": "Our lithium refinery commenced pilot production and is the first spodumene to lithium hydroxide refinery in North America, leveraging a simpler, cheaper and more environmentally friendly process. This refinery enables us to domestically produce critical minerals in support of energy storage, battery manufacturing and ultimately for EV growth.\nWe have begun to produce battery packs for certain Model Ys with our 4680 cells, unlocking an additional vector of supply to help navigate increasingly complex supply chain challenges caused by trade barriers and tariff risks. We now produce dry-electrode for 4680 cells with both anode and cathode made in Austin. We expect both domestic cathode material in Texas and LFP lines in Nevada to begin production in 2026.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->Battery", + "metadata": { + "length": 762, + "summary": "", + "page_nums": [ + 9 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Our", + "lithium", + "refinery", + "commenced", + "pilot", + "production", + "spodumene", + "hydroxide", + "North", + "America", + "leveraging", + "simpler", + "cheaper", + "environmentally", + "friendly", + "process", + "This", + "enables", + "domestically", + "produce", + "critical", + "minerals", + "support", + "energy", + "storage", + "battery", + "manufacturing", + "ultimately", + "EV", + "growth", + "We", + "begun", + "packs", + "Model", + "Ys", + "4680", + "cells", + "unlocking", + "additional", + "vector", + "supply", + "navigate", + "increasingly", + "complex", + "chain", + "challenges", + "caused", + "trade", + "barriers", + "tariff", + "risks", + "dry", + "electrode", + "anode", + "cathode", + "made", + "Austin", + "expect", + "domestic", + "material", + "Texas", + "LFP", + "lines", + "Nevada", + "begin", + "2026" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "0428af2c-bf5d-54a6-a142-09ec146d027e", + "type": "table", + "content": "
RegionProductCapacityStatus
AI Training Compute
TexasCortex 1>100k H100eProduction
Cortex 2-Construction
Battery Manufacturing
NevadaLFP7 GWhEarly Ramp
Texas468040 GWhProduction
Cathode Materials10 GWhEarly Ramp
Lithium Refining30 GWhEarly Ramp
", + "path": "tables/table-6 Facility Status.html", + "metadata": { + "length": 629, + "summary": "table-7\nTable lists AI training and battery manufacturing facilities. Texas hosts Cortex 1 (production) and Cortex 2 (construction). Nevada and Texas have LFP, 4680, cathode materials, and lithium refining plants in various stages.", + "page_nums": [ + 9 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-6 Facility Status.html", + "keywords": [ + "AI Training", + "Battery Mfg", + "Capacity" + ], + "tokens": [] + } + }, + { + "chunk_id": "92dc6e40-720d-55b8-a2e2-c78bd4f59dba", + "type": "image", + "content": "\n## Other Supporting Infrastructure Tesla AI Training Capacity Ramp (H100 equivalent GPUs)0\n[images/image-1-Capacity Growth Projection.jpg]\n", + "path": "images/image-1-Capacity Growth Projection.jpg", + "metadata": { + "length": 124, + "summary": "image-1\nThe chart illustrates a steady increase in existing capacity from mid-2021 through late 2025, characterized by gradual step-wise growth. A significant surge is projected for the future planned capacity starting in early 2026, reaching levels well above the current trajectory.", + "page_nums": [ + 9 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-1-Capacity Growth Projection.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "34777c71-d716-5c39-b18a-bec0d0b64706", + "type": "text", + "content": "We continue to efficiently utilize our existing physical footprint in North America, with targeted augmentation to support the rollout of Robotaxi. While in the short-term, operational workstreams such as charging, cleaning and maintenance can be managed through our existing charging network, service centers and sales and delivery locations, we will have to add more capacity as the service expands. We added over 3,800 net new Supercharging stalls, growing the network 19% year-over-year.\nInstalled Annual Capacity\n\n[tables/table-6 Facility Status.html]\n\nInstalled capacity ≠ current production rate and there may be limitations discovered as production rates approach capacity. Production rates depend on a variety of factors, including equipment uptime, component supply, downtime related to factory upgrades, regulatory considerations and other factors. Construction includes factory and infrastructure buildout as well as tool installation.\n\n## Other Supporting Infrastructure Tesla AI Training Capacity Ramp (H100 equivalent GPUs)0\n[images/image-1-Capacity Growth Projection.jpg]\n\nTesla AI Training Capacity Ramp (H100 equivalent GPUs)", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->Other Supporting Infrastructure", + "metadata": { + "length": 1142, + "summary": "", + "page_nums": [ + 9, + 10 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "continue", + "efficiently", + "utilize", + "existing", + "physical", + "footprint", + "North", + "America", + "targeted", + "augmentation", + "support", + "rollout", + "Robotaxi", + "While", + "short", + "term", + "operational", + "workstreams", + "charging", + "cleaning", + "maintenance", + "managed", + "network", + "service", + "centers", + "sales", + "delivery", + "locations", + "add", + "capacity", + "expands", + "added", + "800", + "net", + "Supercharging", + "stalls", + "growing", + "19%", + "year", + "Installed", + "Annual", + "Capacity", + "current", + "production", + "rate", + "limitations", + "discovered", + "rates", + "approach", + "Production", + "depend", + "variety", + "factors", + "including", + "equipment", + "uptime", + "component", + "supply", + "downtime", + "related", + "factory", + "upgrades", + "regulatory", + "considerations", + "Construction", + "includes", + "infrastructure", + "buildout", + "tool", + "installation", + "Other", + "Supporting", + "Infrastructure", + "Tesla", + "AI", + "Training", + "Ramp", + "H100", + "equivalent", + "GPUs" + ], + "keywords": [], + "connect_to": [ + { + "target": "0428af2c-bf5d-54a6-a142-09ec146d027e", + "relation": "embeds", + "ref": "[tables/table-6 Facility Status.html]", + "position": { + "start": 519, + "end": 556 + } + }, + { + "target": "92dc6e40-720d-55b8-a2e2-c78bd4f59dba", + "relation": "embeds", + "ref": "[images/image-1-Capacity Growth Projection.jpg]", + "position": { + "start": 1040, + "end": 1087 + } + } + ] + } + }, + { + "chunk_id": "778a63c2-7955-514c-84d0-9c2cfd99a489", + "type": "text", + "content": "We continue to enhance FSD (Supervised) $^{1}$ via our end-to-end foundation model trained on both customer and Robotaxi real-world data with our latest version, v14. FSD (Supervised) $^{1}$ increasingly provides safety and convenience functionality that can relieve drivers of many tedious and potentially dangerous aspects of road travel, including giving access to personal transport for those who otherwise have difficulty driving. V14 offers unparalleled driver assistance to safely drive to the customer's destination, find a free parking spot and park at that location. Our global fleet can collect the equivalent of over 500 years of continuous driving data per day $^{2}$ , allowing us to safely deploy and scale capabilities that can handle the long and fat tail of corner cases across diverse geographies and driving environments.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->AI Software", + "metadata": { + "length": 841, + "summary": "", + "page_nums": [ + 10 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "continue", + "enhance", + "FSD", + "Supervised", + "end", + "foundation", + "model", + "trained", + "customer", + "Robotaxi", + "real", + "world", + "data", + "latest", + "version", + "v14", + "increasingly", + "safety", + "convenience", + "functionality", + "relieve", + "drivers", + "tedious", + "potentially", + "dangerous", + "aspects", + "road", + "travel", + "including", + "giving", + "access", + "personal", + "transport", + "difficulty", + "driving", + "V14", + "offers", + "unparalleled", + "driver", + "assistance", + "safely", + "drive", + "destination", + "find", + "free", + "parking", + "spot", + "park", + "location", + "Our", + "global", + "fleet", + "collect", + "equivalent", + "500", + "years", + "continuous", + "day", + "allowing", + "deploy", + "scale", + "capabilities", + "handle", + "long", + "fat", + "tail", + "corner", + "cases", + "diverse", + "geographies", + "environments" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "164babde-7a64-5b81-9a5a-41443b221c40", + "type": "text", + "content": "Development of our in-house, custom designed AI5 and AI6 inference chips for autonomy progressed during the quarter, with production planned for 2027 and 2028, respectively. We are targeting a 50x improvement in performance for AI5 relative to AI4 thanks to 10x raw compute, 9x memory capacity and 5x hardened block quantization and softmax function (the latter enabling efficient low-precision computing without sacrificing model accuracy).", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->AI Inference Compute", + "metadata": { + "length": 441, + "summary": "", + "page_nums": [ + 10 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Development", + "house", + "custom", + "designed", + "AI5", + "AI6", + "inference", + "chips", + "autonomy", + "progressed", + "quarter", + "production", + "planned", + "2027", + "2028", + "We", + "targeting", + "50x", + "improvement", + "performance", + "relative", + "AI4", + "10x", + "raw", + "compute", + "9x", + "memory", + "capacity", + "5x", + "hardened", + "block", + "quantization", + "softmax", + "function", + "enabling", + "efficient", + "low", + "precision", + "computing", + "sacrificing", + "model", + "accuracy" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "e4a241cf-298e-51c1-a928-4c6d21840920", + "type": "image", + "content": "\nWe continue to enhance FSD (Supervised) $^{1}$ via our end-to-end foundation model trained on both customer and Robotaxi real-world data with our latest version, v14. FSD (Supervised) $^{1}$ increasingly provides safety and convenience functionality that can relieve drivers of many tedious and potentially dangerous aspects of road travel, including giving access to personal transport for those who otherwise have difficulty driving. V14 offers unparalleled driver assistance to safely drive to the customer's destination, find a free parking spot and park at that location. Our global fleet can collect the equivalent of over 500 years of continuous driving data per day $^{2}$ , allowing us to safely deploy and scale capabilities that can handle the long and fat tail of corner cases across diverse geographies and driving environments. Cumulative Miles Driven with FSD (Supervised) $^{1}$ (billions)0\n[images/image-2-FSD Mileage Growth.jpg]\n", + "path": "images/image-2-FSD Mileage Growth.jpg", + "metadata": { + "length": 940, + "summary": "image-2\nThe chart illustrates the projected accumulation of miles driven on Full Self-Driving software over time. It distinguishes between two versions: an older version (V11 and before) represented by a blue area, and a newer version (V12 and beyond) shown in red. While mileage for the older version remains relatively flat, the newer version shows exponential growth starting around early 2024, eventually dominating the total distance traveled by late 2025.", + "page_nums": [ + 10 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-2-FSD Mileage Growth.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "1bbb4be7-849a-557b-86de-9d71a3e33ef0", + "type": "image", + "content": "\nDevelopment of our in-house, custom designed AI5 and AI6 inference chips for autonomy progressed during the quarter, with production planned for 2027 and 2028, respectively. We are targeting a 50x improvement in performance for AI5 relative to AI4 thanks to 10x raw compute, 9x memory capacity and 5x hardened block quantization and softmax function (the latter enabling efficient low-precision computing without sacrificing model accuracy). Targeting Step-Function Improvement for our Next-Generation Inference Chip, AI50\n[images/image-3-Tesla Silicon Optimization.jpg]\n", + "path": "images/image-3-Tesla Silicon Optimization.jpg", + "metadata": { + "length": 556, + "summary": "image-3\nThe image displays a high-performance computing chip alongside key performance metrics. It highlights significant improvements in hardened block quantization, memory capacity, and raw compute power compared to previous generations. The overall total improvement is presented as a substantial increase over the AI4 architecture.", + "page_nums": [ + 10 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-3-Tesla Silicon Optimization.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "32bfae6f-b2d4-51d3-94d9-82b9640828dd", + "type": "text", + "content": "The Robotaxi iOS app no longer has a waitlist in the areas we serve. Our vehicles keep getting better with our over-the-air updates, including: Grok (an AI companion) which now supports navigation commands (allowing users to find, add and edit navigation destinations hands-free); Tesla Photobooth which enables users to take photos in their car and download or share via the Tesla mobile app; Supercharger Site Maps which displays Supercharger layouts, nearby businesses and live availability details; Automatic HOV Lane Routing based on interior camera occupancy detection; Phone Left Behind Chime and SpaceX ISS Docking Simulator Game.\n\nWe continue to enhance FSD (Supervised) $^{1}$ via our end-to-end foundation model trained on both customer and Robotaxi real-world data with our latest version, v14. FSD (Supervised) $^{1}$ increasingly provides safety and convenience functionality that can relieve drivers of many tedious and potentially dangerous aspects of road travel, including giving access to personal transport for those who otherwise have difficulty driving. V14 offers unparalleled driver assistance to safely drive to the customer's destination, find a free parking spot and park at that location. Our global fleet can collect the equivalent of over 500 years of continuous driving data per day $^{2}$ , allowing us to safely deploy and scale capabilities that can handle the long and fat tail of corner cases across diverse geographies and driving environments. Cumulative Miles Driven with FSD (Supervised) $^{1}$ (billions)0\n[images/image-2-FSD Mileage Growth.jpg]\n\nCumulative Miles Driven with FSD (Supervised) $^{1}$ (billions)\n\nDevelopment of our in-house, custom designed AI5 and AI6 inference chips for autonomy progressed during the quarter, with production planned for 2027 and 2028, respectively. We are targeting a 50x improvement in performance for AI5 relative to AI4 thanks to 10x raw compute, 9x memory capacity and 5x hardened block quantization and softmax function (the latter enabling efficient low-precision computing without sacrificing model accuracy). Targeting Step-Function Improvement for our Next-Generation Inference Chip, AI50\n[images/image-3-Tesla Silicon Optimization.jpg]\n\nTargeting Step-Function Improvement for our Next-Generation Inference Chip, AI5\n(1) Active driver supervision required; does not make the vehicle autonomous\n(2) Calculated based on continuous hours of driving at an average of 30 miles per hour", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->Automotive and Other Software", + "metadata": { + "length": 2444, + "summary": "The Robotaxi iOS app has removed its waitlist in served areas and now includes features like Grok navigation, Tesla Photobooth, Supercharger maps, automatic HOV routing, phone left-behind chimes, and a SpaceX game. FSD (Supervised) v14 uses an end-to-end foundation model trained on vast real-world data to assist drivers with navigation, parking, and safety, though active supervision remains required. The company is developing custom AI5 and AI6 inference chips for 2027 and 2028, targeting significant performance improvements over previous generations to handle complex driving scenarios globally.", + "page_nums": [ + 10 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "The", + "Robotaxi", + "iOS", + "app", + "longer", + "waitlist", + "areas", + "serve", + "Our", + "vehicles", + "air", + "updates", + "including", + "Grok", + "AI", + "companion", + "supports", + "navigation", + "commands", + "allowing", + "users", + "find", + "add", + "edit", + "destinations", + "hands", + "free", + "Tesla", + "Photobooth", + "enables", + "photos", + "car", + "download", + "share", + "mobile", + "Supercharger", + "Site", + "Maps", + "displays", + "layouts", + "nearby", + "businesses", + "live", + "availability", + "details", + "Automatic", + "HOV", + "Lane", + "Routing", + "based", + "interior", + "camera", + "occupancy", + "detection", + "Phone", + "Left", + "Behind", + "Chime", + "SpaceX", + "ISS", + "Docking", + "Simulator", + "Game", + "We", + "continue", + "enhance", + "FSD", + "Supervised", + "end", + "foundation", + "model", + "trained", + "customer", + "real", + "world", + "data", + "latest", + "version", + "v14", + "increasingly", + "safety", + "convenience", + "functionality", + "relieve", + "drivers", + "tedious", + "potentially", + "dangerous", + "aspects", + "road", + "travel", + "giving", + "access", + "personal", + "transport", + "difficulty", + "driving", + "V14", + "offers", + "unparalleled", + "driver", + "assistance", + "safely", + "drive", + "destination", + "parking", + "spot", + "park", + "location", + "global", + "fleet", + "collect", + "equivalent", + "500", + "years", + "continuous", + "day", + "deploy", + "scale", + "capabilities", + "handle", + "long", + "fat", + "tail", + "corner", + "cases", + "diverse", + "geographies", + "environments", + "Cumulative", + "Miles", + "Driven", + "billions", + "Development", + "house", + "custom", + "designed", + "AI5", + "AI6", + "inference", + "chips", + "autonomy", + "progressed", + "quarter", + "production", + "planned", + "2027", + "2028", + "targeting", + "50x", + "improvement", + "performance", + "relative", + "AI4", + "10x", + "raw", + "compute", + "9x", + "memory", + "capacity", + "5x", + "hardened", + "block", + "quantization", + "softmax", + "function", + "enabling", + "efficient", + "low", + "precision", + "computing", + "sacrificing", + "accuracy", + "Targeting", + "Step", + "Function", + "Improvement", + "Next", + "Generation", + "Inference", + "Chip", + "AI50", + "Active", + "supervision", + "required", + "make", + "vehicle", + "autonomous", + "Calculated", + "hours", + "average", + "30", + "miles", + "hour" + ], + "keywords": [ + "Robotaxi", + "FSD", + "AI Chips" + ], + "connect_to": [ + { + "target": "e4a241cf-298e-51c1-a928-4c6d21840920", + "relation": "embeds", + "ref": "[images/image-2-FSD Mileage Growth.jpg]", + "position": { + "start": 1547, + "end": 1586 + } + }, + { + "target": "1bbb4be7-849a-557b-86de-9d71a3e33ef0", + "relation": "embeds", + "ref": "[images/image-3-Tesla Silicon Optimization.jpg]", + "position": { + "start": 2176, + "end": 2223 + } + } + ] + } + }, + { + "chunk_id": "41533301-58f0-554f-916d-8254cc2700df", + "type": "text", + "content": "We began testing driverless Robotaxis in Austin in December and began removing the safety monitor from customer rides in January on a limited basis, which will unlock further expansion of our Robotaxi fleet and coverage area in the Austin-metro. Our Bay Area ride-hailing service began serving the San Jose Airport in October, with plans to expand to other major airports in the Bay Area upon receiving required permitting.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->Robotaxi", + "metadata": { + "length": 423, + "summary": "", + "page_nums": [ + 10 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "began", + "testing", + "driverless", + "Robotaxis", + "Austin", + "December", + "removing", + "safety", + "monitor", + "customer", + "rides", + "January", + "limited", + "basis", + "unlock", + "expansion", + "Robotaxi", + "fleet", + "coverage", + "area", + "metro", + "Our", + "Bay", + "Area", + "ride", + "hailing", + "service", + "serving", + "San", + "Jose", + "Airport", + "October", + "plans", + "expand", + "major", + "airports", + "receiving", + "required", + "permitting" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "3968a25f-933e-5a7f-a118-07db2e581153", + "type": "text", + "content": "We launched FSD (Supervised) $^{1}$ in South Korea, where customers drove over 1 million kilometers using the software in just one month. While we continue to pursue regulatory approval in China and Europe, we began offering ride-along experiences to consumers in Italy, Germany, France and Switzerland.\nMonthly subscriptions to FSD (Supervised) $^{1}$ continued to grow sequentially and more than doubled in 2025. Starting this quarter, we are transitioning access to FSD (Supervised) $^{1}$ to monthly subscriptions only as we begin to sunset the up-front payment option.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->FSD (Supervised) $^{1}$", + "metadata": { + "length": 573, + "summary": "", + "page_nums": [ + 10 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "launched", + "FSD", + "Supervised", + "South", + "Korea", + "customers", + "drove", + "million", + "kilometers", + "software", + "month", + "While", + "continue", + "pursue", + "regulatory", + "approval", + "China", + "Europe", + "began", + "offering", + "ride", + "experiences", + "consumers", + "Italy", + "Germany", + "France", + "Switzerland", + "Monthly", + "subscriptions", + "continued", + "grow", + "sequentially", + "doubled", + "2025", + "Starting", + "quarter", + "transitioning", + "access", + "monthly", + "begin", + "sunset", + "front", + "payment", + "option" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "76c2f1a2-0b45-52a1-9708-e230345c7fae", + "type": "image", + "content": "\n## FSD (Supervised) $^{1}$ Cumulative Paid Robotaxi Miles0\n[images/image-4-Growth Trend 2025.jpg]\n", + "path": "images/image-4-Growth Trend 2025.jpg", + "metadata": { + "length": 92, + "summary": "image-4\nThe chart illustrates a steady increase in values over time, starting from June 2025 and extending through December 2025. The data shows minimal growth initially, followed by a significant upward trajectory beginning around August, reaching its peak at the end of the year.", + "page_nums": [ + 10 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-4-Growth Trend 2025.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "d4b47369-e206-5d49-afdd-db918ddee012", + "type": "table", + "content": "
StateMetroStatus
CaliforniaSF Bay AreaSafety Driver
TexasAustinRamping Unsupervised
Dallas1H 2026
Houston1H 2026
ArizonaPhoenix1H 2026
FloridaMiami1H 2026
Orlando1H 2026
Tampa1H 2026
NevadaLas Vegas1H 2026
", + "path": "tables/table-7 Autonomous Driving Status.html", + "metadata": { + "length": 571, + "summary": "table-8\nTable lists US states and metro areas with autonomous driving status. California SF Bay Area has safety drivers. Texas Austin is ramping unsupervised. Other locations like Dallas, Houston, Phoenix, Miami, Orlando, Tampa, and Las Vegas are scheduled for 1H 2026.", + "page_nums": [ + 10 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-7 Autonomous Driving Status.html", + "keywords": [ + "safety driver", + "ramping", + "2026 plans" + ], + "tokens": [] + } + }, + { + "chunk_id": "adf89909-d366-51e1-b68c-459f9024191c", + "type": "text", + "content": "Services and Other gross profit of approximately \\$300 million was partly driven by Part Sales and Supercharging. We now offer Tesla Insurance in Florida, as we continue to expand our insurance product to new states. In certain states, customers receive a discount on their insurance premiums when using FSD (Supervised) $^{1}$ . The more you drive with FSD (Supervised) $^{1}$ enabled, the bigger the discount is on your insurance premium – helping, in certain cases, to completely offset the monthly subscription cost for FSD (Supervised) $^{1}$ .\n\n## FSD (Supervised) $^{1}$ Cumulative Paid Robotaxi Miles0\n[images/image-4-Growth Trend 2025.jpg]\n\nCumulative Paid Robotaxi Miles\n\n[tables/table-7 Autonomous Driving Status.html]\n\nPlanned Robotaxi Coverage", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->Automotive Services", + "metadata": { + "length": 742, + "summary": "", + "page_nums": [ + 10, + 12 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Services", + "Other", + "gross", + "profit", + "approximately", + "300", + "million", + "partly", + "driven", + "Part", + "Sales", + "Supercharging", + "We", + "offer", + "Tesla", + "Insurance", + "Florida", + "continue", + "expand", + "insurance", + "product", + "states", + "In", + "customers", + "receive", + "discount", + "premiums", + "FSD", + "Supervised", + "The", + "drive", + "enabled", + "bigger", + "premium", + "helping", + "cases", + "completely", + "offset", + "monthly", + "subscription", + "cost", + "Cumulative", + "Paid", + "Robotaxi", + "Miles0", + "Miles", + "Planned", + "Coverage" + ], + "keywords": [], + "connect_to": [ + { + "target": "76c2f1a2-0b45-52a1-9708-e230345c7fae", + "relation": "embeds", + "ref": "[images/image-4-Growth Trend 2025.jpg]", + "position": { + "start": 610, + "end": 648 + } + }, + { + "target": "d4b47369-e206-5d49-afdd-db918ddee012", + "relation": "embeds", + "ref": "[tables/table-7 Autonomous Driving Status.html]", + "position": { + "start": 682, + "end": 729 + } + } + ] + } + }, + { + "chunk_id": "cf91377e-176a-5768-855a-b859092f4695", + "type": "text", + "content": "On January 16, 2026, Tesla entered into an agreement to invest approximately \\$2 billion to acquire shares of Series E Preferred Stock of xAI as part of their recent publicly-disclosed financing round. Tesla’s investment was made on market terms consistent with those previously agreed to by other investors in the financing round. As set forth in Master Plan Part IV, Tesla is building products and services that bring AI into the physical world. Meanwhile, xAI is developing leading digital AI products and services, such as its large language model (Grok).\nIn that context, and as part of Tesla's broader strategy under Master Plan Part IV, Tesla and xAI also entered into a framework agreement in connection with the investment. Among other things, the framework agreement builds upon the existing relationship between Tesla and xAI by providing a framework for evaluating potential AI collaborations between the companies. Together, the investment and the related framework agreement are intended to enhance Tesla's ability to develop and deploy AI products and services into the physical world at scale. This investment is subject to customary regulatory conditions with the expectation to close in Q1'2026.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OTHER UPDATES", + "metadata": { + "length": 1213, + "summary": "", + "page_nums": [ + 12, + 13 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "On", + "January", + "16", + "2026", + "Tesla", + "entered", + "agreement", + "invest", + "approximately", + "billion", + "acquire", + "shares", + "Series", + "Preferred", + "Stock", + "xAI", + "part", + "recent", + "publicly", + "disclosed", + "financing", + "round", + "investment", + "made", + "market", + "terms", + "consistent", + "previously", + "agreed", + "investors", + "As", + "set", + "Master", + "Plan", + "Part", + "IV", + "building", + "products", + "services", + "bring", + "AI", + "physical", + "world", + "Meanwhile", + "developing", + "leading", + "digital", + "large", + "language", + "model", + "Grok", + "In", + "context", + "broader", + "strategy", + "framework", + "connection", + "Among", + "things", + "builds", + "existing", + "relationship", + "providing", + "evaluating", + "potential", + "collaborations", + "companies", + "Together", + "related", + "intended", + "enhance", + "ability", + "develop", + "deploy", + "scale", + "This", + "subject", + "customary", + "regulatory", + "conditions", + "expectation", + "close", + "Q1" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "79b23d73-b353-5998-a646-93d6739ec465", + "type": "text", + "content": "", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK", + "metadata": { + "length": 0, + "summary": "", + "page_nums": [ + 13 + ], + "document_top_summary": "This document includes:", + "tokens": [], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "8d4c49a6-c531-5fa6-a77d-ad028537fa52", + "type": "text", + "content": "We are focused on maximum capacity utilization at our factories. Deliveries and deployments will be impacted by aggregate demand for our products, supply chain readiness and allocation decisions between sale to customers or use for our owned and operated fleet.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Volume", + "metadata": { + "length": 261, + "summary": "", + "page_nums": [ + 13 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "focused", + "maximum", + "capacity", + "utilization", + "factories", + "Deliveries", + "deployments", + "impacted", + "aggregate", + "demand", + "products", + "supply", + "chain", + "readiness", + "allocation", + "decisions", + "sale", + "customers", + "owned", + "operated", + "fleet" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "426e5119-3ef6-5176-9119-fc044eb90be7", + "type": "text", + "content": "We will manage the businesses such that we ensure a strong balance sheet, maintaining sufficient liquidity to fund our product roadmap, long-term capacity expansion plans – including further vertical integration – and other expenses.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Cash", + "metadata": { + "length": 233, + "summary": "", + "page_nums": [ + 13 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "manage", + "businesses", + "ensure", + "strong", + "balance", + "sheet", + "maintaining", + "sufficient", + "liquidity", + "fund", + "product", + "roadmap", + "long", + "term", + "capacity", + "expansion", + "plans", + "including", + "vertical", + "integration", + "expenses" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "99f54247-8409-5df2-8299-39184863cd09", + "type": "text", + "content": "While we continue to execute on innovations to reduce the cost of manufacturing and operations, over time, we expect our hardware-related profits to be accompanied by an acceleration of AI, software and fleet-based profits.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Profit", + "metadata": { + "length": 223, + "summary": "", + "page_nums": [ + 13 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "While", + "continue", + "execute", + "innovations", + "reduce", + "cost", + "manufacturing", + "operations", + "time", + "expect", + "hardware", + "related", + "profits", + "accompanied", + "acceleration", + "AI", + "software", + "fleet", + "based" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "29e89187-3dbe-5298-87c3-c38d3cbe6883", + "type": "text", + "content": "We continue to evolve and augment our product lineup with a focus on cost, scale and future monetization opportunities via services powered by our AI software. We remain focused on growing our sales volumes through a differentiated and efficiently managed product portfolio, which includes leveraging and optimizing our existing production capacity before building new factories and production lines.\nCybercab, Tesla Semi and Megapack 3 are on schedule for volume production starting in 2026. First generation production lines for Optimus are being installed in anticipation of volume production.\nPHOTOS & CHARTS", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Product", + "metadata": { + "length": 612, + "summary": "", + "page_nums": [ + 13, + 14, + 15 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "We", + "continue", + "evolve", + "augment", + "product", + "lineup", + "focus", + "cost", + "scale", + "future", + "monetization", + "opportunities", + "services", + "powered", + "AI", + "software", + "remain", + "focused", + "growing", + "sales", + "volumes", + "differentiated", + "efficiently", + "managed", + "portfolio", + "includes", + "leveraging", + "optimizing", + "existing", + "production", + "capacity", + "building", + "factories", + "lines", + "Cybercab", + "Tesla", + "Semi", + "Megapack", + "schedule", + "volume", + "starting", + "2026", + "First", + "generation", + "Optimus", + "installed", + "anticipation", + "PHOTOS", + "CHARTS" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "d119baec-2829-56d3-8406-fc994dfe3adf", + "type": "image", + "content": "\nCybercab, Tesla Semi and Megapack 3 are on schedule for volume production starting in 2026. First generation production lines for Optimus are being installed in anticipation of volume production. 0\n[images/image-5-Tesla Model Y Driving.jpg]\n", + "path": "images/image-5-Tesla Model Y Driving.jpg", + "metadata": { + "length": 247, + "summary": "image-5\nA sleek silver electric SUV travels along a winding highway through a scenic landscape. The vehicle is captured in motion with blurred surroundings, emphasizing speed against a backdrop of rolling hills and distant mountains under a bright sky.", + "page_nums": [ + 15 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-5-Tesla Model Y Driving.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "bbe9d89e-eebd-56b1-b7c0-7eda12dd3377", + "type": "text", + "content": "Cybercab, Tesla Semi and Megapack 3 are on schedule for volume production starting in 2026. First generation production lines for Optimus are being installed in anticipation of volume production. 0\n[images/image-5-Tesla Model Y Driving.jpg]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Product-->MODEL Y - 2025 BEST IN CLASS EURO NCAP $^{(1)}$ SMALL SUV", + "metadata": { + "length": 245, + "summary": "", + "page_nums": [ + 15, + 16 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Cybercab", + "Tesla", + "Semi", + "Megapack", + "schedule", + "volume", + "production", + "starting", + "2026", + "First", + "generation", + "lines", + "Optimus", + "installed", + "anticipation", + "page", + "16" + ], + "keywords": [], + "connect_to": [ + { + "target": "d119baec-2829-56d3-8406-fc994dfe3adf", + "relation": "embeds", + "ref": "[images/image-5-Tesla Model Y Driving.jpg]", + "position": { + "start": 214, + "end": 256 + } + } + ] + } + }, + { + "chunk_id": "f62cb7da-1f39-5ea6-a2f4-21ff5f5ff0d1", + "type": "image", + "content": "\n 0\n[images/image-6-Red Tesla on Coastal Road.jpg]\n", + "path": "images/image-6-Red Tesla on Coastal Road.jpg", + "metadata": { + "length": 68, + "summary": "image-6\nA red electric sedan drives along a winding asphalt road carved into a steep, rocky mountainside. The vehicle is captured in motion with a blurred background, emphasizing speed as it navigates the curve. To the right of the road lies a calm body of blue water, while the left side features a rugged cliff face covered in sparse green vegetation under a clear sky.", + "page_nums": [ + 16 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-6-Red Tesla on Coastal Road.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "02f7e756-798f-5675-9dd9-6ce9123f92d4", + "type": "text", + "content": " 0\n[images/image-6-Red Tesla on Coastal Road.jpg]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Product-->MODEL 3 - 2025 BEST IN CLASS EURO NCAP $^{(1)}$ LARGE FAMILY CAR", + "metadata": { + "length": 66, + "summary": "", + "page_nums": [ + 16, + 17 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "page", + "15", + "17" + ], + "keywords": [], + "connect_to": [ + { + "target": "f62cb7da-1f39-5ea6-a2f4-21ff5f5ff0d1", + "relation": "embeds", + "ref": "[images/image-6-Red Tesla on Coastal Road.jpg]", + "position": { + "start": 35, + "end": 81 + } + } + ] + } + }, + { + "chunk_id": "ca26e264-c569-50b1-bef6-8cee03a027f2", + "type": "image", + "content": "\n 0\n[images/image-7-Tesla Interior Interface.jpg]\n", + "path": "images/image-7-Tesla Interior Interface.jpg", + "metadata": { + "length": 68, + "summary": "image-7\nThe image displays the interior of a Tesla vehicle, focusing on the driver's perspective. A person is interacting with the large central touchscreen display, which shows navigation maps and vehicle controls. The steering wheel features the Tesla logo, and ambient lighting accents are visible along the dashboard. Through the windshield, a modern stone building is seen outside.", + "page_nums": [ + 17 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-7-Tesla Interior Interface.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "494118dd-5788-526b-a139-407d1cfb0d20", + "type": "text", + "content": " 0\n[images/image-7-Tesla Interior Interface.jpg]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Product-->FSD (SUPERVISED) $^{1}$ – V14 OFFERS UNPARALLELED DRIVER ASSISTANCE", + "metadata": { + "length": 66, + "summary": "", + "page_nums": [ + 17, + 18 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "page", + "16", + "18" + ], + "keywords": [], + "connect_to": [ + { + "target": "ca26e264-c569-50b1-bef6-8cee03a027f2", + "relation": "embeds", + "ref": "[images/image-7-Tesla Interior Interface.jpg]", + "position": { + "start": 35, + "end": 80 + } + } + ] + } + }, + { + "chunk_id": "47fd2c58-a064-544f-ac09-2ddd83eeb248", + "type": "image", + "content": "\n 0\n[images/image-8-Tesla Interior.jpg]\n", + "path": "images/image-8-Tesla Interior.jpg", + "metadata": { + "length": 68, + "summary": "image-8\nThe image displays the driver's perspective inside a Tesla vehicle, featuring a minimalist dashboard with a large central touchscreen. The steering wheel is visible on the left side, and the car appears to be in motion on a city street during daylight hours.", + "page_nums": [ + 18 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-8-Tesla Interior.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "6ac7765a-4da5-573a-963b-f35ab3796f6f", + "type": "text", + "content": " 0\n[images/image-8-Tesla Interior.jpg]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Product-->DRIVERLESS ROBOTAXI - TESTING IN AUSTIN", + "metadata": { + "length": 66, + "summary": "", + "page_nums": [ + 18, + 19 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "page", + "17", + "19" + ], + "keywords": [], + "connect_to": [ + { + "target": "47fd2c58-a064-544f-ac09-2ddd83eeb248", + "relation": "embeds", + "ref": "[images/image-8-Tesla Interior.jpg]", + "position": { + "start": 35, + "end": 70 + } + } + ] + } + }, + { + "chunk_id": "04fc84c0-3bc5-50c2-a7d8-40b4f22a257e", + "type": "image", + "content": "\n 0\n[images/image-9-Tesla Cybertruck in Snow.jpg]\n", + "path": "images/image-9-Tesla Cybertruck in Snow.jpg", + "metadata": { + "length": 68, + "summary": "image-9\nA futuristic electric pickup truck is shown driving on a frozen, snow-covered surface. The vehicle features its signature angular design and metallic finish, with snow clinging to the rear bumper and wheel wells. It is set against a backdrop of distant mountains under a twilight sky.", + "page_nums": [ + 19 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-9-Tesla Cybertruck in Snow.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "4ebd8699-3d0c-523d-8288-9b7b5b6853aa", + "type": "image", + "content": "\n### DRIVERLESS ROBOTAXI - TESTING IN AUSTIN 0\n[images/image-10-Tesla Semi Trucks.jpg]\n", + "path": "images/image-10-Tesla Semi Trucks.jpg", + "metadata": { + "length": 96, + "summary": "image-10\nTwo white electric semi-trucks are parked side-by-side in an outdoor lot. The vehicles feature a futuristic, aerodynamic design with large windshields and distinctive horizontal headlights. They are positioned against a backdrop of industrial buildings and hills under a cloudy sky.", + "page_nums": [ + 19 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-10-Tesla Semi Trucks.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "bda2b508-b228-5ab9-b619-775c024822fa", + "type": "image", + "content": "\n### CYBERCAB - COLD WEATHER TESTING IN ALASKA 0\n[images/image-11-US Lightning Map.jpg]\n", + "path": "images/image-11-US Lightning Map.jpg", + "metadata": { + "length": 98, + "summary": "image-11\nA map of the United States displays numerous red markers with lightning symbols. These indicators are concentrated heavily along the West Coast, particularly in California, and throughout Texas. Additional clusters appear in the Southeast near Atlanta and scattered locations in the Midwest and Northeast.", + "page_nums": [ + 35 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-11-US Lightning Map.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "4cfe7ce9-5f93-5816-9505-ba562683ee42", + "type": "text", + "content": " 0\n[images/image-9-Tesla Cybertruck in Snow.jpg]\n\n\n### DRIVERLESS ROBOTAXI - TESTING IN AUSTIN 0\n[images/image-10-Tesla Semi Trucks.jpg]\n\nTESLA SEMI - MEGACHARGER NETWORK PLANNED SITES FOR 2026\n\n### CYBERCAB - COLD WEATHER TESTING IN ALASKA 0\n[images/image-11-US Lightning Map.jpg]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Product-->CYBERCAB - COLD WEATHER TESTING IN ALASKA", + "metadata": { + "length": 318, + "summary": "", + "page_nums": [ + 19, + 22, + 35 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "page", + "18", + "35", + "DRIVERLESS", + "ROBOTAXI", + "TESTING", + "IN", + "AUSTIN", + "TESLA", + "SEMI", + "MEGACHARGER", + "NETWORK", + "PLANNED", + "SITES", + "FOR", + "2026", + "CYBERCAB", + "COLD", + "WEATHER", + "ALASKA", + "22" + ], + "keywords": [], + "connect_to": [ + { + "target": "04fc84c0-3bc5-50c2-a7d8-40b4f22a257e", + "relation": "embeds", + "ref": "[images/image-9-Tesla Cybertruck in Snow.jpg]", + "position": { + "start": 35, + "end": 80 + } + }, + { + "target": "4ebd8699-3d0c-523d-8288-9b7b5b6853aa", + "relation": "embeds", + "ref": "[images/image-10-Tesla Semi Trucks.jpg]", + "position": { + "start": 145, + "end": 184 + } + }, + { + "target": "bda2b508-b228-5ab9-b619-775c024822fa", + "relation": "embeds", + "ref": "[images/image-11-US Lightning Map.jpg]", + "position": { + "start": 307, + "end": 345 + } + } + ] + } + }, + { + "chunk_id": "bf02caea-5069-5866-9a6c-5f0ff28c981b", + "type": "image", + "content": "\n 0\n[images/image-12-Tesla Factory Milestone.jpg]\n", + "path": "images/image-12-Tesla Factory Milestone.jpg", + "metadata": { + "length": 69, + "summary": "image-12\nFactory workers and staff gather for a group photo on an automotive assembly line to celebrate the production of the 900th vehicle. A white car is positioned centrally in front of the crowd, while employees hold silver balloons displaying the number \"900\" to mark this significant manufacturing achievement.", + "page_nums": [ + 22 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-12-Tesla Factory Milestone.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "07535539-a2f9-5b79-ac6b-d41cf785ec88", + "type": "text", + "content": " 0\n[images/image-12-Tesla Factory Milestone.jpg]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Product-->GIGAFACTORY SHANGHAI - 9 MILLIONTH VEHICLE PRODUCED (GLOBALLY)", + "metadata": { + "length": 67, + "summary": "", + "page_nums": [ + 22, + 23 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "page", + "35", + "23" + ], + "keywords": [], + "connect_to": [ + { + "target": "bf02caea-5069-5866-9a6c-5f0ff28c981b", + "relation": "embeds", + "ref": "[images/image-12-Tesla Factory Milestone.jpg]", + "position": { + "start": 35, + "end": 80 + } + } + ] + } + }, + { + "chunk_id": "a860be78-1920-5ae1-996e-40c19fc83386", + "type": "image", + "content": "\n 0\n[images/image-13-Tesla Factory Milestone.jpg]\n", + "path": "images/image-13-Tesla Factory Milestone.jpg", + "metadata": { + "length": 69, + "summary": "image-13\nA large group of factory workers gathers inside a manufacturing facility to celebrate a significant achievement. Several employees in the front row hold up oversized gold balloons that spell out \"600,000,\" marking a major production milestone for the company. The background reveals an industrial setting filled with machinery and assembly lines.", + "page_nums": [ + 23 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-13-Tesla Factory Milestone.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "b356395d-c199-500b-9879-6c7973de3bba", + "type": "image", + "content": "\n### GIGAFACTORY SHANGHAI - 9 MILLIONTH VEHICLE PRODUCED (GLOBALLY) 0\n[images/image-14-Vehicle Delivery Trends.jpg]\n", + "path": "images/image-14-Vehicle Delivery Trends.jpg", + "metadata": { + "length": 119, + "summary": "image-14\nA bar chart displays quarterly vehicle delivery volumes in millions of units spanning from the first quarter of 2023 through the fourth quarter of 2025. The data illustrates fluctuating delivery figures across the timeline, with values generally ranging between approximately 0.3 and 0.5 million units per quarter.", + "page_nums": [ + 23 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-14-Vehicle Delivery Trends.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "10b8881e-6d2c-5737-8ffa-b75db9f1c850", + "type": "image", + "content": "\n 0\n[images/image-15-Quarterly Cash Flow.jpg]\n", + "path": "images/image-15-Quarterly Cash Flow.jpg", + "metadata": { + "length": 69, + "summary": "image-15\nThe chart compares operating cash flow and free cash flow across multiple quarters from 2023 through 2025. Blue bars represent operating cash flow, while red bars indicate free cash flow. Operating cash flow remains consistently positive throughout the period, whereas free cash flow fluctuates significantly, including a notable negative value in early 2024.", + "page_nums": [ + 23 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-15-Quarterly Cash Flow.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "8126ecb1-7d09-5687-8290-988e8bbe8eec", + "type": "image", + "content": "\n 0\n[images/image-16-Financial Performance Chart.jpg]\n", + "path": "images/image-16-Financial Performance Chart.jpg", + "metadata": { + "length": 69, + "summary": "image-16\nThis bar chart compares Net Income and Adjusted EBITDA across quarterly periods from 2023 through 2025. The blue bars represent Net Income while the red bars indicate Adjusted EBITDA, with values measured in billions of dollars. A significant spike in Net Income is visible during the fourth quarter of 2023, whereas Adjusted EBITDA remains consistently higher than Net Income throughout most of the timeline.", + "page_nums": [ + 23 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-16-Financial Performance Chart.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "702bff82-1736-584d-956e-e7bf7eac4039", + "type": "text", + "content": " 0\n[images/image-13-Tesla Factory Milestone.jpg]\n\n\n### GIGAFACTORY SHANGHAI - 9 MILLIONTH VEHICLE PRODUCED (GLOBALLY) 0\n[images/image-14-Vehicle Delivery Trends.jpg]\n\n\n 0\n[images/image-15-Quarterly Cash Flow.jpg]\n\n\n 0\n[images/image-16-Financial Performance Chart.jpg]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->Product-->GIGAFACTORY NEVADA - 6 MILLIONTH DRIVE UNIT PRODUCED", + "metadata": { + "length": 327, + "summary": "", + "page_nums": [ + 23, + 25 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "page", + "22", + "25", + "GIGAFACTORY", + "SHANGHAI", + "MILLIONTH", + "VEHICLE", + "PRODUCED", + "GLOBALLY", + "23" + ], + "keywords": [], + "connect_to": [ + { + "target": "a860be78-1920-5ae1-996e-40c19fc83386", + "relation": "embeds", + "ref": "[images/image-13-Tesla Factory Milestone.jpg]", + "position": { + "start": 35, + "end": 80 + } + }, + { + "target": "b356395d-c199-500b-9879-6c7973de3bba", + "relation": "embeds", + "ref": "[images/image-14-Vehicle Delivery Trends.jpg]", + "position": { + "start": 168, + "end": 213 + } + }, + { + "target": "10b8881e-6d2c-5737-8ffa-b75db9f1c850", + "relation": "embeds", + "ref": "[images/image-15-Quarterly Cash Flow.jpg]", + "position": { + "start": 251, + "end": 292 + } + }, + { + "target": "8126ecb1-7d09-5687-8290-988e8bbe8eec", + "relation": "embeds", + "ref": "[images/image-16-Financial Performance Chart.jpg]", + "position": { + "start": 330, + "end": 379 + } + } + ] + } + }, + { + "chunk_id": "2504a912-1296-5922-a39b-5aa2f2634d75", + "type": "image", + "content": "\n 0\n[images/image-17-Projected Vehicle Deliveries.jpg]\n", + "path": "images/image-17-Projected Vehicle Deliveries.jpg", + "metadata": { + "length": 69, + "summary": "image-17\nThe bar chart illustrates a forecast of vehicle deliveries in millions of units spanning from the first quarter of 2023 through the fourth quarter of 2025. The data indicates an upward trend starting in early 2023, reaching a peak around late 2023 and continuing at high levels throughout 2024 before showing a slight decline toward the end of the projection period in 2025.", + "page_nums": [ + 25 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-17-Projected Vehicle Deliveries.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "26481825-241e-53e6-b380-dbfa166ef8cd", + "type": "image", + "content": "\n 0\n[images/image-18-Cash Flow Trends.jpg]\n", + "path": "images/image-18-Cash Flow Trends.jpg", + "metadata": { + "length": 69, + "summary": "image-18\nThe chart compares operating and free cash flow across multiple quarters from 2023 through 2025. Operating cash flow is consistently higher than free cash flow throughout the timeline, with both metrics showing significant fluctuations over time.", + "page_nums": [ + 25 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-18-Cash Flow Trends.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "f19dcb65-d67c-51d4-a0fe-112a38f24d8c", + "type": "image", + "content": "\n 0\n[images/image-19-Financial Performance Forecast.jpg]\n", + "path": "images/image-19-Financial Performance Forecast.jpg", + "metadata": { + "length": 69, + "summary": "image-19\nThe chart displays a comparison of Net Income and Adjusted EBITDA over a multi-year period. It features paired bars for each quarter, showing that while earnings fluctuate, the adjusted metric remains consistently higher than net income throughout the timeline.", + "page_nums": [ + 25 + ], + "document_top_summary": "This document includes:", + "file_path": "images/image-19-Financial Performance Forecast.jpg", + "keywords": [], + "tokens": [] + } + }, + { + "chunk_id": "5e17588f-ea71-56df-be53-135da93fb3a0", + "type": "text", + "content": " 0\n[images/image-17-Projected Vehicle Deliveries.jpg]\n\n\n 0\n[images/image-18-Cash Flow Trends.jpg]\n\n\n 0\n[images/image-19-Financial Performance Forecast.jpg]", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->KEY METRICS TRAILING 12 MONTHS (TTM) (Unaudited)", + "metadata": { + "length": 207, + "summary": "", + "page_nums": [ + 25, + 26 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "page", + "25", + "26" + ], + "keywords": [], + "connect_to": [ + { + "target": "2504a912-1296-5922-a39b-5aa2f2634d75", + "relation": "embeds", + "ref": "[images/image-17-Projected Vehicle Deliveries.jpg]", + "position": { + "start": 35, + "end": 85 + } + }, + { + "target": "26481825-241e-53e6-b380-dbfa166ef8cd", + "relation": "embeds", + "ref": "[images/image-18-Cash Flow Trends.jpg]", + "position": { + "start": 123, + "end": 161 + } + }, + { + "target": "f19dcb65-d67c-51d4-a0fe-112a38f24d8c", + "relation": "embeds", + "ref": "[images/image-19-Financial Performance Forecast.jpg]", + "position": { + "start": 199, + "end": 251 + } + } + ] + } + }, + { + "chunk_id": "c82a6fed-f085-5ec3-b10a-0381723fa3c9", + "type": "text", + "content": "Total quarterly revenue decreased 3% YoY to \\$24.9B. YoY, revenue was impacted by the following items $^{(1)}$ :\n- decrease in vehicle deliveries\n- lower regulatory credit revenue\n+ growth in Energy Generation and Storage\n+ growth in Services and Other\n+ positive FX impact of \\$0.3B $^{1}$\n\\+ growth in other automotive ancillary sales, partly driven by an increase in FSD subscriptions\n\\+ higher vehicle average selling price (ASP) (excl. FX impact $^{1}$ ), inclusive of mix impact", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->KEY METRICS TRAILING 12 MONTHS (TTM) (Unaudited)-->Revenue", + "metadata": { + "length": 484, + "summary": "", + "page_nums": [ + 26 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Total", + "quarterly", + "revenue", + "decreased", + "3%", + "YoY", + "24.9", + "impacted", + "items", + "decrease", + "vehicle", + "deliveries", + "lower", + "regulatory", + "credit", + "growth", + "Energy", + "Generation", + "Storage", + "Services", + "Other", + "positive", + "FX", + "impact", + "0.3", + "automotive", + "ancillary", + "sales", + "partly", + "driven", + "increase", + "FSD", + "subscriptions", + "higher", + "average", + "selling", + "price", + "ASP", + "excl", + "inclusive", + "mix" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "946fabf6-000f-557a-9d81-0b8c1a25aafb", + "type": "text", + "content": "Our quarterly operating income decreased 11% YoY to \\$1.4B, resulting in a 5.7% operating margin. YoY, operating income was primarily impacted by the following items $^{(1)}$ :\n- increase in SBC and Restructuring and Other charges\n- increase in operating expenses (excl. SBC and Restructuring and Other) driven by AI and other R&D projects and SG&A\n- higher average cost per vehicle due to lower fixed cost absorption for certain models and an increase in tariffs\n- decrease in vehicle deliveries\n- lower regulatory credit revenue\n+ higher vehicle average gross profit due to mix and pricing impacts\n+ growth in Energy Generation and Storage gross profit\n+ growth in Services and Other gross profit\n+ growth in other automotive ancillary sales, partly driven by an increase in FSD subscriptions", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->KEY METRICS TRAILING 12 MONTHS (TTM) (Unaudited)-->Profitability", + "metadata": { + "length": 794, + "summary": "", + "page_nums": [ + 26 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Our", + "quarterly", + "operating", + "income", + "decreased", + "11%", + "YoY", + "1.4", + "resulting", + "5.7%", + "margin", + "primarily", + "impacted", + "items", + "increase", + "SBC", + "Restructuring", + "Other", + "charges", + "expenses", + "excl", + "driven", + "AI", + "projects", + "SG", + "higher", + "average", + "cost", + "vehicle", + "due", + "lower", + "fixed", + "absorption", + "models", + "tariffs", + "decrease", + "deliveries", + "regulatory", + "credit", + "revenue", + "gross", + "profit", + "mix", + "pricing", + "impacts", + "growth", + "Energy", + "Generation", + "Storage", + "Services", + "automotive", + "ancillary", + "sales", + "partly", + "FSD", + "subscriptions" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "001d4b8a-b264-5cae-86a5-8c419eabbec0", + "type": "text", + "content": "Quarter-end cash, cash equivalents and investments was \\$44.1B. The sequential increase of \\$2.4B was primarily the result of positive free cash flow.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->KEY METRICS TRAILING 12 MONTHS (TTM) (Unaudited)-->Cash", + "metadata": { + "length": 150, + "summary": "", + "page_nums": [ + 26, + 27 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Quarter", + "end", + "cash", + "equivalents", + "investments", + "44.1", + "The", + "sequential", + "increase", + "2.4", + "primarily", + "result", + "positive", + "free", + "flow" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "3ff046c6-93f4-5d22-a6af-11ecf2660ce3", + "type": "table", + "content": "
In millions of USD or shares as applicable, except per share dataQ4-2024Q1-2025Q2-2025Q3-2025Q4-2025
REVENUES
Automotive sales18,65912,92515,78720,35916,750
Automotive regulatory credits692595439417542
Automotive leasing447447435429401
Total automotive revenues19,79813,96716,66121,20517,693
Energy generation and storage3,0612,7302,7893,4153,837
Services and other2,8482,6383,0463,4753,371
Total revenues25,70719,33522,49628,09524,901
COST OF REVENUES
Automotive sales16,26811,46113,56717,36513,874
Automotive leasing242239228225206
Total automotive cost of revenues16,51011,70013,79517,59014,080
Energy generation and storage2,2891,9451,9432,3422,739
Services and other2,7292,5372,8803,1093,073
Total cost of revenues21,52816,18218,61823,04119,892
Gross profit4,1793,1533,8785,0545,009
OPERATING EXPENSES
Research and development1,2761,4091,5891,6301,783
Selling, general and administrative1,3131,2511,3661,5621,655
Restructuring and other794—238162
Total operating expenses2,5962,7542,9553,4303,600
INCOME FROM OPERATIONS1,5833999231,6241,409
Interest income442400392439449
Interest expense(96)(91)(86)(76)(85)
Other income (expense), net (1)595(119)320(28)(592)
INCOME BEFORE INCOME TAXES (1)2,5245891,5491,9591,181
Provision for income taxes (1)381169359570325
NET INCOME (1)2,1434201,1901,389856
Net income attributable to noncontrolling interests and redeemable noncontrolling interests in subsidiaries1511181616
NET INCOME ATTRIBUTABLE TO COMMON STOCKHOLDERS (1)2,1284091,1721,373840
Less: Buy-out of noncontrolling interest3————
NET INCOME USED IN COMPUTING NET INCOME PER SHARE OF COMMON STOCK (1)2,1254091,1721,373840
Net income per share of common stock attributable to common stockholders
Basic (1)$ 0.66$ 0.13$ 0.36$ 0.43$ 0.26
Diluted (1)$ 0.60$ 0.12$ 0.33$ 0.39$ 0.24
Weighted average shares used in computing net income per share of common stock
Basic3,2133,2183,2233,2273,231
Diluted3,5173,5213,5193,5263,539
", + "path": "tables/table-8 Q4 2024-Q4 2025 Rev.html", + "metadata": { + "length": 4195, + "summary": "table-9\nFinancial table showing quarterly revenues, costs, and net income from Q4 2024 to Q4 2025 in millions USD.", + "page_nums": [ + 27 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-8 Q4 2024-Q4 2025 Rev.html", + "keywords": [ + "revenue", + "expenses", + "income" + ], + "tokens": [] + } + }, + { + "chunk_id": "dc7fe1d0-57ac-514a-a80a-c46825591b4c", + "type": "table", + "content": "
In millions of USD31-Dec-2431-Mar-2530-Jun-2530-Sep-2531-Dec-25
ASSETS
Current assets
Cash, cash equivalents and investments36,56336,99636,78241,64744,059
Accounts receivable, net4,4183,7823,8384,7034,576
Inventory12,01713,70614,57012,27612,392
Prepaid expenses and other current assets5,3624,9055,9436,0277,615
Total current assets58,36059,38961,13364,65368,642
Operating lease vehicles, net5,5815,4775,2305,0194,912
Energy generation and storage systems, net4,9244,8554,7884,6734,604
Property, plant and equipment, net35,83637,08838,57439,40740,643
Operating lease right-of-use assets5,1605,3305,6335,7836,027
Digital assets (2)1,0769511,2351,3151,008
Deferred tax assets (2)6,5246,6876,7216,6376,925
Other non-current assets4,6095,3345,2536,2485,045
Total assets (2)122,070125,111128,567133,735137,806
LIABILITIES AND EQUITY
Current liabilities
Accounts payable12,47413,47113,21212,81913,371
Accrued liabilities and other10,72310,80211,51912,79113,279
Deferred revenue3,1683,2433,2373,7563,424
Current portion of debt and finance leases (1)2,4562,2372,0401,9241,640
Total current liabilities28,82129,75330,00831,29031,714
Debt and finance leases, net of current portion (1)5,7575,2925,1805,7786,736
Deferred revenue, net of current portion3,3173,6103,7643,7463,631
Other long-term liabilities10,49511,03811,54312,20512,860
Total liabilities48,39049,69350,49553,01954,941
Redeemable noncontrolling interests in subsidiaries6362615958
Total stockholders' equity (2)72,91374,65377,31479,97082,137
Noncontrolling interests in subsidiaries704703697687670
Total liabilities and equity (2)122,070125,111128,567133,735137,806
(1) Breakdown of our debt is as follows:
Non-recourse debt7,8717,2386,9537,4588,150
Recourse debt76333
Days sales outstanding1419151417
Days payable outstanding5872655261
", + "path": "tables/table-9 Balance Sheet 2024-25.html", + "metadata": { + "length": 3909, + "summary": "table-10\nFinancial table showing assets, liabilities, and equity from Dec 2024 to Dec 2025. Total assets grew from 122B to 138B USD.", + "page_nums": [ + 27 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-9 Balance Sheet 2024-25.html", + "keywords": [ + "assets", + "liabilities", + "equity" + ], + "tokens": [] + } + }, + { + "chunk_id": "0bbfd678-18cc-5152-8775-a4c805e47d57", + "type": "table", + "content": "
In millions of USDQ4-2024Q1-2025Q2-2025Q3-2025Q4-2025
CASH FLOWS FROM OPERATING ACTIVITIES
Net income (1)2,1434201,1901,389856
Adjustments to reconcile net income to net cash provided by operating activities:
Depreciation, amortization and impairment1,4961,4471,4331,6251,643
Stock-based compensation579573635663954
Deferred income taxes (1)6(43)52225(111)
Digital assets (gain) loss, net (1)(347)125(284)(80)307
Other(93)188187333378
Changes in operating assets and liabilities1030(554)(673)2,083(214)
Net cash provided by operating activities4,8142,1562,5406,2383,813
CASH FLOWS FROM INVESTING ACTIVITIES
Capital expenditures (2)(2,780)(1,492)(2,394)(2,248)(2,393)
Purchases of investments(15,158)(6,015)(7,485)(11,402)(12,207)
Proceeds from maturities of investments10,3355,8566,9359,2958,072
Net cash used in investing activities(7,603)(1,651)(2,944)(4,355)(6,528)
CASH FLOWS FROM FINANCING ACTIVITIES
Net cash flows from other debt activities(108)(50)(23)410963
Net borrowings (repayments) under vehicle and energy product financing677(674)(400)81(377)
Net cash flows from noncontrolling interests – Solar(37)(22)(14)(20)(22)
Other453414215512146
Net cash provided by (used in) financing activities985(332)(222)983710
Effect of exchange rate changes on cash and cash equivalents and restricted cash(133)40111(17)37
Net (decrease) increase in cash and cash equivalents and restricted cash(1,937)213(515)2,849(1,968)
Cash and cash equivalents and restricted cash at beginning of period18,97417,03717,25016,73519,584
Cash and cash equivalents and restricted cash at end of period17,03717,25016,73519,58417,616
", + "path": "tables/table-10 Cash Flow Q4-25.html", + "metadata": { + "length": 3137, + "summary": "table-11\nTable shows quarterly cash flows from operating, investing, and financing activities in millions USD for Q4 2024 to Q4 2025.", + "page_nums": [ + 27 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-10 Cash Flow Q4-25.html", + "keywords": [ + "cash flow", + "operating", + "investing" + ], + "tokens": [] + } + }, + { + "chunk_id": "675abba3-3bcf-5b5d-94dd-a9e9a711e78b", + "type": "table", + "content": "
In millions of USD or shares as applicable, except per share dataQ4-2024Q1-2025Q2-2025Q3-2025Q4-2025
Net income attributable to common stockholders (GAAP) (1)2,1284091,1721,373840
Stock-based compensation expense, net of tax249428443459682
Digital assets (gain) loss, net of tax (1)(270)97(222)(62)239
Net income attributable to common stockholders (non-GAAP) (1) (2)2,1079341,3931,7701,761
Less: Buy-outs of noncontrolling interests3————
Net income used in computing diluted EPS attributable to common stockholders (non-GAAP) (1) (2)2,1049341,3931,7701,761
EPS attributable to common stockholders, diluted (GAAP) (1)0.600.120.330.390.24
Stock-based compensation expense, net of tax, per share0.080.120.130.130.19
Digital assets (gain) loss, net of tax, per share (1)(0.08)0.03(0.06)(0.02)0.07
EPS attributable to common stockholders, diluted (non-GAAP) (1) (2)0.600.270.400.500.50
Shares used in EPS calculation, diluted (GAAP and non-GAAP)3,5173,5213,5193,5263,539
Net income attributable to common stockholders (GAAP) (1)2,1284091,1721,373840
Interest expense9691867685
Provision for income taxes (1)381169359570325
Depreciation, amortization and impairment1,4961,4471,4331,6251,643
Stock-based compensation expense579573635663954
Digital assets (gain) loss, net (1)(347)125(284)(80)307
Adjusted EBITDA (non-GAAP) (1) (3)4,3332,8143,4014,2274,154
Total revenues25,70719,33522,49628,09524,901
Adjusted EBITDA margin (non-GAAP) (1) (3)16.9%14.6%15.1%15.0%16.7%
Automotive gross margin (GAAP)16.6%16.2%17.2%17.0%20.4%
Less: Total regulatory credit revenue recognized3.0%3.7%2.2%1.6%2.5%
Automotive gross margin excluding regulatory credit sales (non-GAAP)13.6%12.5%15.0%15.4%17.9%
", + "path": "tables/table-11 Q4 2024-Q4 2025.html", + "metadata": { + "length": 3184, + "summary": "table-12\nFinancial table showing GAAP and non-GAAP metrics for quarters Q4-2024 through Q4-2025, including net income, EPS, adjusted EBITDA, and automotive margins.", + "page_nums": [ + 27 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-11 Q4 2024-Q4 2025.html", + "keywords": [ + "earnings", + "EBITDA", + "margins" + ], + "tokens": [] + } + }, + { + "chunk_id": "9a6a2ecd-f489-5cf2-971a-acb1610cbef8", + "type": "table", + "content": "
In millions of USD or shares as applicable, except per share data20212022202320242025
Net income attributable to common stockholders (GAAP)5,51912,55614,9977,0913,794
Stock-based compensation expense, net of tax2,1211,5601,8121,3282,012
Digital assets loss (gain), net of tax79160—(459)52
Release of valuation allowance on deferred tax assets——(5,927)——
Net income attributable to common stockholders (non-GAAP)(1)7,71914,27610,8827,9605,858
Less: Buy-outs of noncontrolling interests(5)(27)(2)(39)—
Less: Dilutive convertible debt(9)(1)———
Net income used in computing diluted EPS attributable to common stockholders (non-GAAP)(1)7,73314,30410,8847,9995,858
EPS attributable to common stockholders, diluted (GAAP)1.633.624.302.041.08
Stock-based compensation expense, net of tax, per share0.630.450.520.380.57
Digital assets loss (gain), net of tax, per share0.020.05—(0.13)0.01
Release of valuation allowance on deferred tax assets——(1.70)——
EPS attributable to common stockholders, diluted (non-GAAP)(1)2.284.123.122.291.66
Shares used in EPS calculation, diluted (GAAP and non-GAAP)3,3863,4753,4853,4983,528
Net income attributable to common stockholders (GAAP)5,51912,55614,9977,0913,794
Interest expense371191156350338
Provision for (benefit from) income taxes6991132(5,001)1,8371,423
Depreciation, amortization and impairment2,9113,7474,6675,3686,148
Stock-based compensation expense2,1211,5601,8121,9992,825
Digital assets loss (gain), net101204—(589)68
Adjusted EBITDA (non-GAAP)(2)11,72219,39016,63116,05614,596
Total revenues53,82381,46296,77397,69094,827
Adjusted EBITDA margin (non-GAAP)(2)21.8%23.8%17.2%16.4%15.4%
Automotive gross margin (GAAP)29.3%28.5%19.4%18.4%17.8%
Less: Total regulatory credit revenue recognized2.3%1.8%1.7%3.0%2.4%
Automotive gross margin excluding regulatory credit sales (non-GAAP)27.0%26.7%17.7%15.4%15.4%
", + "path": "tables/table-12 Financial Metrics 2021-25.html", + "metadata": { + "length": 3537, + "summary": "table-13\nTable shows financial data from 2021 to 2025 including GAAP and non-GAAP net income, EPS, adjusted EBITDA, and margins.", + "page_nums": [ + 27 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-12 Financial Metrics 2021-25.html", + "keywords": [ + "Net Income", + "EBITDA", + "EPS" + ], + "tokens": [] + } + }, + { + "chunk_id": "2ca08ebb-8851-5594-a629-470029525540", + "type": "table", + "content": "
In millions of USD2Q-20223Q-20224Q-20221Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20252Q-20253Q-20254Q-2025
Net cash provided by operating activities (GAAP)2,3515,1003,2782,5133,0653,3084,3702423,6126,2554,8142,1562,5406,2383,813
Capital expenditures (1)(1,730)(1,803)(1,858)(2,073)(2,060)(2,459)(2,307)(2,777)(2,272)(3,513)(2,780)(1,492)(2,394)(2,248)(2,393)
Free cash flow (non-GAAP) (1)6213,2971,4204401,0058492,063(2,535)1,3402,7422,0346641463,9901,420
In millions of USD2Q-20223Q-20224Q-20221Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20 252Q-20253Q-20254Q-2025
Net income attributable to common stockholders (GAAP) (2)2,2593,2923,6872,5132,7031,8537,9281,3901,4002,1732,1284091,1721,373840
Interest expense445333292838617686929691867685
Provision for (benefit from) income taxes (2)205305276261323167(5,752)483371602381169359570325
Depreciation, amortization and impairment9229569891,0461,1541,2351,2321,2461,2781,3481,4961,4471,4331,6251,643
Stock-based compensation expense361362419418445465484524439457579573635663954
Digital assets loss (gain), net (2)170—34————(335)100(7)(347)125(284)(80)307
Adjusted EBITDA (non-GAAP) (2) (3)3,9614,9685,4384,2674,6533,7583,9533,3843,6744,6654,3332,8143,4014,2274,154
", + "path": "tables/table-13 Financial Data 2022-25.html", + "metadata": { + "length": 3084, + "summary": "table-14\nTable shows quarterly financials from 2Q-2022 to 4Q-2025 in millions USD, including operating cash flow, capex, free cash flow, net income, and adjusted EBITDA.", + "page_nums": [ + 27 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-13 Financial Data 2022-25.html", + "keywords": [ + "cash flow", + "EBITDA", + "net income" + ], + "tokens": [] + } + }, + { + "chunk_id": "6c7a815e-17db-5583-abeb-5ded325f3ce4", + "type": "table", + "content": "
In millions of USD1Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20252Q-20253Q-20254Q-2025
Net cash provided by operating activities – TTM (GAAP)13,24213,95612,16413,25610,98511,53214,47914,92316,83715,76515,74814,747
Capital expenditures – TTM (1)(7,464)(7,794)(8,450)(8,899)(9,603)(9,815)(10,869)(11,342)(10,057)(10,179)(8,914)(8,527)
Free cash flow – TTM (non-GAAP) (1)5,7786,1623,7144,3571,3821,7173,6103,5816,7805,5866,8346,220
In millions of USD1Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20252Q-20253Q-20254Q-20
Net income attributable to common stockholders – TTM (GAAP) (2)11,75112,19510,75614,99713,87412,57112,8917,0916,1105,8825,0823,794
Interest expense – TTM159143128156203261315350365365349338
Provision for (benefit from) income taxes – TTM (2)1,0471,1651,027(5,001)(4,779)(4,731)(4,296)1,8371,5231,5111,4791,423
Depreciation, amortization and impairment – TTM3,9134,1454,4244,6674,8674,9915,1045,3685,5695,7246,0016,148
Stock-based compensation expense – TTM1,5601,6441,7471,8121,9181,9121,9041,9992,0482,2442,4502,825
Digital assets loss (gain), net – TTM (2)2043434—(335)(235)(242)(589)(129)(513)(586)68
Adjusted EBITDA – TTM (non-GAAP) (2) (3)18,63419,32618,11616,63115,74814,76915,67616,05615,48615,21314,77514,596
", + "path": "tables/table-14 Financial Metrics 2023-25.html", + "metadata": { + "length": 2886, + "summary": "table-15\nTable shows quarterly financial data from 1Q 2023 to 4Q 2025, including operating cash flow, capital expenditures, free cash flow, net income, and adjusted EBITDA in millions of USD.", + "page_nums": [ + 27 + ], + "document_top_summary": "This document includes:", + "file_path": "tables/table-14 Financial Metrics 2023-25.html", + "keywords": [ + "cash flow", + "EBITDA", + "net income" + ], + "tokens": [] + } + }, + { + "chunk_id": "15c18264-8e1e-5db6-b3c9-70cd181d1f39", + "type": "text", + "content": "STATEMENT OF OPERATIONS\n(Unaudited)\n\n[tables/table-8 Q4 2024-Q4 2025 Rev.html]\n\nBALANCE SHEET\n(Unaudited)\n\n[tables/table-9 Balance Sheet 2024-25.html]\n\nSTATEMENT OF CASH FLOWS\n(Unaudited)\n\n[tables/table-10 Cash Flow Q4-25.html]\n\nRECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION (Unaudited)\n\n[tables/table-11 Q4 2024-Q4 2025.html]\n\nRECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION\n(Unaudited)\n\n[tables/table-12 Financial Metrics 2021-25.html]\n\nRECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION\n(Unaudited)\n\n[tables/table-13 Financial Data 2022-25.html]\n\n\n[tables/table-14 Financial Metrics 2023-25.html]\n\nTTM = Trailing twelve months\n(1) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted.\n(2) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.\n(3) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->FINANCIAL STATEMENTS", + "metadata": { + "length": 1418, + "summary": "", + "page_nums": [ + 27, + 34 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "STATEMENT", + "OF", + "OPERATIONS", + "Unaudited", + "BALANCE", + "SHEET", + "CASH", + "FLOWS", + "RECONCILIATION", + "GAAP", + "TO", + "NON", + "FINANCIAL", + "INFORMATION", + "TTM", + "Trailing", + "twelve", + "months", + "Beginning", + "Q1", + "25", + "Capital", + "expenditures", + "presented", + "inclusive", + "purchases", + "energy", + "generation", + "storage", + "systems", + "prior", + "periods", + "adjusted", + "As", + "result", + "adoption", + "crypto", + "assets", + "standard", + "previously", + "reported", + "quarterly", + "2024", + "recast", + "Adjusted", + "EBITDA", + "net", + "digital", + "gains", + "losses" + ], + "keywords": [], + "connect_to": [ + { + "target": "3ff046c6-93f4-5d22-a6af-11ecf2660ce3", + "relation": "embeds", + "ref": "[tables/table-8 Q4 2024-Q4 2025 Rev.html]", + "position": { + "start": 37, + "end": 78 + } + }, + { + "target": "dc7fe1d0-57ac-514a-a80a-c46825591b4c", + "relation": "embeds", + "ref": "[tables/table-9 Balance Sheet 2024-25.html]", + "position": { + "start": 107, + "end": 150 + } + }, + { + "target": "0bbfd678-18cc-5152-8775-a4c805e47d57", + "relation": "embeds", + "ref": "[tables/table-10 Cash Flow Q4-25.html]", + "position": { + "start": 189, + "end": 227 + } + }, + { + "target": "675abba3-3bcf-5b5d-94dd-a9e9a711e78b", + "relation": "embeds", + "ref": "[tables/table-11 Q4 2024-Q4 2025.html]", + "position": { + "start": 299, + "end": 337 + } + }, + { + "target": "9a6a2ecd-f489-5cf2-971a-acb1610cbef8", + "relation": "embeds", + "ref": "[tables/table-12 Financial Metrics 2021-25.html]", + "position": { + "start": 409, + "end": 457 + } + }, + { + "target": "2ca08ebb-8851-5594-a629-470029525540", + "relation": "embeds", + "ref": "[tables/table-13 Financial Data 2022-25.html]", + "position": { + "start": 529, + "end": 574 + } + }, + { + "target": "6c7a815e-17db-5583-abeb-5ded325f3ce4", + "relation": "embeds", + "ref": "[tables/table-14 Financial Metrics 2023-25.html]", + "position": { + "start": 577, + "end": 625 + } + } + ] + } + }, + { + "chunk_id": "31babb3c-1abf-5f64-90c7-e8e8a5384bbd", + "type": "text", + "content": "Tesla will provide a live webcast of its fourth quarter 2025 financial results conference call beginning at 4:30 p.m. CT on January 28, 2026 at ir.tesla.com. This webcast will also be available for replay for approximately one year thereafter.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->WEBCAST INFORMATION", + "metadata": { + "length": 243, + "summary": "", + "page_nums": [ + 34 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Tesla", + "provide", + "live", + "webcast", + "fourth", + "quarter", + "2025", + "financial", + "results", + "conference", + "call", + "beginning", + "30", + "CT", + "January", + "28", + "2026", + "ir", + "tesla", + "This", + "replay", + "approximately", + "year" + ], + "keywords": [], + "connect_to": [] + } + }, + { + "chunk_id": "19e3f236-d15a-5b2a-8589-43d28c5316fe", + "type": "text", + "content": "When used in this update, certain terms have the following meanings. Our vehicle deliveries include only vehicles that have been transferred to end customers with all paperwork correctly completed. Our energy product deployment volume includes both customer units when installed and equipment sales at time of delivery. \"Net income attributable to common stockholders (non-GAAP)\" is equal to (i) net income attributable to common stockholders before (ii)(a) stock-based compensation expense, net of tax, (b) digital assets (gain) loss, net of tax and (c) release of valuation allowance on deferred tax assets. \"Adjusted EBITDA (non-GAAP)\" is equal to (i) net income attributable to common stockholders before (ii)(a) interest expense, (b) provision for (benefit from) income taxes, (c) depreciation, amortization and impairment, (d) stock-based compensation expense and (e) digital assets loss (gain), net. \"Free cash flow\" is operating cash flow less capital expenditures. Average cost per vehicle is cost of automotive sales divided by new vehicle deliveries (excluding operating leases). \"Days sales outstanding\" is equal to (i) average accounts receivable, net for the period divided by (ii) total revenues and multiplied by (iii) the number of days in the period. \"Days payable outstanding\" is equal to (i) average accounts payable for the period divided by (ii) total cost of revenues and multiplied by (iii) the number of days in the period. \"Days of supply\" is calculated by dividing new car ending inventory by the relevant period's deliveries and using trading days. Constant currency impacts are calculated by comparing actuals against current results converted into USD using average exchange rates from the prior period.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->CERTAIN TERMS", + "metadata": { + "length": 1733, + "summary": "This passage defines key financial and operational terms used in a specific update. It clarifies that vehicle deliveries refer to units transferred to end customers with completed paperwork, while energy product deployment includes installed customer units and equipment sales. The text details non-GAAP measures: 'Net income attributable to common stockholders' adjusts for stock-based compensation, digital asset gains/losses, and valuation allowances; 'Adjusted EBITDA' excludes interest, taxes, depreciation, amortization, impairment, stock-based compensation, and digital asset impacts. 'Free cash flow' is defined as operating cash flow minus capital expenditures. Operational metrics include 'Average cost per vehicle' (automotive sales cost divided by new deliveries excluding leases), 'Days sales outstanding' (average receivables divided by revenue times days in period), 'Days payable outstanding' (average payables divided by cost of revenues times days in period), and 'Days of supply' (ending inventory divided by deliveries using trading days). Finally, constant currency impacts are calculated by comparing actuals against results converted to USD using prior period average exchange rates.", + "page_nums": [ + 34 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "When", + "update", + "terms", + "meanings", + "Our", + "vehicle", + "deliveries", + "include", + "vehicles", + "transferred", + "end", + "customers", + "paperwork", + "correctly", + "completed", + "energy", + "product", + "deployment", + "volume", + "includes", + "customer", + "units", + "installed", + "equipment", + "sales", + "time", + "delivery", + "Net", + "income", + "attributable", + "common", + "stockholders", + "GAAP", + "equal", + "net", + "ii", + "stock", + "based", + "compensation", + "expense", + "tax", + "digital", + "assets", + "gain", + "loss", + "release", + "valuation", + "allowance", + "deferred", + "Adjusted", + "EBITDA", + "interest", + "provision", + "benefit", + "taxes", + "depreciation", + "amortization", + "impairment", + "Free", + "cash", + "flow", + "operating", + "capital", + "expenditures", + "Average", + "cost", + "automotive", + "divided", + "excluding", + "leases", + "Days", + "outstanding", + "average", + "accounts", + "receivable", + "period", + "total", + "revenues", + "multiplied", + "iii", + "number", + "days", + "payable", + "supply", + "calculated", + "dividing", + "car", + "ending", + "inventory", + "relevant", + "trading", + "Constant", + "currency", + "impacts", + "comparing", + "actuals", + "current", + "results", + "converted", + "USD", + "exchange", + "rates", + "prior" + ], + "keywords": [ + "non-GAAP metrics", + "vehicle deliveries", + "cash flow" + ], + "connect_to": [] + } + }, + { + "chunk_id": "332edac0-294c-5e5b-8599-a0e52b25e53a", + "type": "text", + "content": "Consolidated financial information has been presented in accordance with GAAP as well as on a non-GAAP basis to supplement our consolidated financial results. Our non-GAAP financial measures include non-GAAP net income (loss) attributable to common stockholders, non-GAAP net income (loss) attributable to common stockholders on a diluted per share basis (calculated using weighted average shares for GAAP diluted net income (loss) attributable to common stockholders), Adjusted EBITDA margin, non-GAAP automotive gross margin and free cash flow. These non-GAAP financial measures also facilitate management's internal comparisons to Tesla's historical performance as well as comparisons to the operating results of other companies. Management believes that it is useful to supplement its GAAP financial statements with this non-GAAP information because management uses such information internally for its operating, budgeting and financial planning purposes. Management also believes that presentation of the non-GAAP financial measures provides useful information to our investors regarding our financial condition and results of operations, so that investors can see through the eyes of Tesla management regarding important financial metrics that Tesla uses to run the business and allowing investors to better understand Tesla's performance. Non-GAAP information is not prepared under a comprehensive set of accounting rules and therefore, should only be read in conjunction with financial information reported under U.S. GAAP when understanding Tesla's operating performance. A reconciliation between GAAP and non-GAAP financial information is provided above.", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->NON-GAAP FINANCIAL INFORMATION", + "metadata": { + "length": 1664, + "summary": "Tesla presents consolidated financial information under both GAAP and non-GAAP standards to supplement its results. Key non-GAAP measures include net income attributable to common stockholders, diluted per share figures, Adjusted EBITDA margin, automotive gross margin, and free cash flow. These metrics aid internal management comparisons with historical performance and other companies, supporting operating, budgeting, and planning activities. Management believes these non-GAAP figures provide investors with a clearer view of Tesla's financial condition and operational results by reflecting the metrics used to run the business. However, since non-GAAP data is not prepared under comprehensive accounting rules, it should be read alongside U.S. GAAP information for a complete understanding of Tesla's performance. A reconciliation between GAAP and non-GAAP data is provided elsewhere.", + "page_nums": [ + 34 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Consolidated", + "financial", + "information", + "presented", + "accordance", + "GAAP", + "basis", + "supplement", + "consolidated", + "results", + "Our", + "measures", + "include", + "net", + "income", + "loss", + "attributable", + "common", + "stockholders", + "diluted", + "share", + "calculated", + "weighted", + "average", + "shares", + "Adjusted", + "EBITDA", + "margin", + "automotive", + "gross", + "free", + "cash", + "flow", + "These", + "facilitate", + "management", + "internal", + "comparisons", + "Tesla", + "historical", + "performance", + "operating", + "companies", + "Management", + "believes", + "statements", + "internally", + "budgeting", + "planning", + "purposes", + "presentation", + "investors", + "condition", + "operations", + "eyes", + "important", + "metrics", + "run", + "business", + "allowing", + "understand", + "Non", + "prepared", + "comprehensive", + "set", + "accounting", + "rules", + "read", + "conjunction", + "reported", + "understanding", + "reconciliation", + "provided" + ], + "keywords": [ + "non-GAAP", + "GAAP", + "financial measures" + ], + "connect_to": [] + } + }, + { + "chunk_id": "34263854-525e-5a28-b9fd-a88d4813756e", + "type": "text", + "content": "Certain statements in this update, including, but not limited to, statements in the “Outlook” section; statements relating to the development, strategy, ramp, production and capacity, demand and market growth, cost, pricing and profitability, investment, deliveries, deployment, availability and other features and improvements and timing of existing and future Tesla products and services and supporting infrastructure; statements regarding operating margin, operating profits, spending and liquidity; and statements regarding expansion, improvements and/or ramp and related timing at our facilities are “forward-looking statements” within the meaning of the Private Securities Litigation Reform Act of 1995. Forward-looking statements are based on assumptions and management’s current expectations, involve certain risks and uncertainties, and are not guarantees. Future results may differ materially from those expressed in any forward-looking statement. The following important factors, without limitation, could cause actual results to differ materially from those in the forward-looking statements: the risk of delays in launching and/or manufacturing our products, services and features cost-effectively; our ability to build and/or grow our products and services, sales, delivery, installation, servicing and charging capabilities and effectively manage this growth; our ability to successfully and timely develop, introduce and scale, as well as our consumers’ demand for, products and services based on artificial intelligence, robotics and automation, electric vehicles, advanced driver assistance systems, and ride-hailing services generally and our vehicles and services specifically; the ability of suppliers to deliver components according to schedules, prices, quality and volumes acceptable to us, and our ability to manage such components effectively; any issues with lithium-ion cells or other components manufactured at our factories; our ability to ramp our factories in accordance with our plans; our ability to procure supply of battery cells, including through our own manufacturing; risks relating to international operations and expansion, including unfavorable and uncertain regulatory, political, economic, tax, tariff, export controls and labor conditions; any failures by Tesla products to perform as expected or if product recalls occur; the risk of product liability claims; competition in the automotive, transportation and energy product and services and robotics markets; our ability to maintain public credibility and confidence in our long-term business prospects; our ability to manage risks relating to our various product financing programs; the status of government and economic incentives for electric vehicles and energy products; our ability to attract, hire and retain key employees and qualified personnel; our ability to maintain the security of our information and production and product systems; our compliance with various regulations and laws applicable to our operations and products, which may evolve from time to time; risks relating to our indebtedness and financing strategies; and adverse foreign exchange movements. More information on potential factors that could affect our financial results is included from time to time in our Securities and Exchange Commission filings and reports, including the risks identified under the section captioned “Risk Factors” in our annual report on Form 10-K filed with the SEC on January 30, 2025 and subsequent quarterly reports on Form 10-Q. Tesla disclaims any obligation to update information contained in these forward-looking statements whether as a result of new information, future events or otherwise.\nTESLA", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->FORWARD-LOOKING STATEMENTS", + "metadata": { + "length": 3711, + "summary": "This passage from Tesla outlines that various statements in the update, particularly regarding future outlooks, product development, production capacity, financial metrics, and facility expansions, constitute forward-looking statements under the Private Securities Litigation Reform Act of 1995. These statements reflect management's current expectations based on assumptions and are subject to risks and uncertainties, meaning actual results may differ materially. The text lists numerous specific risk factors that could impact outcomes, including manufacturing delays, supply chain challenges, regulatory hurdles, competition, product performance issues, employee retention, and foreign exchange fluctuations. Tesla advises investors to consult SEC filings, specifically the Form 10-K filed on January 30, 2025, for detailed risk disclosures. The company explicitly disclaims any obligation to update these forward-looking statements due to new information or future events.", + "page_nums": [ + 34 + ], + "document_top_summary": "This document includes:", + "tokens": [ + "Certain", + "statements", + "update", + "including", + "limited", + "Outlook", + "section", + "relating", + "development", + "strategy", + "ramp", + "production", + "capacity", + "demand", + "market", + "growth", + "cost", + "pricing", + "profitability", + "investment", + "deliveries", + "deployment", + "availability", + "features", + "improvements", + "timing", + "existing", + "future", + "Tesla", + "products", + "services", + "supporting", + "infrastructure", + "operating", + "margin", + "profits", + "spending", + "liquidity", + "expansion", + "related", + "facilities", + "forward", + "meaning", + "Private", + "Securities", + "Litigation", + "Reform", + "Act", + "1995", + "Forward", + "based", + "assumptions", + "management", + "current", + "expectations", + "involve", + "risks", + "uncertainties", + "guarantees", + "Future", + "results", + "differ", + "materially", + "expressed", + "statement", + "The", + "important", + "factors", + "limitation", + "actual", + "risk", + "delays", + "launching", + "manufacturing", + "effectively", + "ability", + "build", + "grow", + "sales", + "delivery", + "installation", + "servicing", + "charging", + "capabilities", + "manage", + "successfully", + "timely", + "develop", + "introduce", + "scale", + "consumers", + "artificial", + "intelligence", + "robotics", + "automation", + "electric", + "vehicles", + "advanced", + "driver", + "assistance", + "systems", + "ride", + "hailing", + "generally", + "specifically", + "suppliers", + "deliver", + "components", + "schedules", + "prices", + "quality", + "volumes", + "acceptable", + "issues", + "lithium", + "ion", + "cells", + "manufactured", + "factories", + "accordance", + "plans", + "procure", + "supply", + "battery", + "international", + "operations", + "unfavorable", + "uncertain", + "regulatory", + "political", + "economic", + "tax", + "tariff", + "export", + "controls", + "labor", + "conditions", + "failures", + "perform", + "expected", + "product", + "recalls", + "occur", + "liability", + "claims", + "competition", + "automotive", + "transportation", + "energy", + "markets", + "maintain", + "public", + "credibility", + "confidence", + "long", + "term", + "business", + "prospects", + "financing", + "programs", + "status", + "government", + "incentives", + "attract", + "hire", + "retain", + "key", + "employees", + "qualified", + "personnel", + "security", + "information", + "compliance", + "regulations", + "laws", + "applicable", + "evolve", + "time", + "indebtedness", + "strategies", + "adverse", + "foreign", + "exchange", + "movements", + "More", + "potential", + "affect", + "financial", + "included", + "Exchange", + "Commission", + "filings", + "reports", + "identified", + "captioned", + "Risk", + "Factors", + "annual", + "report", + "Form", + "10", + "filed", + "SEC", + "January", + "30", + "2025", + "subsequent", + "quarterly", + "disclaims", + "obligation", + "contained", + "result", + "events", + "TESLA" + ], + "keywords": [ + "forward-looking statements", + "risk factors", + "legal disclaimer" + ], + "connect_to": [] + } + } + ] +} \ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/doc_nav.json b/apps/api/app/data/demo_documents/tsla-q4-2025/doc_nav.json new file mode 100644 index 000000000..d27f1ba4c --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/doc_nav.json @@ -0,0 +1,163 @@ +{ + "version": "1.0", + "file_name": "TSLA-Q4-2025-Update.pdf", + "stats": { + "total_chunks": 70, + "text_chunks": 36, + "image_chunks": 19, + "table_chunks": 15, + "max_depth": 1 + }, + "sections": [ + { + "title": "Root", + "path": "Default_Root/TSLA-Q4-2025-Update.pdf-->HIGHLIGHTS", + "level": 1, + "summary": "[tables/table-0 Tesla 2025 Results.html]", + "chunk_count": 36, + "children": [] + } + ], + "resources": { + "images": [ + { + "path": "images/image-1-Capacity Growth Projection.jpg", + "summary": "image-1 The chart illustrates a steady increase in existing capacity from mid-2021 through late 2025, characterized by gradual step-wise growth. A significant surge is projected for the future planned capacity starting in early 2026, reaching levels well above the current trajectory." + }, + { + "path": "images/image-2-FSD Mileage Growth.jpg", + "summary": "image-2 The chart illustrates the projected accumulation of miles driven on Full Self-Driving software over time. It distinguishes between two versions: an older version (V11 and before) represented by a blue area, and a newer version (V12 and beyond) shown in red. While mileage for the older version remains relatively flat, the newer version shows exponential growth starting around early 2024, eventually dominating the total distance traveled by late 2025." + }, + { + "path": "images/image-3-Tesla Silicon Optimization.jpg", + "summary": "image-3 The image displays a high-performance computing chip alongside key performance metrics. It highlights significant improvements in hardened block quantization, memory capacity, and raw compute power compared to previous generations. The overall total improvement is presented as a substantial increase over the AI4 architecture." + }, + { + "path": "images/image-4-Growth Trend 2025.jpg", + "summary": "image-4 The chart illustrates a steady increase in values over time, starting from June 2025 and extending through December 2025. The data shows minimal growth initially, followed by a significant upward trajectory beginning around August, reaching its peak at the end of the year." + }, + { + "path": "images/image-5-Tesla Model Y Driving.jpg", + "summary": "image-5 A sleek silver electric SUV travels along a winding highway through a scenic landscape. The vehicle is captured in motion with blurred surroundings, emphasizing speed against a backdrop of rolling hills and distant mountains under a bright sky." + }, + { + "path": "images/image-6-Red Tesla on Coastal Road.jpg", + "summary": "image-6 A red electric sedan drives along a winding asphalt road carved into a steep, rocky mountainside. The vehicle is captured in motion with a blurred background, emphasizing speed as it navigates the curve. To the right of the road lies a calm body of blue water, while the left side features a rugged cliff face covered in sparse green vegetation under a clear sky." + }, + { + "path": "images/image-7-Tesla Interior Interface.jpg", + "summary": "image-7 The image displays the interior of a Tesla vehicle, focusing on the driver's perspective. A person is interacting with the large central touchscreen display, which shows navigation maps and vehicle controls. The steering wheel features the Tesla logo, and ambient lighting accents are visible along the dashboard. Through the windshield, a modern stone building is seen outside." + }, + { + "path": "images/image-8-Tesla Interior.jpg", + "summary": "image-8 The image displays the driver's perspective inside a Tesla vehicle, featuring a minimalist dashboard with a large central touchscreen. The steering wheel is visible on the left side, and the car appears to be in motion on a city street during daylight hours." + }, + { + "path": "images/image-9-Tesla Cybertruck in Snow.jpg", + "summary": "image-9 A futuristic electric pickup truck is shown driving on a frozen, snow-covered surface. The vehicle features its signature angular design and metallic finish, with snow clinging to the rear bumper and wheel wells. It is set against a backdrop of distant mountains under a twilight sky." + }, + { + "path": "images/image-10-Tesla Semi Trucks.jpg", + "summary": "image-10 Two white electric semi-trucks are parked side-by-side in an outdoor lot. The vehicles feature a futuristic, aerodynamic design with large windshields and distinctive horizontal headlights. They are positioned against a backdrop of industrial buildings and hills under a cloudy sky." + }, + { + "path": "images/image-11-US Lightning Map.jpg", + "summary": "image-11 A map of the United States displays numerous red markers with lightning symbols. These indicators are concentrated heavily along the West Coast, particularly in California, and throughout Texas. Additional clusters appear in the Southeast near Atlanta and scattered locations in the Midwest and Northeast." + }, + { + "path": "images/image-12-Tesla Factory Milestone.jpg", + "summary": "image-12 Factory workers and staff gather for a group photo on an automotive assembly line to celebrate the production of the 900th vehicle. A white car is positioned centrally in front of the crowd, while employees hold silver balloons displaying the number \"900\" to mark this significant manufacturing achievement." + }, + { + "path": "images/image-13-Tesla Factory Milestone.jpg", + "summary": "image-13 A large group of factory workers gathers inside a manufacturing facility to celebrate a significant achievement. Several employees in the front row hold up oversized gold balloons that spell out \"600,000,\" marking a major production milestone for the company. The background reveals an industrial setting filled with machinery and assembly lines." + }, + { + "path": "images/image-14-Vehicle Delivery Trends.jpg", + "summary": "image-14 A bar chart displays quarterly vehicle delivery volumes in millions of units spanning from the first quarter of 2023 through the fourth quarter of 2025. The data illustrates fluctuating delivery figures across the timeline, with values generally ranging between approximately 0.3 and 0.5 million units per quarter." + }, + { + "path": "images/image-15-Quarterly Cash Flow.jpg", + "summary": "image-15 The chart compares operating cash flow and free cash flow across multiple quarters from 2023 through 2025. Blue bars represent operating cash flow, while red bars indicate free cash flow. Operating cash flow remains consistently positive throughout the period, whereas free cash flow fluctuates significantly, including a notable negative value in early 2024." + }, + { + "path": "images/image-16-Financial Performance Chart.jpg", + "summary": "image-16 This bar chart compares Net Income and Adjusted EBITDA across quarterly periods from 2023 through 2025. The blue bars represent Net Income while the red bars indicate Adjusted EBITDA, with values measured in billions of dollars. A significant spike in Net Income is visible during the fourth quarter of 2023, whereas Adjusted EBITDA remains consistently higher than Net Income throughout most of the timeline." + }, + { + "path": "images/image-17-Projected Vehicle Deliveries.jpg", + "summary": "image-17 The bar chart illustrates a forecast of vehicle deliveries in millions of units spanning from the first quarter of 2023 through the fourth quarter of 2025. The data indicates an upward trend starting in early 2023, reaching a peak around late 2023 and continuing at high levels throughout 2024 before showing a slight decline toward the end of the projection period in 2025." + }, + { + "path": "images/image-18-Cash Flow Trends.jpg", + "summary": "image-18 The chart compares operating and free cash flow across multiple quarters from 2023 through 2025. Operating cash flow is consistently higher than free cash flow throughout the timeline, with both metrics showing significant fluctuations over time." + }, + { + "path": "images/image-19-Financial Performance Forecast.jpg", + "summary": "image-19 The chart displays a comparison of Net Income and Adjusted EBITDA over a multi-year period. It features paired bars for each quarter, showing that while earnings fluctuate, the adjusted metric remains consistently higher than net income throughout the timeline." + } + ], + "tables": [ + { + "path": "tables/table-0 Tesla 2025 Results.html", + "summary": "table-1 Tesla reported strong 2025 financials with $4.4B operating income and expanded AI initiatives including Robotaxi and Optimus." + }, + { + "path": "tables/table-1 Q4 2025 Financials.html", + "summary": "table-2 Table shows Tesla's quarterly financials through Q4 2025, including revenues, gross profit, operating income, and free cash flow in millions of dollars." + }, + { + "path": "tables/table-2 Financial Data 2021-25.html", + "summary": "table-3 Table shows financial metrics from 2021 to 2025, including revenues, gross profit, operating income, and free cash flow in millions of dollars." + }, + { + "path": "tables/table-3 Tesla Q4-2025 Data.html", + "summary": "table-4 Table shows Tesla's quarterly production, deliveries, and inventory from Q4 2024 to Q4 2025. Total deliveries dropped 16% YoY in Q4 2025." + }, + { + "path": "tables/table-4 Tesla 2021-2025 Data.html", + "summary": "table-5 Table shows Tesla's production, deliveries, and infrastructure metrics from 2021 to 2025. Total production and deliveries peaked in 2023 then declined by 2025." + }, + { + "path": "tables/table-5 Tesla Production.html", + "summary": "table-6 Tesla operates global facilities in California, Shanghai, Berlin, Texas, and Nevada for Model 3/Y, S/X, Cybertruck, Semi, Megapack, Powerwall, and Optimus." + }, + { + "path": "tables/table-6 Facility Status.html", + "summary": "table-7 Table lists AI training and battery manufacturing facilities. Texas hosts Cortex 1 (production) and Cortex 2 (construction). Nevada and Texas have LFP, 4680, cathode materials, and lithium refining plants in various stages." + }, + { + "path": "tables/table-7 Autonomous Driving Status.html", + "summary": "table-8 Table lists US states and metro areas with autonomous driving status. California SF Bay Area has safety drivers. Texas Austin is ramping unsupervised. Other locations like Dallas, Houston, Phoenix, Miami, Orlando, Tampa, and Las Vegas are scheduled for 1H 2026." + }, + { + "path": "tables/table-8 Q4 2024-Q4 2025 Rev.html", + "summary": "table-9 Financial table showing quarterly revenues, costs, and net income from Q4 2024 to Q4 2025 in millions USD." + }, + { + "path": "tables/table-9 Balance Sheet 2024-25.html", + "summary": "table-10 Financial table showing assets, liabilities, and equity from Dec 2024 to Dec 2025. Total assets grew from 122B to 138B USD." + }, + { + "path": "tables/table-10 Cash Flow Q4-25.html", + "summary": "table-11 Table shows quarterly cash flows from operating, investing, and financing activities in millions USD for Q4 2024 to Q4 2025." + }, + { + "path": "tables/table-11 Q4 2024-Q4 2025.html", + "summary": "table-12 Financial table showing GAAP and non-GAAP metrics for quarters Q4-2024 through Q4-2025, including net income, EPS, adjusted EBITDA, and automotive margins." + }, + { + "path": "tables/table-12 Financial Metrics 2021-25.html", + "summary": "table-13 Table shows financial data from 2021 to 2025 including GAAP and non-GAAP net income, EPS, adjusted EBITDA, and margins." + }, + { + "path": "tables/table-13 Financial Data 2022-25.html", + "summary": "table-14 Table shows quarterly financials from 2Q-2022 to 4Q-2025 in millions USD, including operating cash flow, capex, free cash flow, net income, and adjusted EBITDA." + }, + { + "path": "tables/table-14 Financial Metrics 2023-25.html", + "summary": "table-15 Table shows quarterly financial data from 1Q 2023 to 4Q 2025, including operating cash flow, capital expenditures, free cash flow, net income, and adjusted EBITDA in millions of USD." + } + ] + } +} \ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/full.md b/apps/api/app/data/demo_documents/tsla-q4-2025/full.md new file mode 100644 index 000000000..be7372e31 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/full.md @@ -0,0 +1,333 @@ + +# Q4 and FY 2025 Update + + +Highlights 03 + +Financial Summary 04 + +Operational Summary 06 + +Manufacturing & Hardware 08 + +Supporting Infrastructure 09 + +AI & Software 10 + +Services 11 + +Other Updates 12 + +Outlook 13 + +Photos & Charts 14 + +Key Metrics 24 + +Financial Statements 27 + +Additional Information 34 + + +# HIGHLIGHTS + + +
Profitability$4.4B GAAP operating income in 2025; $1.4B in Q42025 marked a critical year for Tesla as we further expanded our mission and continued our transition from a hardware-centric business to a physical AI company. We laid the foundation for the future of Tesla as we further advanced FSD (Supervised) $^{4}$ , launched our Robotaxi service, began installing production lines for Cybercab and fine-tuned our production-primed Optimus design while expanding our AI training infrastructure.
$3.8B GAAP net income in 2025; $0.8B in Q4
$5.9B non-GAAP net income $^{1}$ in 2025; $1.8B in Q4
CashOperating cash flow of $14.7B in 2025; $3.8B in Q4Our approach to autonomous vehicles and humanoid robots mirrors the way we approached electric vehicles and energy storage – at the system level where we identify the limiting factor and develop bespoke and scalable solutions (batteries, power electronics, inverters, software, AI silicon, etc.) to optimize for cost, functionality, efficiency and safety. Our vertical integration has enabled us to achieve economies of scale in a profitable manner, quickly troubleshoot bottlenecks in production and iteratively optimize our technologies more rapidly than others.
Free cash flow $^{2}$ of $6.2B in 2025; $1.4B in Q4In 2025, we completed the refresh of our vehicle lineup with the launch of the new Model Y, including additional variants. We believe that maintaining an optimized and efficient product portfolio, with a continued focus on high-value features such as long range, best-in-class software and autonomy, is the correct strategy to win the autos market of the future. Similarly, we continued to evolve our energy offerings for commercial, utility and retail customers, as we position ourselves as a supplier of choice for clean, affordable and rapidly deployable energy capacity ahead of expected sustained demand growth for electricity.
$7.5B increase in our cash and investments $^{3}$ in 2025 to $44.1B
OperationsBegan removing safety monitor from our Robotaxis in Austin in JanuaryIn 2026, we will further invest in the infrastructure needed to support clean energy and transport and autonomous robots, including the ramp of six new production lines across vehicle, robots, energy storage and battery manufacturing, while further leveraging our existing factory, charging and service center footprint to support future growth.
Record Q4 & FY'25 energy storage deployments
Record vehicle deliveries in APAC
+ +# SUMMARY + +FINANCIAL SUMMARY +(Unaudited) + +
($ in millions, except percentages and per share data)Q4-2024Q1-2025Q2-2025Q3-2025Q4-2025YoY
Total automotive revenues19,79813,96716,66121,20517,693-11%
Energy generation and storage revenue3,0612,7302,7893,4153,83725%
Services and other revenue2,8482,6383,0463,4753,37118%
Total revenues25,70719,33522,49628,09524,901-3%
Total gross profit4,1793,1533,8785,0545,00920%
Total GAAP gross margin16.3%16.3%17.2%18.0%20.1%386 bp
Operating expenses2,5962,7542,9553,4303,60039%
Income from operations1,5833999231,6241,409-11%
Operating margin6.2%2.1%4.1%5.8%5.7%-50 bp
Adjusted EBITDA (1) (2)4,3332,8143,4014,2274,154-4%
Adjusted EBITDA margin (1) (2)16.9%14.6%15.1%15.0%16.7%-17 bp
Net income attributable to common stockholders (GAAP) (1)2,1284091,1721,373840-61%
Net income attributable to common stockholders (non-GAAP) (1) (3)2,1079341,3931,7701,761-16%
EPS attributable to common stockholders, diluted (GAAP) (1)0.600.120.330.390.24-60%
EPS attributable to common stockholders, diluted (non-GAAP) (1) (3)0.600.270.400.500.50-17%
Net cash provided by operating activities4,8142,1562,5406,2383,813-21%
Capital expenditures (4)(2,780)(1,492)(2,394)(2,248)(2,393)-14%
Free cash flow (4)2,0346641463,9901,420-30%
Cash, cash equivalents and investments36,56336,99636,78241,64744,05921%
+ +(1) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast. +(2) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted. +(3) Beginning in Q1'25, Net income attributable to common stockholders (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted. +(4) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted. + +FINANCIAL SUMMARY +(Unaudited) + +
($ in millions, except percentages and per share data)20212022202320242025YoY
Total automotive revenues47,23271,46282,41977,07069,526-10%
Energy generation and storage revenue2,7893,9096,03510,08612,77127%
Services and other revenue3,8026,0918,31910,53412,53019%
Total revenues53,82381,46296,77397,69094,827-3%
Total gross profit13,60620,85317,66017,45017,094-2%
Total GAAP gross margin25.3%25.6%18.2%17.9%18.0%16 bp
Operating expenses7,0837,1978,76910,37412,73923%
Income from operations6,52313,6568,8917,0764,355-38%
Operating margin12.1%16.8%9.2%7.2%4.6%-265 bp
Adjusted EBITDA (1)11,72219,39016,63116,05614,596-9%
Adjusted EBITDA margin (1)21.8%23.8%17.2%16.4%15.4%-104 bp
Net income attributable to common stockholders (GAAP)5,51912,55614,9977,0913,794-46%
Net income attributable to common stockholders (non-GAAP) (2)7,71914,27610,8827,9605,858-26%
EPS attributable to common stockholders, diluted (GAAP)1.633.624.302.041.08-47%
EPS attributable to common stockholders, diluted (non-GAAP) (2)2.284.123.122.291.66-28%
Net cash provided by operating activities11,49714,72413,25614,92314,747-1%
Capital expenditures (3)(6,514)(7,163)(8,899)(11,342)(8,527)-25%
Free cash flow (3)4,9837,5614,3573,5816,22074%
Cash, cash equivalents and investments17,70722,18529,09436,56344,05921%
+ +(1) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted. +(2) Beginning in Q1'25, Net income attributable to common stockholders (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted. +(3) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted. + +OPERATIONAL SUMMARY +(Unaudited) + +
Q4-2024Q1-2025Q2-2025Q3-2025Q4-2025YoY
Model 3/Y production436,718345,454396,835435,826422,652-3%
Other models production22,72717,16113,40911,62411,706-48%
Total production459,445362,615410,244447,450434,358-5%
Model 3/Y deliveries471,930323,800373,728481,166406,585-14%
Other models deliveries23,64012,88110,39415,93311,642-51%
Total deliveries495,570336,681384,122497,099418,227-16%
of which subject to operating lease accounting26,96213,7216,67010,23010,996-59%
Cumulative $deliveries^{(1)}$ (all-time; mil)7.37.68.08.58.922%
Active FSD $Subscriptions^{(2)}$ (mil)0.80.80.91.01.138%
Total end of quarter operating lease (new vehicle) $count^{(3)}$ 180,523179,930172,882167,163163,075-10%
Global vehicle inventory (days of supply) $^{(4)}$ 122224101525%
Storage deployed (GWh)11.010.49.612.514.229%
Tesla locations1,3591,3901,4541,4981,55314%
Supercharger stations6,9757,1317,3777,7538,18217%
Supercharger connectors65,49567,31670,22873,81777,68219%
+ +OPERATIONAL SUMMARY +(Unaudited) + +
20212022202320242025YoY
Model 3/Y production906,0321,298,4341,775,1591,679,3381,600,767-5%
Other models production24,39071,17770,82694,10553,900-43%
Total production930,4221,369,6111,845,9851,773,4431,654,667-7%
Model 3/Y deliveries911,2421,247,1461,739,7071,704,0931,585,279-7%
Other models deliveries24,98066,70568,87485,13350,850-40%
Total deliveries936,2221,313,8511,808,5811,789,2261,636,129-9%
of which subject to operating lease accounting60,91247,58272,22660,00341,617-31%
Cumulative $deliveries^{(1)}$ (all-time; mil)2.33.75.57.38.922%
Active FSD $Subscriptions^{(2)}$ (mil)0.40.50.60.81.138%
Total end of year operating lease (new vehicle) count120,342140,667176,564180,523163,075-10%
Global vehicle inventory (days of supply) $^{(3)}$ 61616131515%
Storage deployed (GWh)4.06.514.731.446.749%
Tesla locations6449631,2081,3591,55314%
Supercharger stations3,4764,6785,9526,9758,18217%
Supercharger connectors31,49842,41954,89265,49577,68219%
+ + +# Automotive + +While automotive sales declined sequentially, gross margin (even when excluding the impact of regulatory credits) improved. The APAC region continued to show strength across multiple markets and set a record for deliveries in the quarter. We continued the rollout of Model Y variants across markets in Q4, including the standard and performance versions. + +Preparations continue in North America for the production ramps of Tesla Semi and Cybercab, both commencing 1H26, and production of the next-generation Roadster. + +# Energy generation and storage + +We achieved our highest quarterly energy storage deployments, driven by record Megapack deployments. Total gross profit rose, both sequentially and year-over-year, to a record \$1.1 billion, marking the fifth consecutive record quarter. We plan to begin Megapack 3 and Megablock production at Megafactory Houston in 2026. In 2025, our global Powerwall network supported more than 89,000 Virtual Power Plant events across over 1 million installed units, allowing homeowners to save over \$1 billion in electricity bills as Virtual Power Plant participation continues to scale rapidly. + +# Robotics + +We made further progress on the Optimus program in 2025. In Q1 of this year, we plan to unveil the Gen 3 version of Optimus, which will include major upgrades from version 2.5, including our latest hand design. The Gen 3 is our first design meant for mass production. Preparations are underway for the first production line, including supply chain readiness, with start of production planned before the end of 2026 and eventual planned capacity of 1 million robots per year. + +Installed Annual Manufacturing Capacity + +
RegionProductCapacityStatus
Automotive
CaliforniaModel 3 / Model Y>550,000Production
Model S / Model X100,000Production
ShanghaiModel 3 / Model Y>950,000Production
BerlinModel Y>375,000Production
TexasModel Y>250,000Production
Cybertruck>125,000Production
Cybercab-Tooling
NevadaTesla Semi-Tooling
TBDRoadster-Design development
Energy Generation and Storage
CaliforniaMegapack40 GWhProduction
NevadaPowerwall>6 GWhProduction
ShanghaiMegapack40 GWhProduction
TexasMegapack-Construction
Robotics
CaliforniaOptimus-Construction
+ +Installed capacity ≠ current production rate and there may be limitations discovered as production rates approach capacity. Production rates depend on a variety of factors, including equipment uptime, component supply, downtime related to factory upgrades, regulatory considerations and other factors. Construction includes factory and infrastructure buildout as well as tool installation. + + +# AI Training Compute + +We are currently building Cortex 2 at Gigafactory Texas to further increase our AI training compute capacity. In the first half of 2026, we plan to more than double the size of onsite compute in Texas (in terms of H100 equivalents). We aim to maximize capital efficiency by scaling training compute judiciously, including when the training backlog gets too long or in anticipation of greater demand from our engineers to support our AI-related offerings. + +# Battery + +Our lithium refinery commenced pilot production and is the first spodumene to lithium hydroxide refinery in North America, leveraging a simpler, cheaper and more environmentally friendly process. This refinery enables us to domestically produce critical minerals in support of energy storage, battery manufacturing and ultimately for EV growth. + +We have begun to produce battery packs for certain Model Ys with our 4680 cells, unlocking an additional vector of supply to help navigate increasingly complex supply chain challenges caused by trade barriers and tariff risks. We now produce dry-electrode for 4680 cells with both anode and cathode made in Austin. We expect both domestic cathode material in Texas and LFP lines in Nevada to begin production in 2026. + +# Other Supporting Infrastructure + +We continue to efficiently utilize our existing physical footprint in North America, with targeted augmentation to support the rollout of Robotaxi. While in the short-term, operational workstreams such as charging, cleaning and maintenance can be managed through our existing charging network, service centers and sales and delivery locations, we will have to add more capacity as the service expands. We added over 3,800 net new Supercharging stalls, growing the network 19% year-over-year. + +Installed Annual Capacity + +
RegionProductCapacityStatus
AI Training Compute
TexasCortex 1>100k H100eProduction
Cortex 2-Construction
Battery Manufacturing
NevadaLFP7 GWhEarly Ramp
Texas468040 GWhProduction
Cathode Materials10 GWhEarly Ramp
Lithium Refining30 GWhEarly Ramp
+ +Installed capacity ≠ current production rate and there may be limitations discovered as production rates approach capacity. Production rates depend on a variety of factors, including equipment uptime, component supply, downtime related to factory upgrades, regulatory considerations and other factors. Construction includes factory and infrastructure buildout as well as tool installation. + +![](images/c5419b096d319ebffb59e266e803a4f96d687486c0d5acf1ddbd4ac05c322704.jpg) +Tesla AI Training Capacity Ramp (H100 equivalent GPUs) + + +# AI Software + +We continue to enhance FSD (Supervised) $^{1}$ via our end-to-end foundation model trained on both customer and Robotaxi real-world data with our latest version, v14. FSD (Supervised) $^{1}$ increasingly provides safety and convenience functionality that can relieve drivers of many tedious and potentially dangerous aspects of road travel, including giving access to personal transport for those who otherwise have difficulty driving. V14 offers unparalleled driver assistance to safely drive to the customer's destination, find a free parking spot and park at that location. Our global fleet can collect the equivalent of over 500 years of continuous driving data per day $^{2}$ , allowing us to safely deploy and scale capabilities that can handle the long and fat tail of corner cases across diverse geographies and driving environments. + +# AI Inference Compute + +Development of our in-house, custom designed AI5 and AI6 inference chips for autonomy progressed during the quarter, with production planned for 2027 and 2028, respectively. We are targeting a 50x improvement in performance for AI5 relative to AI4 thanks to 10x raw compute, 9x memory capacity and 5x hardened block quantization and softmax function (the latter enabling efficient low-precision computing without sacrificing model accuracy). + +# Automotive and Other Software + +The Robotaxi iOS app no longer has a waitlist in the areas we serve. Our vehicles keep getting better with our over-the-air updates, including: Grok (an AI companion) which now supports navigation commands (allowing users to find, add and edit navigation destinations hands-free); Tesla Photobooth which enables users to take photos in their car and download or share via the Tesla mobile app; Supercharger Site Maps which displays Supercharger layouts, nearby businesses and live availability details; Automatic HOV Lane Routing based on interior camera occupancy detection; Phone Left Behind Chime and SpaceX ISS Docking Simulator Game. + +![](images/9f4063992747fdc3132e661986a68e701bee9186c21a5a3fde658ed89f9e81e6.jpg) + +Cumulative Miles Driven with FSD (Supervised) $^{1}$ (billions) +![](images/48fc2d9f835f11c1f7727e32e63c14619355a55be1227aa504a3bc4c8fb80df7.jpg) +Targeting Step-Function Improvement for our Next-Generation Inference Chip, AI5 + +(1) Active driver supervision required; does not make the vehicle autonomous +(2) Calculated based on continuous hours of driving at an average of 30 miles per hour + +# Robotaxi + +We began testing driverless Robotaxis in Austin in December and began removing the safety monitor from customer rides in January on a limited basis, which will unlock further expansion of our Robotaxi fleet and coverage area in the Austin-metro. Our Bay Area ride-hailing service began serving the San Jose Airport in October, with plans to expand to other major airports in the Bay Area upon receiving required permitting. + +# FSD (Supervised) $^{1}$ + +We launched FSD (Supervised) $^{1}$ in South Korea, where customers drove over 1 million kilometers using the software in just one month. While we continue to pursue regulatory approval in China and Europe, we began offering ride-along experiences to consumers in Italy, Germany, France and Switzerland. + +Monthly subscriptions to FSD (Supervised) $^{1}$ continued to grow sequentially and more than doubled in 2025. Starting this quarter, we are transitioning access to FSD (Supervised) $^{1}$ to monthly subscriptions only as we begin to sunset the up-front payment option. + +# Automotive Services + +Services and Other gross profit of approximately \$300 million was partly driven by Part Sales and Supercharging. We now offer Tesla Insurance in Florida, as we continue to expand our insurance product to new states. In certain states, customers receive a discount on their insurance premiums when using FSD (Supervised) $^{1}$ . The more you drive with FSD (Supervised) $^{1}$ enabled, the bigger the discount is on your insurance premium – helping, in certain cases, to completely offset the monthly subscription cost for FSD (Supervised) $^{1}$ . + +![](images/0988c8ad6fe18ff5072a2abf6a77a630f3902fcc1d80846fbf065e7811b99e87.jpg) + +Cumulative Paid Robotaxi Miles + +
StateMetroStatus
CaliforniaSF Bay AreaSafety Driver
TexasAustinRamping Unsupervised
Dallas1H 2026
Houston1H 2026
ArizonaPhoenix1H 2026
FloridaMiami1H 2026
Orlando1H 2026
Tampa1H 2026
NevadaLas Vegas1H 2026
+ +Planned Robotaxi Coverage + + +# OTHER UPDATES + +On January 16, 2026, Tesla entered into an agreement to invest approximately \$2 billion to acquire shares of Series E Preferred Stock of xAI as part of their recent publicly-disclosed financing round. Tesla’s investment was made on market terms consistent with those previously agreed to by other investors in the financing round. As set forth in Master Plan Part IV, Tesla is building products and services that bring AI into the physical world. Meanwhile, xAI is developing leading digital AI products and services, such as its large language model (Grok). + +In that context, and as part of Tesla's broader strategy under Master Plan Part IV, Tesla and xAI also entered into a framework agreement in connection with the investment. Among other things, the framework agreement builds upon the existing relationship between Tesla and xAI by providing a framework for evaluating potential AI collaborations between the companies. Together, the investment and the related framework agreement are intended to enhance Tesla's ability to develop and deploy AI products and services into the physical world at scale. This investment is subject to customary regulatory conditions with the expectation to close in Q1'2026. + + +# OUTLOOK + +# Volume + +We are focused on maximum capacity utilization at our factories. Deliveries and deployments will be impacted by aggregate demand for our products, supply chain readiness and allocation decisions between sale to customers or use for our owned and operated fleet. + +# Cash + +We will manage the businesses such that we ensure a strong balance sheet, maintaining sufficient liquidity to fund our product roadmap, long-term capacity expansion plans – including further vertical integration – and other expenses. + +# Profit + +While we continue to execute on innovations to reduce the cost of manufacturing and operations, over time, we expect our hardware-related profits to be accompanied by an acceleration of AI, software and fleet-based profits. + +# Product + +We continue to evolve and augment our product lineup with a focus on cost, scale and future monetization opportunities via services powered by our AI software. We remain focused on growing our sales volumes through a differentiated and efficiently managed product portfolio, which includes leveraging and optimizing our existing production capacity before building new factories and production lines. + +Cybercab, Tesla Semi and Megapack 3 are on schedule for volume production starting in 2026. First generation production lines for Optimus are being installed in anticipation of volume production. + + +PHOTOS & CHARTS + + +# MODEL Y - 2025 BEST IN CLASS EURO NCAP $^{(1)}$ SMALL SUV + +![](images/158a8cd9950de369e065221dc0e852dbc1e830dc0f883a496bd9be77d6029db3.jpg) + + +# MODEL 3 - 2025 BEST IN CLASS EURO NCAP $^{(1)}$ LARGE FAMILY CAR + +![](images/07ce9a96f3e5d92fa1641b00312412b3deaf22189aa8904659176fa2bcd62e75.jpg) + + +# FSD (SUPERVISED) $^{1}$ – V14 OFFERS UNPARALLELED DRIVER ASSISTANCE + +![](images/b1f1448eab61493236c1530fcc2c05a7365b9ef91af9be38d3faf74ec71b5ae9.jpg) + + +# DRIVERLESS ROBOTAXI - TESTING IN AUSTIN + +![](images/cda4f95399f8a58b55750a199bb9434e1b7806478b1ca775e4c7c0737656b5de.jpg) + + +# CYBERCAB - COLD WEATHER TESTING IN ALASKA + +![](images/d4e7263d8fb8951d59a0c80292c412f4b75d36fb36a35fe058140095dddfbce1.jpg) + +![](images/ade7e435748c90f4f859d0d36b9856e2ba25bcb3de67e173f6a38e70039d9c2c.jpg) + + +TESLA SEMI - MEGACHARGER NETWORK PLANNED SITES FOR 2026 +![](images/8e2b663e678f56be153dc891d3913f90ed7e90060ca0b164434d28336d2c8820.jpg) + + +# GIGAFACTORY SHANGHAI - 9 MILLIONTH VEHICLE PRODUCED (GLOBALLY) + +![](images/0ec324caa888be92ff136948d030c18f5dba98b8d91fcdfe7a3a6f6f5b8660af.jpg) + + +# GIGAFACTORY NEVADA - 6 MILLIONTH DRIVE UNIT PRODUCED + +![](images/960786f1a7ef3448079eafcc69ce08cef09109fa0577b30c08f8fda23c394483.jpg) + +![](images/920fc64098ae7b1ce10f289ec9513682e5581ac225922fd4b75ba9c45125a352.jpg) + +![](images/1e7cdefaaf215c38ac82a80e892bd3bb9f87b0b2dfc423b6ec2e3525fb99d11f.jpg) + +![](images/afc60e5cf57f455cd6fa8ffafb95cb736170fcaef6b76d4e10c394bf81cb7849.jpg) + + +# KEY METRICS TRAILING 12 MONTHS (TTM) (Unaudited) + +![](images/12044b68a9c0ec6bd1c5e10a65a295f2ca643bce67aff18015086f71bfc4a5e7.jpg) + +![](images/a701d35afba0dcc13ebd0a273857f53b6c616039fae2341a2b927075ac37f64e.jpg) + +![](images/ff5078d01c1a41d45496ab1b851b69109387b3f2b158681ce7b326707f6716ba.jpg) + + +# Revenue + +Total quarterly revenue decreased 3% YoY to \$24.9B. YoY, revenue was impacted by the following items $^{(1)}$ : + +- decrease in vehicle deliveries +- lower regulatory credit revenue ++ growth in Energy Generation and Storage ++ growth in Services and Other ++ positive FX impact of \$0.3B $^{1}$ + +\+ growth in other automotive ancillary sales, partly driven by an increase in FSD subscriptions + +\+ higher vehicle average selling price (ASP) (excl. FX impact $^{1}$ ), inclusive of mix impact + +# Profitability + +Our quarterly operating income decreased 11% YoY to \$1.4B, resulting in a 5.7% operating margin. YoY, operating income was primarily impacted by the following items $^{(1)}$ : + +- increase in SBC and Restructuring and Other charges +- increase in operating expenses (excl. SBC and Restructuring and Other) driven by AI and other R&D projects and SG&A +- higher average cost per vehicle due to lower fixed cost absorption for certain models and an increase in tariffs +- decrease in vehicle deliveries +- lower regulatory credit revenue ++ higher vehicle average gross profit due to mix and pricing impacts ++ growth in Energy Generation and Storage gross profit ++ growth in Services and Other gross profit ++ growth in other automotive ancillary sales, partly driven by an increase in FSD subscriptions + +# Cash + +Quarter-end cash, cash equivalents and investments was \$44.1B. The sequential increase of \$2.4B was primarily the result of positive free cash flow. + + +# FINANCIAL STATEMENTS + +STATEMENT OF OPERATIONS +(Unaudited) + +
In millions of USD or shares as applicable, except per share dataQ4-2024Q1-2025Q2-2025Q3-2025Q4-2025
REVENUES
Automotive sales18,65912,92515,78720,35916,750
Automotive regulatory credits692595439417542
Automotive leasing447447435429401
Total automotive revenues19,79813,96716,66121,20517,693
Energy generation and storage3,0612,7302,7893,4153,837
Services and other2,8482,6383,0463,4753,371
Total revenues25,70719,33522,49628,09524,901
COST OF REVENUES
Automotive sales16,26811,46113,56717,36513,874
Automotive leasing242239228225206
Total automotive cost of revenues16,51011,70013,79517,59014,080
Energy generation and storage2,2891,9451,9432,3422,739
Services and other2,7292,5372,8803,1093,073
Total cost of revenues21,52816,18218,61823,04119,892
Gross profit4,1793,1533,8785,0545,009
OPERATING EXPENSES
Research and development1,2761,4091,5891,6301,783
Selling, general and administrative1,3131,2511,3661,5621,655
Restructuring and other794—238162
Total operating expenses2,5962,7542,9553,4303,600
INCOME FROM OPERATIONS1,5833999231,6241,409
Interest income442400392439449
Interest expense(96)(91)(86)(76)(85)
Other income (expense), net (1)595(119)320(28)(592)
INCOME BEFORE INCOME TAXES (1)2,5245891,5491,9591,181
Provision for income taxes (1)381169359570325
NET INCOME (1)2,1434201,1901,389856
Net income attributable to noncontrolling interests and redeemable noncontrolling interests in subsidiaries1511181616
NET INCOME ATTRIBUTABLE TO COMMON STOCKHOLDERS (1)2,1284091,1721,373840
Less: Buy-out of noncontrolling interest3————
NET INCOME USED IN COMPUTING NET INCOME PER SHARE OF COMMON STOCK (1)2,1254091,1721,373840
Net income per share of common stock attributable to common stockholders
Basic (1)$ 0.66$ 0.13$ 0.36$ 0.43$ 0.26
Diluted (1)$ 0.60$ 0.12$ 0.33$ 0.39$ 0.24
Weighted average shares used in computing net income per share of common stock
Basic3,2133,2183,2233,2273,231
Diluted3,5173,5213,5193,5263,539
+ +BALANCE SHEET +(Unaudited) + +
In millions of USD31-Dec-2431-Mar-2530-Jun-2530-Sep-2531-Dec-25
ASSETS
Current assets
Cash, cash equivalents and investments36,56336,99636,78241,64744,059
Accounts receivable, net4,4183,7823,8384,7034,576
Inventory12,01713,70614,57012,27612,392
Prepaid expenses and other current assets5,3624,9055,9436,0277,615
Total current assets58,36059,38961,13364,65368,642
Operating lease vehicles, net5,5815,4775,2305,0194,912
Energy generation and storage systems, net4,9244,8554,7884,6734,604
Property, plant and equipment, net35,83637,08838,57439,40740,643
Operating lease right-of-use assets5,1605,3305,6335,7836,027
Digital assets (2)1,0769511,2351,3151,008
Deferred tax assets (2)6,5246,6876,7216,6376,925
Other non-current assets4,6095,3345,2536,2485,045
Total assets (2)122,070125,111128,567133,735137,806
LIABILITIES AND EQUITY
Current liabilities
Accounts payable12,47413,47113,21212,81913,371
Accrued liabilities and other10,72310,80211,51912,79113,279
Deferred revenue3,1683,2433,2373,7563,424
Current portion of debt and finance leases (1)2,4562,2372,0401,9241,640
Total current liabilities28,82129,75330,00831,29031,714
Debt and finance leases, net of current portion (1)5,7575,2925,1805,7786,736
Deferred revenue, net of current portion3,3173,6103,7643,7463,631
Other long-term liabilities10,49511,03811,54312,20512,860
Total liabilities48,39049,69350,49553,01954,941
Redeemable noncontrolling interests in subsidiaries6362615958
Total stockholders' equity (2)72,91374,65377,31479,97082,137
Noncontrolling interests in subsidiaries704703697687670
Total liabilities and equity (2)122,070125,111128,567133,735137,806
(1) Breakdown of our debt is as follows:
Non-recourse debt7,8717,2386,9537,4588,150
Recourse debt76333
Days sales outstanding1419151417
Days payable outstanding5872655261
+ +STATEMENT OF CASH FLOWS +(Unaudited) + +
In millions of USDQ4-2024Q1-2025Q2-2025Q3-2025Q4-2025
CASH FLOWS FROM OPERATING ACTIVITIES
Net income (1)2,1434201,1901,389856
Adjustments to reconcile net income to net cash provided by operating activities:
Depreciation, amortization and impairment1,4961,4471,4331,6251,643
Stock-based compensation579573635663954
Deferred income taxes (1)6(43)52225(111)
Digital assets (gain) loss, net (1)(347)125(284)(80)307
Other(93)188187333378
Changes in operating assets and liabilities1030(554)(673)2,083(214)
Net cash provided by operating activities4,8142,1562,5406,2383,813
CASH FLOWS FROM INVESTING ACTIVITIES
Capital expenditures (2)(2,780)(1,492)(2,394)(2,248)(2,393)
Purchases of investments(15,158)(6,015)(7,485)(11,402)(12,207)
Proceeds from maturities of investments10,3355,8566,9359,2958,072
Net cash used in investing activities(7,603)(1,651)(2,944)(4,355)(6,528)
CASH FLOWS FROM FINANCING ACTIVITIES
Net cash flows from other debt activities(108)(50)(23)410963
Net borrowings (repayments) under vehicle and energy product financing677(674)(400)81(377)
Net cash flows from noncontrolling interests – Solar(37)(22)(14)(20)(22)
Other453414215512146
Net cash provided by (used in) financing activities985(332)(222)983710
Effect of exchange rate changes on cash and cash equivalents and restricted cash(133)40111(17)37
Net (decrease) increase in cash and cash equivalents and restricted cash(1,937)213(515)2,849(1,968)
Cash and cash equivalents and restricted cash at beginning of period18,97417,03717,25016,73519,584
Cash and cash equivalents and restricted cash at end of period17,03717,25016,73519,58417,616
+ +RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION (Unaudited) + +
In millions of USD or shares as applicable, except per share dataQ4-2024Q1-2025Q2-2025Q3-2025Q4-2025
Net income attributable to common stockholders (GAAP) (1)2,1284091,1721,373840
Stock-based compensation expense, net of tax249428443459682
Digital assets (gain) loss, net of tax (1)(270)97(222)(62)239
Net income attributable to common stockholders (non-GAAP) (1) (2)2,1079341,3931,7701,761
Less: Buy-outs of noncontrolling interests3————
Net income used in computing diluted EPS attributable to common stockholders (non-GAAP) (1) (2)2,1049341,3931,7701,761
EPS attributable to common stockholders, diluted (GAAP) (1)0.600.120.330.390.24
Stock-based compensation expense, net of tax, per share0.080.120.130.130.19
Digital assets (gain) loss, net of tax, per share (1)(0.08)0.03(0.06)(0.02)0.07
EPS attributable to common stockholders, diluted (non-GAAP) (1) (2)0.600.270.400.500.50
Shares used in EPS calculation, diluted (GAAP and non-GAAP)3,5173,5213,5193,5263,539
Net income attributable to common stockholders (GAAP) (1)2,1284091,1721,373840
Interest expense9691867685
Provision for income taxes (1)381169359570325
Depreciation, amortization and impairment1,4961,4471,4331,6251,643
Stock-based compensation expense579573635663954
Digital assets (gain) loss, net (1)(347)125(284)(80)307
Adjusted EBITDA (non-GAAP) (1) (3)4,3332,8143,4014,2274,154
Total revenues25,70719,33522,49628,09524,901
Adjusted EBITDA margin (non-GAAP) (1) (3)16.9%14.6%15.1%15.0%16.7%
Automotive gross margin (GAAP)16.6%16.2%17.2%17.0%20.4%
Less: Total regulatory credit revenue recognized3.0%3.7%2.2%1.6%2.5%
Automotive gross margin excluding regulatory credit sales (non-GAAP)13.6%12.5%15.0%15.4%17.9%
+ +RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION +(Unaudited) + +
In millions of USD or shares as applicable, except per share data20212022202320242025
Net income attributable to common stockholders (GAAP)5,51912,55614,9977,0913,794
Stock-based compensation expense, net of tax2,1211,5601,8121,3282,012
Digital assets loss (gain), net of tax79160—(459)52
Release of valuation allowance on deferred tax assets——(5,927)——
Net income attributable to common stockholders (non-GAAP)(1)7,71914,27610,8827,9605,858
Less: Buy-outs of noncontrolling interests(5)(27)(2)(39)—
Less: Dilutive convertible debt(9)(1)———
Net income used in computing diluted EPS attributable to common stockholders (non-GAAP)(1)7,73314,30410,8847,9995,858
EPS attributable to common stockholders, diluted (GAAP)1.633.624.302.041.08
Stock-based compensation expense, net of tax, per share0.630.450.520.380.57
Digital assets loss (gain), net of tax, per share0.020.05—(0.13)0.01
Release of valuation allowance on deferred tax assets——(1.70)——
EPS attributable to common stockholders, diluted (non-GAAP)(1)2.284.123.122.291.66
Shares used in EPS calculation, diluted (GAAP and non-GAAP)3,3863,4753,4853,4983,528
Net income attributable to common stockholders (GAAP)5,51912,55614,9977,0913,794
Interest expense371191156350338
Provision for (benefit from) income taxes6991132(5,001)1,8371,423
Depreciation, amortization and impairment2,9113,7474,6675,3686,148
Stock-based compensation expense2,1211,5601,8121,9992,825
Digital assets loss (gain), net101204—(589)68
Adjusted EBITDA (non-GAAP)(2)11,72219,39016,63116,05614,596
Total revenues53,82381,46296,77397,69094,827
Adjusted EBITDA margin (non-GAAP)(2)21.8%23.8%17.2%16.4%15.4%
Automotive gross margin (GAAP)29.3%28.5%19.4%18.4%17.8%
Less: Total regulatory credit revenue recognized2.3%1.8%1.7%3.0%2.4%
Automotive gross margin excluding regulatory credit sales (non-GAAP)27.0%26.7%17.7%15.4%15.4%
+ +RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION +(Unaudited) + +
In millions of USD2Q-20223Q-20224Q-20221Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20252Q-20253Q-20254Q-2025
Net cash provided by operating activities (GAAP)2,3515,1003,2782,5133,0653,3084,3702423,6126,2554,8142,1562,5406,2383,813
Capital expenditures (1)(1,730)(1,803)(1,858)(2,073)(2,060)(2,459)(2,307)(2,777)(2,272)(3,513)(2,780)(1,492)(2,394)(2,248)(2,393)
Free cash flow (non-GAAP) (1)6213,2971,4204401,0058492,063(2,535)1,3402,7422,0346641463,9901,420
In millions of USD2Q-20223Q-20224Q-20221Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20 252Q-20253Q-20254Q-2025
Net income attributable to common stockholders (GAAP) (2)2,2593,2923,6872,5132,7031,8537,9281,3901,4002,1732,1284091,1721,373840
Interest expense445333292838617686929691867685
Provision for (benefit from) income taxes (2)205305276261323167(5,752)483371602381169359570325
Depreciation, amortization and impairment9229569891,0461,1541,2351,2321,2461,2781,3481,4961,4471,4331,6251,643
Stock-based compensation expense361362419418445465484524439457579573635663954
Digital assets loss (gain), net (2)170—34————(335)100(7)(347)125(284)(80)307
Adjusted EBITDA (non-GAAP) (2) (3)3,9614,9685,4384,2674,6533,7583,9533,3843,6744,6654,3332,8143,4014,2274,154
+ +
In millions of USD1Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20252Q-20253Q-20254Q-2025
Net cash provided by operating activities – TTM (GAAP)13,24213,95612,16413,25610,98511,53214,47914,92316,83715,76515,74814,747
Capital expenditures – TTM (1)(7,464)(7,794)(8,450)(8,899)(9,603)(9,815)(10,869)(11,342)(10,057)(10,179)(8,914)(8,527)
Free cash flow – TTM (non-GAAP) (1)5,7786,1623,7144,3571,3821,7173,6103,5816,7805,5866,8346,220
In millions of USD1Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20252Q-20253Q-20254Q-20
Net income attributable to common stockholders – TTM (GAAP) (2)11,75112,19510,75614,99713,87412,57112,8917,0916,1105,8825,0823,794
Interest expense – TTM159143128156203261315350365365349338
Provision for (benefit from) income taxes – TTM (2)1,0471,1651,027(5,001)(4,779)(4,731)(4,296)1,8371,5231,5111,4791,423
Depreciation, amortization and impairment – TTM3,9134,1454,4244,6674,8674,9915,1045,3685,5695,7246,0016,148
Stock-based compensation expense – TTM1,5601,6441,7471,8121,9181,9121,9041,9992,0482,2442,4502,825
Digital assets loss (gain), net – TTM (2)2043434—(335)(235)(242)(589)(129)(513)(586)68
Adjusted EBITDA – TTM (non-GAAP) (2) (3)18,63419,32618,11616,63115,74814,76915,67616,05615,48615,21314,77514,596
+ +TTM = Trailing twelve months +(1) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted. +(2) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast. +(3) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted. + + +# WEBCAST INFORMATION + +Tesla will provide a live webcast of its fourth quarter 2025 financial results conference call beginning at 4:30 p.m. CT on January 28, 2026 at ir.tesla.com. This webcast will also be available for replay for approximately one year thereafter. + +# CERTAIN TERMS + +When used in this update, certain terms have the following meanings. Our vehicle deliveries include only vehicles that have been transferred to end customers with all paperwork correctly completed. Our energy product deployment volume includes both customer units when installed and equipment sales at time of delivery. "Net income attributable to common stockholders (non-GAAP)" is equal to (i) net income attributable to common stockholders before (ii)(a) stock-based compensation expense, net of tax, (b) digital assets (gain) loss, net of tax and (c) release of valuation allowance on deferred tax assets. "Adjusted EBITDA (non-GAAP)" is equal to (i) net income attributable to common stockholders before (ii)(a) interest expense, (b) provision for (benefit from) income taxes, (c) depreciation, amortization and impairment, (d) stock-based compensation expense and (e) digital assets loss (gain), net. "Free cash flow" is operating cash flow less capital expenditures. Average cost per vehicle is cost of automotive sales divided by new vehicle deliveries (excluding operating leases). "Days sales outstanding" is equal to (i) average accounts receivable, net for the period divided by (ii) total revenues and multiplied by (iii) the number of days in the period. "Days payable outstanding" is equal to (i) average accounts payable for the period divided by (ii) total cost of revenues and multiplied by (iii) the number of days in the period. "Days of supply" is calculated by dividing new car ending inventory by the relevant period's deliveries and using trading days. Constant currency impacts are calculated by comparing actuals against current results converted into USD using average exchange rates from the prior period. + +# NON-GAAP FINANCIAL INFORMATION + +Consolidated financial information has been presented in accordance with GAAP as well as on a non-GAAP basis to supplement our consolidated financial results. Our non-GAAP financial measures include non-GAAP net income (loss) attributable to common stockholders, non-GAAP net income (loss) attributable to common stockholders on a diluted per share basis (calculated using weighted average shares for GAAP diluted net income (loss) attributable to common stockholders), Adjusted EBITDA margin, non-GAAP automotive gross margin and free cash flow. These non-GAAP financial measures also facilitate management's internal comparisons to Tesla's historical performance as well as comparisons to the operating results of other companies. Management believes that it is useful to supplement its GAAP financial statements with this non-GAAP information because management uses such information internally for its operating, budgeting and financial planning purposes. Management also believes that presentation of the non-GAAP financial measures provides useful information to our investors regarding our financial condition and results of operations, so that investors can see through the eyes of Tesla management regarding important financial metrics that Tesla uses to run the business and allowing investors to better understand Tesla's performance. Non-GAAP information is not prepared under a comprehensive set of accounting rules and therefore, should only be read in conjunction with financial information reported under U.S. GAAP when understanding Tesla's operating performance. A reconciliation between GAAP and non-GAAP financial information is provided above. + +# FORWARD-LOOKING STATEMENTS + +Certain statements in this update, including, but not limited to, statements in the “Outlook” section; statements relating to the development, strategy, ramp, production and capacity, demand and market growth, cost, pricing and profitability, investment, deliveries, deployment, availability and other features and improvements and timing of existing and future Tesla products and services and supporting infrastructure; statements regarding operating margin, operating profits, spending and liquidity; and statements regarding expansion, improvements and/or ramp and related timing at our facilities are “forward-looking statements” within the meaning of the Private Securities Litigation Reform Act of 1995. Forward-looking statements are based on assumptions and management’s current expectations, involve certain risks and uncertainties, and are not guarantees. Future results may differ materially from those expressed in any forward-looking statement. The following important factors, without limitation, could cause actual results to differ materially from those in the forward-looking statements: the risk of delays in launching and/or manufacturing our products, services and features cost-effectively; our ability to build and/or grow our products and services, sales, delivery, installation, servicing and charging capabilities and effectively manage this growth; our ability to successfully and timely develop, introduce and scale, as well as our consumers’ demand for, products and services based on artificial intelligence, robotics and automation, electric vehicles, advanced driver assistance systems, and ride-hailing services generally and our vehicles and services specifically; the ability of suppliers to deliver components according to schedules, prices, quality and volumes acceptable to us, and our ability to manage such components effectively; any issues with lithium-ion cells or other components manufactured at our factories; our ability to ramp our factories in accordance with our plans; our ability to procure supply of battery cells, including through our own manufacturing; risks relating to international operations and expansion, including unfavorable and uncertain regulatory, political, economic, tax, tariff, export controls and labor conditions; any failures by Tesla products to perform as expected or if product recalls occur; the risk of product liability claims; competition in the automotive, transportation and energy product and services and robotics markets; our ability to maintain public credibility and confidence in our long-term business prospects; our ability to manage risks relating to our various product financing programs; the status of government and economic incentives for electric vehicles and energy products; our ability to attract, hire and retain key employees and qualified personnel; our ability to maintain the security of our information and production and product systems; our compliance with various regulations and laws applicable to our operations and products, which may evolve from time to time; risks relating to our indebtedness and financing strategies; and adverse foreign exchange movements. More information on potential factors that could affect our financial results is included from time to time in our Securities and Exchange Commission filings and reports, including the risks identified under the section captioned “Risk Factors” in our annual report on Form 10-K filed with the SEC on January 30, 2025 and subsequent quarterly reports on Form 10-Q. Tesla disclaims any obligation to update information contained in these forward-looking statements whether as a result of new information, future events or otherwise. + +TESLA \ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-1-Capacity Growth Projection.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-1-Capacity Growth Projection.jpg new file mode 100644 index 000000000..47856b7e8 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-1-Capacity Growth Projection.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-10-Tesla Semi Trucks.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-10-Tesla Semi Trucks.jpg new file mode 100644 index 000000000..0128b0885 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-10-Tesla Semi Trucks.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-11-US Lightning Map.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-11-US Lightning Map.jpg new file mode 100644 index 000000000..97b7550c1 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-11-US Lightning Map.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-12-Tesla Factory Milestone.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-12-Tesla Factory Milestone.jpg new file mode 100644 index 000000000..68ffca651 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-12-Tesla Factory Milestone.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-13-Tesla Factory Milestone.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-13-Tesla Factory Milestone.jpg new file mode 100644 index 000000000..07396b797 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-13-Tesla Factory Milestone.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-14-Vehicle Delivery Trends.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-14-Vehicle Delivery Trends.jpg new file mode 100644 index 000000000..a84014a9f Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-14-Vehicle Delivery Trends.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-15-Quarterly Cash Flow.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-15-Quarterly Cash Flow.jpg new file mode 100644 index 000000000..7a06aebf3 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-15-Quarterly Cash Flow.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-16-Financial Performance Chart.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-16-Financial Performance Chart.jpg new file mode 100644 index 000000000..42e9b155e Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-16-Financial Performance Chart.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-17-Projected Vehicle Deliveries.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-17-Projected Vehicle Deliveries.jpg new file mode 100644 index 000000000..e669bbc15 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-17-Projected Vehicle Deliveries.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-18-Cash Flow Trends.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-18-Cash Flow Trends.jpg new file mode 100644 index 000000000..99b8f2700 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-18-Cash Flow Trends.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-19-Financial Performance Forecast.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-19-Financial Performance Forecast.jpg new file mode 100644 index 000000000..3a28a0165 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-19-Financial Performance Forecast.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-2-FSD Mileage Growth.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-2-FSD Mileage Growth.jpg new file mode 100644 index 000000000..c40682061 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-2-FSD Mileage Growth.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-3-Tesla Silicon Optimization.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-3-Tesla Silicon Optimization.jpg new file mode 100644 index 000000000..7f9eea688 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-3-Tesla Silicon Optimization.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-4-Growth Trend 2025.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-4-Growth Trend 2025.jpg new file mode 100644 index 000000000..ace450520 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-4-Growth Trend 2025.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-5-Tesla Model Y Driving.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-5-Tesla Model Y Driving.jpg new file mode 100644 index 000000000..386ada071 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-5-Tesla Model Y Driving.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-6-Red Tesla on Coastal Road.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-6-Red Tesla on Coastal Road.jpg new file mode 100644 index 000000000..c84f0be23 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-6-Red Tesla on Coastal Road.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-7-Tesla Interior Interface.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-7-Tesla Interior Interface.jpg new file mode 100644 index 000000000..03b5e639c Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-7-Tesla Interior Interface.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-8-Tesla Interior.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-8-Tesla Interior.jpg new file mode 100644 index 000000000..a24acf918 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-8-Tesla Interior.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-9-Tesla Cybertruck in Snow.jpg b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-9-Tesla Cybertruck in Snow.jpg new file mode 100644 index 000000000..3a30afeef Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image-9-Tesla Cybertruck in Snow.jpg differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/manifest.json b/apps/api/app/data/demo_documents/tsla-q4-2025/manifest.json new file mode 100644 index 000000000..642fadb61 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/manifest.json @@ -0,0 +1,30 @@ +{ + "version": "2.0", + "job_id": "job_b7886be54d96", + "data_id": null, + "source_file_name": "TSLA-Q4-2025-Update.pdf", + "processing_date": "2026-05-12T02:42:26.779662Z", + "processing": { + "page_count": 35, + "billing_status": "charged", + "cost": { + "micro_dollars": 52500, + "credits": 0.0525 + }, + "timing": { + "started_at": "2026-05-12T02:41:42.463885+00:00", + "completed_at": "2026-05-12T02:42:26.292306+00:00", + "duration_ms": 43828 + } + }, + "statistics": { + "total_chunks": 70, + "text_chunks": 36, + "image_chunks": 19, + "table_chunks": 15, + "total_pages": null + }, + "HIERARCHY": { + "Root": {} + } +} \ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/original.pdf b/apps/api/app/data/demo_documents/tsla-q4-2025/original.pdf new file mode 100644 index 000000000..87cdb1be5 Binary files /dev/null and b/apps/api/app/data/demo_documents/tsla-q4-2025/original.pdf differ diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-0 Tesla 2025 Results.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-0 Tesla 2025 Results.html new file mode 100644 index 000000000..459a4b394 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-0 Tesla 2025 Results.html @@ -0,0 +1 @@ +
Profitability$4.4B GAAP operating income in 2025; $1.4B in Q42025 marked a critical year for Tesla as we further expanded our mission and continued our transition from a hardware-centric business to a physical AI company. We laid the foundation for the future of Tesla as we further advanced FSD (Supervised) $^{4}$ , launched our Robotaxi service, began installing production lines for Cybercab and fine-tuned our production-primed Optimus design while expanding our AI training infrastructure.
$3.8B GAAP net income in 2025; $0.8B in Q4
$5.9B non-GAAP net income $^{1}$ in 2025; $1.8B in Q4
CashOperating cash flow of $14.7B in 2025; $3.8B in Q4Our approach to autonomous vehicles and humanoid robots mirrors the way we approached electric vehicles and energy storage – at the system level where we identify the limiting factor and develop bespoke and scalable solutions (batteries, power electronics, inverters, software, AI silicon, etc.) to optimize for cost, functionality, efficiency and safety. Our vertical integration has enabled us to achieve economies of scale in a profitable manner, quickly troubleshoot bottlenecks in production and iteratively optimize our technologies more rapidly than others.
Free cash flow $^{2}$ of $6.2B in 2025; $1.4B in Q4In 2025, we completed the refresh of our vehicle lineup with the launch of the new Model Y, including additional variants. We believe that maintaining an optimized and efficient product portfolio, with a continued focus on high-value features such as long range, best-in-class software and autonomy, is the correct strategy to win the autos market of the future. Similarly, we continued to evolve our energy offerings for commercial, utility and retail customers, as we position ourselves as a supplier of choice for clean, affordable and rapidly deployable energy capacity ahead of expected sustained demand growth for electricity.
$7.5B increase in our cash and investments $^{3}$ in 2025 to $44.1B
OperationsBegan removing safety monitor from our Robotaxis in Austin in JanuaryIn 2026, we will further invest in the infrastructure needed to support clean energy and transport and autonomous robots, including the ramp of six new production lines across vehicle, robots, energy storage and battery manufacturing, while further leveraging our existing factory, charging and service center footprint to support future growth.
Record Q4 & FY'25 energy storage deployments
Record vehicle deliveries in APAC
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-1 Q4 2025 Financials.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-1 Q4 2025 Financials.html new file mode 100644 index 000000000..c2da74ef2 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-1 Q4 2025 Financials.html @@ -0,0 +1 @@ +
($ in millions, except percentages and per share data)Q4-2024Q1-2025Q2-2025Q3-2025Q4-2025YoY
Total automotive revenues19,79813,96716,66121,20517,693-11%
Energy generation and storage revenue3,0612,7302,7893,4153,83725%
Services and other revenue2,8482,6383,0463,4753,37118%
Total revenues25,70719,33522,49628,09524,901-3%
Total gross profit4,1793,1533,8785,0545,00920%
Total GAAP gross margin16.3%16.3%17.2%18.0%20.1%386 bp
Operating expenses2,5962,7542,9553,4303,60039%
Income from operations1,5833999231,6241,409-11%
Operating margin6.2%2.1%4.1%5.8%5.7%-50 bp
Adjusted EBITDA (1) (2)4,3332,8143,4014,2274,154-4%
Adjusted EBITDA margin (1) (2)16.9%14.6%15.1%15.0%16.7%-17 bp
Net income attributable to common stockholders (GAAP) (1)2,1284091,1721,373840-61%
Net income attributable to common stockholders (non-GAAP) (1) (3)2,1079341,3931,7701,761-16%
EPS attributable to common stockholders, diluted (GAAP) (1)0.600.120.330.390.24-60%
EPS attributable to common stockholders, diluted (non-GAAP) (1) (3)0.600.270.400.500.50-17%
Net cash provided by operating activities4,8142,1562,5406,2383,813-21%
Capital expenditures (4)(2,780)(1,492)(2,394)(2,248)(2,393)-14%
Free cash flow (4)2,0346641463,9901,420-30%
Cash, cash equivalents and investments36,56336,99636,78241,64744,05921%
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-10 Cash Flow Q4-25.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-10 Cash Flow Q4-25.html new file mode 100644 index 000000000..a99dda47a --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-10 Cash Flow Q4-25.html @@ -0,0 +1 @@ +
In millions of USDQ4-2024Q1-2025Q2-2025Q3-2025Q4-2025
CASH FLOWS FROM OPERATING ACTIVITIES
Net income (1)2,1434201,1901,389856
Adjustments to reconcile net income to net cash provided by operating activities:
Depreciation, amortization and impairment1,4961,4471,4331,6251,643
Stock-based compensation579573635663954
Deferred income taxes (1)6(43)52225(111)
Digital assets (gain) loss, net (1)(347)125(284)(80)307
Other(93)188187333378
Changes in operating assets and liabilities1030(554)(673)2,083(214)
Net cash provided by operating activities4,8142,1562,5406,2383,813
CASH FLOWS FROM INVESTING ACTIVITIES
Capital expenditures (2)(2,780)(1,492)(2,394)(2,248)(2,393)
Purchases of investments(15,158)(6,015)(7,485)(11,402)(12,207)
Proceeds from maturities of investments10,3355,8566,9359,2958,072
Net cash used in investing activities(7,603)(1,651)(2,944)(4,355)(6,528)
CASH FLOWS FROM FINANCING ACTIVITIES
Net cash flows from other debt activities(108)(50)(23)410963
Net borrowings (repayments) under vehicle and energy product financing677(674)(400)81(377)
Net cash flows from noncontrolling interests – Solar(37)(22)(14)(20)(22)
Other453414215512146
Net cash provided by (used in) financing activities985(332)(222)983710
Effect of exchange rate changes on cash and cash equivalents and restricted cash(133)40111(17)37
Net (decrease) increase in cash and cash equivalents and restricted cash(1,937)213(515)2,849(1,968)
Cash and cash equivalents and restricted cash at beginning of period18,97417,03717,25016,73519,584
Cash and cash equivalents and restricted cash at end of period17,03717,25016,73519,58417,616
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-11 Q4 2024-Q4 2025.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-11 Q4 2024-Q4 2025.html new file mode 100644 index 000000000..5d6569ca2 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-11 Q4 2024-Q4 2025.html @@ -0,0 +1 @@ +
In millions of USD or shares as applicable, except per share dataQ4-2024Q1-2025Q2-2025Q3-2025Q4-2025
Net income attributable to common stockholders (GAAP) (1)2,1284091,1721,373840
Stock-based compensation expense, net of tax249428443459682
Digital assets (gain) loss, net of tax (1)(270)97(222)(62)239
Net income attributable to common stockholders (non-GAAP) (1) (2)2,1079341,3931,7701,761
Less: Buy-outs of noncontrolling interests3————
Net income used in computing diluted EPS attributable to common stockholders (non-GAAP) (1) (2)2,1049341,3931,7701,761
EPS attributable to common stockholders, diluted (GAAP) (1)0.600.120.330.390.24
Stock-based compensation expense, net of tax, per share0.080.120.130.130.19
Digital assets (gain) loss, net of tax, per share (1)(0.08)0.03(0.06)(0.02)0.07
EPS attributable to common stockholders, diluted (non-GAAP) (1) (2)0.600.270.400.500.50
Shares used in EPS calculation, diluted (GAAP and non-GAAP)3,5173,5213,5193,5263,539
Net income attributable to common stockholders (GAAP) (1)2,1284091,1721,373840
Interest expense9691867685
Provision for income taxes (1)381169359570325
Depreciation, amortization and impairment1,4961,4471,4331,6251,643
Stock-based compensation expense579573635663954
Digital assets (gain) loss, net (1)(347)125(284)(80)307
Adjusted EBITDA (non-GAAP) (1) (3)4,3332,8143,4014,2274,154
Total revenues25,70719,33522,49628,09524,901
Adjusted EBITDA margin (non-GAAP) (1) (3)16.9%14.6%15.1%15.0%16.7%
Automotive gross margin (GAAP)16.6%16.2%17.2%17.0%20.4%
Less: Total regulatory credit revenue recognized3.0%3.7%2.2%1.6%2.5%
Automotive gross margin excluding regulatory credit sales (non-GAAP)13.6%12.5%15.0%15.4%17.9%
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-12 Financial Metrics 2021-25.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-12 Financial Metrics 2021-25.html new file mode 100644 index 000000000..af396802d --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-12 Financial Metrics 2021-25.html @@ -0,0 +1 @@ +
In millions of USD or shares as applicable, except per share data20212022202320242025
Net income attributable to common stockholders (GAAP)5,51912,55614,9977,0913,794
Stock-based compensation expense, net of tax2,1211,5601,8121,3282,012
Digital assets loss (gain), net of tax79160—(459)52
Release of valuation allowance on deferred tax assets——(5,927)——
Net income attributable to common stockholders (non-GAAP)(1)7,71914,27610,8827,9605,858
Less: Buy-outs of noncontrolling interests(5)(27)(2)(39)—
Less: Dilutive convertible debt(9)(1)———
Net income used in computing diluted EPS attributable to common stockholders (non-GAAP)(1)7,73314,30410,8847,9995,858
EPS attributable to common stockholders, diluted (GAAP)1.633.624.302.041.08
Stock-based compensation expense, net of tax, per share0.630.450.520.380.57
Digital assets loss (gain), net of tax, per share0.020.05—(0.13)0.01
Release of valuation allowance on deferred tax assets——(1.70)——
EPS attributable to common stockholders, diluted (non-GAAP)(1)2.284.123.122.291.66
Shares used in EPS calculation, diluted (GAAP and non-GAAP)3,3863,4753,4853,4983,528
Net income attributable to common stockholders (GAAP)5,51912,55614,9977,0913,794
Interest expense371191156350338
Provision for (benefit from) income taxes6991132(5,001)1,8371,423
Depreciation, amortization and impairment2,9113,7474,6675,3686,148
Stock-based compensation expense2,1211,5601,8121,9992,825
Digital assets loss (gain), net101204—(589)68
Adjusted EBITDA (non-GAAP)(2)11,72219,39016,63116,05614,596
Total revenues53,82381,46296,77397,69094,827
Adjusted EBITDA margin (non-GAAP)(2)21.8%23.8%17.2%16.4%15.4%
Automotive gross margin (GAAP)29.3%28.5%19.4%18.4%17.8%
Less: Total regulatory credit revenue recognized2.3%1.8%1.7%3.0%2.4%
Automotive gross margin excluding regulatory credit sales (non-GAAP)27.0%26.7%17.7%15.4%15.4%
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-13 Financial Data 2022-25.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-13 Financial Data 2022-25.html new file mode 100644 index 000000000..f850e6ec0 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-13 Financial Data 2022-25.html @@ -0,0 +1 @@ +
In millions of USD2Q-20223Q-20224Q-20221Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20252Q-20253Q-20254Q-2025
Net cash provided by operating activities (GAAP)2,3515,1003,2782,5133,0653,3084,3702423,6126,2554,8142,1562,5406,2383,813
Capital expenditures (1)(1,730)(1,803)(1,858)(2,073)(2,060)(2,459)(2,307)(2,777)(2,272)(3,513)(2,780)(1,492)(2,394)(2,248)(2,393)
Free cash flow (non-GAAP) (1)6213,2971,4204401,0058492,063(2,535)1,3402,7422,0346641463,9901,420
In millions of USD2Q-20223Q-20224Q-20221Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20 252Q-20253Q-20254Q-2025
Net income attributable to common stockholders (GAAP) (2)2,2593,2923,6872,5132,7031,8537,9281,3901,4002,1732,1284091,1721,373840
Interest expense445333292838617686929691867685
Provision for (benefit from) income taxes (2)205305276261323167(5,752)483371602381169359570325
Depreciation, amortization and impairment9229569891,0461,1541,2351,2321,2461,2781,3481,4961,4471,4331,6251,643
Stock-based compensation expense361362419418445465484524439457579573635663954
Digital assets loss (gain), net (2)170—34————(335)100(7)(347)125(284)(80)307
Adjusted EBITDA (non-GAAP) (2) (3)3,9614,9685,4384,2674,6533,7583,9533,3843,6744,6654,3332,8143,4014,2274,154
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-14 Financial Metrics 2023-25.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-14 Financial Metrics 2023-25.html new file mode 100644 index 000000000..f1645375c --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-14 Financial Metrics 2023-25.html @@ -0,0 +1 @@ +
In millions of USD1Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20252Q-20253Q-20254Q-2025
Net cash provided by operating activities – TTM (GAAP)13,24213,95612,16413,25610,98511,53214,47914,92316,83715,76515,74814,747
Capital expenditures – TTM (1)(7,464)(7,794)(8,450)(8,899)(9,603)(9,815)(10,869)(11,342)(10,057)(10,179)(8,914)(8,527)
Free cash flow – TTM (non-GAAP) (1)5,7786,1623,7144,3571,3821,7173,6103,5816,7805,5866,8346,220
In millions of USD1Q-20232Q-20233Q-20234Q-20231Q-20242Q-20243Q-20244Q-20241Q-20252Q-20253Q-20254Q-20
Net income attributable to common stockholders – TTM (GAAP) (2)11,75112,19510,75614,99713,87412,57112,8917,0916,1105,8825,0823,794
Interest expense – TTM159143128156203261315350365365349338
Provision for (benefit from) income taxes – TTM (2)1,0471,1651,027(5,001)(4,779)(4,731)(4,296)1,8371,5231,5111,4791,423
Depreciation, amortization and impairment – TTM3,9134,1454,4244,6674,8674,9915,1045,3685,5695,7246,0016,148
Stock-based compensation expense – TTM1,5601,6441,7471,8121,9181,9121,9041,9992,0482,2442,4502,825
Digital assets loss (gain), net – TTM (2)2043434—(335)(235)(242)(589)(129)(513)(586)68
Adjusted EBITDA – TTM (non-GAAP) (2) (3)18,63419,32618,11616,63115,74814,76915,67616,05615,48615,21314,77514,596
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-2 Financial Data 2021-25.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-2 Financial Data 2021-25.html new file mode 100644 index 000000000..c1da2bc3d --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-2 Financial Data 2021-25.html @@ -0,0 +1 @@ +
($ in millions, except percentages and per share data)20212022202320242025YoY
Total automotive revenues47,23271,46282,41977,07069,526-10%
Energy generation and storage revenue2,7893,9096,03510,08612,77127%
Services and other revenue3,8026,0918,31910,53412,53019%
Total revenues53,82381,46296,77397,69094,827-3%
Total gross profit13,60620,85317,66017,45017,094-2%
Total GAAP gross margin25.3%25.6%18.2%17.9%18.0%16 bp
Operating expenses7,0837,1978,76910,37412,73923%
Income from operations6,52313,6568,8917,0764,355-38%
Operating margin12.1%16.8%9.2%7.2%4.6%-265 bp
Adjusted EBITDA (1)11,72219,39016,63116,05614,596-9%
Adjusted EBITDA margin (1)21.8%23.8%17.2%16.4%15.4%-104 bp
Net income attributable to common stockholders (GAAP)5,51912,55614,9977,0913,794-46%
Net income attributable to common stockholders (non-GAAP) (2)7,71914,27610,8827,9605,858-26%
EPS attributable to common stockholders, diluted (GAAP)1.633.624.302.041.08-47%
EPS attributable to common stockholders, diluted (non-GAAP) (2)2.284.123.122.291.66-28%
Net cash provided by operating activities11,49714,72413,25614,92314,747-1%
Capital expenditures (3)(6,514)(7,163)(8,899)(11,342)(8,527)-25%
Free cash flow (3)4,9837,5614,3573,5816,22074%
Cash, cash equivalents and investments17,70722,18529,09436,56344,05921%
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-3 Tesla Q4-2025 Data.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-3 Tesla Q4-2025 Data.html new file mode 100644 index 000000000..8e4222644 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-3 Tesla Q4-2025 Data.html @@ -0,0 +1 @@ +
Q4-2024Q1-2025Q2-2025Q3-2025Q4-2025YoY
Model 3/Y production436,718345,454396,835435,826422,652-3%
Other models production22,72717,16113,40911,62411,706-48%
Total production459,445362,615410,244447,450434,358-5%
Model 3/Y deliveries471,930323,800373,728481,166406,585-14%
Other models deliveries23,64012,88110,39415,93311,642-51%
Total deliveries495,570336,681384,122497,099418,227-16%
of which subject to operating lease accounting26,96213,7216,67010,23010,996-59%
Cumulative $deliveries^{(1)}$ (all-time; mil)7.37.68.08.58.922%
Active FSD $Subscriptions^{(2)}$ (mil)0.80.80.91.01.138%
Total end of quarter operating lease (new vehicle) $count^{(3)}$ 180,523179,930172,882167,163163,075-10%
Global vehicle inventory (days of supply) $^{(4)}$ 122224101525%
Storage deployed (GWh)11.010.49.612.514.229%
Tesla locations1,3591,3901,4541,4981,55314%
Supercharger stations6,9757,1317,3777,7538,18217%
Supercharger connectors65,49567,31670,22873,81777,68219%
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-4 Tesla 2021-2025 Data.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-4 Tesla 2021-2025 Data.html new file mode 100644 index 000000000..65641b145 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-4 Tesla 2021-2025 Data.html @@ -0,0 +1 @@ +
20212022202320242025YoY
Model 3/Y production906,0321,298,4341,775,1591,679,3381,600,767-5%
Other models production24,39071,17770,82694,10553,900-43%
Total production930,4221,369,6111,845,9851,773,4431,654,667-7%
Model 3/Y deliveries911,2421,247,1461,739,7071,704,0931,585,279-7%
Other models deliveries24,98066,70568,87485,13350,850-40%
Total deliveries936,2221,313,8511,808,5811,789,2261,636,129-9%
of which subject to operating lease accounting60,91247,58272,22660,00341,617-31%
Cumulative $deliveries^{(1)}$ (all-time; mil)2.33.75.57.38.922%
Active FSD $Subscriptions^{(2)}$ (mil)0.40.50.60.81.138%
Total end of year operating lease (new vehicle) count120,342140,667176,564180,523163,075-10%
Global vehicle inventory (days of supply) $^{(3)}$ 61616131515%
Storage deployed (GWh)4.06.514.731.446.749%
Tesla locations6449631,2081,3591,55314%
Supercharger stations3,4764,6785,9526,9758,18217%
Supercharger connectors31,49842,41954,89265,49577,68219%
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-5 Tesla Production.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-5 Tesla Production.html new file mode 100644 index 000000000..a95f02150 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-5 Tesla Production.html @@ -0,0 +1 @@ +
RegionProductCapacityStatus
Automotive
CaliforniaModel 3 / Model Y>550,000Production
Model S / Model X100,000Production
ShanghaiModel 3 / Model Y>950,000Production
BerlinModel Y>375,000Production
TexasModel Y>250,000Production
Cybertruck>125,000Production
Cybercab-Tooling
NevadaTesla Semi-Tooling
TBDRoadster-Design development
Energy Generation and Storage
CaliforniaMegapack40 GWhProduction
NevadaPowerwall>6 GWhProduction
ShanghaiMegapack40 GWhProduction
TexasMegapack-Construction
Robotics
CaliforniaOptimus-Construction
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-6 Facility Status.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-6 Facility Status.html new file mode 100644 index 000000000..3c1a54bb9 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-6 Facility Status.html @@ -0,0 +1 @@ +
RegionProductCapacityStatus
AI Training Compute
TexasCortex 1>100k H100eProduction
Cortex 2-Construction
Battery Manufacturing
NevadaLFP7 GWhEarly Ramp
Texas468040 GWhProduction
Cathode Materials10 GWhEarly Ramp
Lithium Refining30 GWhEarly Ramp
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-7 Autonomous Driving Status.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-7 Autonomous Driving Status.html new file mode 100644 index 000000000..21f90f521 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-7 Autonomous Driving Status.html @@ -0,0 +1 @@ +
StateMetroStatus
CaliforniaSF Bay AreaSafety Driver
TexasAustinRamping Unsupervised
Dallas1H 2026
Houston1H 2026
ArizonaPhoenix1H 2026
FloridaMiami1H 2026
Orlando1H 2026
Tampa1H 2026
NevadaLas Vegas1H 2026
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-8 Q4 2024-Q4 2025 Rev.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-8 Q4 2024-Q4 2025 Rev.html new file mode 100644 index 000000000..793ce0765 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-8 Q4 2024-Q4 2025 Rev.html @@ -0,0 +1 @@ +
In millions of USD or shares as applicable, except per share dataQ4-2024Q1-2025Q2-2025Q3-2025Q4-2025
REVENUES
Automotive sales18,65912,92515,78720,35916,750
Automotive regulatory credits692595439417542
Automotive leasing447447435429401
Total automotive revenues19,79813,96716,66121,20517,693
Energy generation and storage3,0612,7302,7893,4153,837
Services and other2,8482,6383,0463,4753,371
Total revenues25,70719,33522,49628,09524,901
COST OF REVENUES
Automotive sales16,26811,46113,56717,36513,874
Automotive leasing242239228225206
Total automotive cost of revenues16,51011,70013,79517,59014,080
Energy generation and storage2,2891,9451,9432,3422,739
Services and other2,7292,5372,8803,1093,073
Total cost of revenues21,52816,18218,61823,04119,892
Gross profit4,1793,1533,8785,0545,009
OPERATING EXPENSES
Research and development1,2761,4091,5891,6301,783
Selling, general and administrative1,3131,2511,3661,5621,655
Restructuring and other794—238162
Total operating expenses2,5962,7542,9553,4303,600
INCOME FROM OPERATIONS1,5833999231,6241,409
Interest income442400392439449
Interest expense(96)(91)(86)(76)(85)
Other income (expense), net (1)595(119)320(28)(592)
INCOME BEFORE INCOME TAXES (1)2,5245891,5491,9591,181
Provision for income taxes (1)381169359570325
NET INCOME (1)2,1434201,1901,389856
Net income attributable to noncontrolling interests and redeemable noncontrolling interests in subsidiaries1511181616
NET INCOME ATTRIBUTABLE TO COMMON STOCKHOLDERS (1)2,1284091,1721,373840
Less: Buy-out of noncontrolling interest3————
NET INCOME USED IN COMPUTING NET INCOME PER SHARE OF COMMON STOCK (1)2,1254091,1721,373840
Net income per share of common stock attributable to common stockholders
Basic (1)$ 0.66$ 0.13$ 0.36$ 0.43$ 0.26
Diluted (1)$ 0.60$ 0.12$ 0.33$ 0.39$ 0.24
Weighted average shares used in computing net income per share of common stock
Basic3,2133,2183,2233,2273,231
Diluted3,5173,5213,5193,5263,539
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-9 Balance Sheet 2024-25.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-9 Balance Sheet 2024-25.html new file mode 100644 index 000000000..fa828c3ca --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table-9 Balance Sheet 2024-25.html @@ -0,0 +1 @@ +
In millions of USD31-Dec-2431-Mar-2530-Jun-2530-Sep-2531-Dec-25
ASSETS
Current assets
Cash, cash equivalents and investments36,56336,99636,78241,64744,059
Accounts receivable, net4,4183,7823,8384,7034,576
Inventory12,01713,70614,57012,27612,392
Prepaid expenses and other current assets5,3624,9055,9436,0277,615
Total current assets58,36059,38961,13364,65368,642
Operating lease vehicles, net5,5815,4775,2305,0194,912
Energy generation and storage systems, net4,9244,8554,7884,6734,604
Property, plant and equipment, net35,83637,08838,57439,40740,643
Operating lease right-of-use assets5,1605,3305,6335,7836,027
Digital assets (2)1,0769511,2351,3151,008
Deferred tax assets (2)6,5246,6876,7216,6376,925
Other non-current assets4,6095,3345,2536,2485,045
Total assets (2)122,070125,111128,567133,735137,806
LIABILITIES AND EQUITY
Current liabilities
Accounts payable12,47413,47113,21212,81913,371
Accrued liabilities and other10,72310,80211,51912,79113,279
Deferred revenue3,1683,2433,2373,7563,424
Current portion of debt and finance leases (1)2,4562,2372,0401,9241,640
Total current liabilities28,82129,75330,00831,29031,714
Debt and finance leases, net of current portion (1)5,7575,2925,1805,7786,736
Deferred revenue, net of current portion3,3173,6103,7643,7463,631
Other long-term liabilities10,49511,03811,54312,20512,860
Total liabilities48,39049,69350,49553,01954,941
Redeemable noncontrolling interests in subsidiaries6362615958
Total stockholders' equity (2)72,91374,65377,31479,97082,137
Noncontrolling interests in subsidiaries704703697687670
Total liabilities and equity (2)122,070125,111128,567133,735137,806
(1) Breakdown of our debt is as follows:
Non-recourse debt7,8717,2386,9537,4588,150
Recourse debt76333
Days sales outstanding1419151417
Days payable outstanding5872655261
\ No newline at end of file diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/toc_hierarchies.json b/apps/api/app/data/demo_documents/tsla-q4-2025/toc_hierarchies.json new file mode 100644 index 000000000..15f097952 --- /dev/null +++ b/apps/api/app/data/demo_documents/tsla-q4-2025/toc_hierarchies.json @@ -0,0 +1,30 @@ +[ + { + "toc_range": [ + 1, + 15 + ], + "scan_range": [ + 1, + 194 + ], + "toc_with_level": "| id | heading | level |\n|----|------------------------------|-------|\n| 0 | # Q4 and FY 2025 Update | 1 |\n| 1 | Highlights 03 | 2 |\n| 2 | Financial Summary 04 | 2 |\n| 3 | Operational Summary 06 | 2 |\n| 4 | Manufacturing & Hardware 08 | 2 |\n| 5 | Supporting Infrastructure 09 | 2 |\n| 6 | AI & Software 10 | 2 |\n| 7 | Services 11 | 2 |\n| 8 | Other Updates 12 | 2 |\n| 9 | Outlook 13 | 2 |\n| 10 | Photos & Charts 14 | 2 |\n| 11 | Key Metrics 24 | 2 |\n| 12 | Financial Statements 27 | 2 |\n| 13 | Additional Information 34 | 2 |", + "toc_tree": { + "# Q4 and FY 2025 Update": { + "Highlights 03": {}, + "Financial Summary 04": {}, + "Operational Summary 06": {}, + "Manufacturing & Hardware 08": {}, + "Supporting Infrastructure 09": {}, + "AI & Software 10": {}, + "Services 11": {}, + "Other Updates 12": {}, + "Outlook 13": {}, + "Photos & Charts 14": {}, + "Key Metrics 24": {}, + "Financial Statements 27": {}, + "Additional Information 34": {} + } + } + } +] \ No newline at end of file diff --git a/apps/api/app/services/demo_document_service.py b/apps/api/app/services/demo_document_service.py new file mode 100644 index 000000000..ea50a9b1d --- /dev/null +++ b/apps/api/app/services/demo_document_service.py @@ -0,0 +1,814 @@ +"""API-owned canonical demo document catalog and materialization.""" + +from __future__ import annotations + +import json +import math +import shutil +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone +from functools import lru_cache +from hashlib import blake2b +from pathlib import Path +from typing import Any +from urllib.parse import quote +from uuid import uuid4 + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from shared.core.exceptions.domain_exceptions import ValidationException +from shared.models.database.demo_materialization import DemoMaterialization +from shared.models.database.document import Document +from shared.models.database.job import Job +from shared.models.database.job_result import JobResult +from shared.services.retrieval.cache_service import invalidate_retrieval_cache_namespaces +from shared.services.retrieval.publication_service import RetrievalPublicationService +from shared.services.storage.result_storage import get_result_storage + + +@dataclass(frozen=True) +class DemoCitationDefinition: + """Curated answer citation that resolves to a canonical demo chunk.""" + + section_path: str + description: str + content: str + + +@dataclass(frozen=True) +class DemoExampleDefinition: + """Curated user-facing demo question and answer.""" + + id: str + question: str + answer: str + citations: tuple[DemoCitationDefinition, ...] + + +@dataclass(frozen=True) +class DemoSourceDefinition: + """Canonical demo source metadata and local asset pointers.""" + + demo_source_id: str + canonical_document_id: str + title: str + mime_type: str + size_bytes: int + asset_directory: str + chunk_count: int + examples: tuple[DemoExampleDefinition, ...] + + +@dataclass(frozen=True) +class MaterializedDemoSource: + """User-owned copy of one canonical demo source.""" + + demo_source_id: str + document_id: str + status: str + title: str + mime_type: str + size_bytes: int + chunk_count: int + + +_DATA_ROOT = Path(__file__).resolve().parents[1] / "data" / "demo_documents" +_ASSET_DIRECTORY_NAMES = frozenset({"images", "tables"}) +_DEMO_SOURCE_DEFINITIONS: tuple[DemoSourceDefinition, ...] = ( + DemoSourceDefinition( + demo_source_id="demo-tsla-q4-2025", + canonical_document_id="demo-doc-tsla-q4-2025", + title="TSLA-Q4-2025-Update.pdf", + mime_type="application/pdf", + size_bytes=5_648_867, + asset_directory="tsla-q4-2025", + chunk_count=70, + examples=( + DemoExampleDefinition( + id="demo-tsla-q4-2025-xai", + question="What does the document say about Tesla's xAI investment?", + answer=( + "Tesla entered an agreement on January 16, 2026 to invest " + "approximately $2 billion in xAI Series E Preferred Stock.\n\n" + "The document also says Tesla and xAI entered a framework " + "agreement to evaluate AI collaboration, with the investment " + "expected to close in Q1 2026 subject to customary regulatory " + "conditions." + ), + citations=( + DemoCitationDefinition( + section_path=( + "Default_Root/TSLA-Q4-2025-Update.pdf-->OTHER UPDATES" + ), + description="xAI investment", + content=( + "On January 16, 2026, Tesla entered into an agreement " + "to invest approximately" + ), + ), + ), + ), + DemoExampleDefinition( + id="demo-tsla-q4-2025-energy-storage", + question="What does the document say about energy storage?", + answer=( + "Tesla achieved its highest quarterly energy storage " + "deployments, driven by record Megapack deployments.\n\n" + "Energy gross profit reached a record $1.1 billion, marking " + "the fifth consecutive record quarter.\n\n" + "Tesla also plans to begin Megapack 3 and Megablock " + "production at Megafactory Houston in 2026." + ), + citations=( + DemoCitationDefinition( + section_path=( + "Default_Root/TSLA-Q4-2025-Update.pdf-->SUMMARY-->" + "Energy generation and storage" + ), + description="Storage deployment growth", + content=( + "We achieved our highest quarterly energy storage " + "deployments, driven by record Megapack deployments." + ), + ), + ), + ), + DemoExampleDefinition( + id="demo-tsla-q4-2025-production-plans", + question="What production plans does Tesla mention for 2026?", + answer=( + "Tesla says Cybercab, Tesla Semi, and Megapack 3 are on " + "schedule for volume production starting in 2026.\n\n" + "The same product update also notes that first-generation " + "Optimus production lines are being installed before volume " + "production." + ), + citations=( + DemoCitationDefinition( + section_path=( + "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK-->" + "Product" + ), + description="2026 production plans", + content=( + "Cybercab, Tesla Semi and Megapack 3 are on schedule " + "for volume production starting in 2026." + ), + ), + ), + ), + ), + ), +) + + +class DemoDocumentService: + """Serves canonical demo data and copies it into user namespaces.""" + + def __init__( + self, + *, + publication_service: RetrievalPublicationService | None = None, + ) -> None: + self._publication_service = publication_service or RetrievalPublicationService() + + def get_catalog(self) -> dict[str, Any]: + """Return the cacheable canonical demo source catalog.""" + return { + "sources": [ + self._source_catalog_payload(source) + for source in _DEMO_SOURCE_DEFINITIONS + ], + } + + def list_chunks( + self, + *, + demo_source_id: str, + page: int, + page_size: int, + ) -> dict[str, Any] | None: + """Return paginated canonical demo chunks.""" + source = _get_source_definition(demo_source_id) + if source is None: + return None + + chunks = _load_source_chunks(source) + start = (page - 1) * page_size + page_chunks = chunks[start : start + page_size] + return { + "demo_source_id": source.demo_source_id, + "canonical_document_id": source.canonical_document_id, + "title": source.title, + "mime_type": source.mime_type, + "chunks": [ + _chunk_payload(source=source, chunk=chunk) + for chunk in page_chunks + ], + "pagination": { + "page": page, + "page_size": page_size, + "total": len(chunks), + "total_pages": math.ceil(len(chunks) / page_size) if chunks else 0, + }, + } + + def get_chunk( + self, + *, + demo_source_id: str, + demo_chunk_id: str, + ) -> dict[str, Any] | None: + """Return one canonical demo chunk by canonical row id or parser chunk id.""" + source = _get_source_definition(demo_source_id) + if source is None: + return None + + for chunk in _load_source_chunks(source): + if demo_chunk_id in {_canonical_chunk_id(source, chunk), chunk["chunk_id"]}: + return { + "demo_source_id": source.demo_source_id, + "canonical_document_id": source.canonical_document_id, + "chunk": _chunk_payload(source=source, chunk=chunk), + } + + return None + + def get_original_file_path(self, *, demo_source_id: str) -> Path | None: + """Return the canonical original file path for a demo source.""" + source = _get_source_definition(demo_source_id) + if source is None: + return None + + file_path = _source_directory(source) / "original.pdf" + return file_path if file_path.is_file() else None + + def get_asset_file_path( + self, + *, + demo_source_id: str, + asset_path: str, + ) -> Path | None: + """Return a canonical parsed media/table asset path.""" + source = _get_source_definition(demo_source_id) + if source is None: + return None + + source_directory = _source_directory(source).resolve() + normalized_asset_path = _normalize_asset_path(asset_path) + if normalized_asset_path is None: + return None + + candidate = (source_directory / normalized_asset_path).resolve() + if not candidate.is_relative_to(source_directory): + return None + + return candidate if candidate.is_file() else None + + async def materialize_sources( + self, + db: AsyncSession, + *, + user_id: str, + namespace: str, + demo_source_ids: list[str], + ) -> list[MaterializedDemoSource]: + """Copy selected canonical demo sources into a user namespace.""" + selected_demo_source_ids = _deduplicate_source_ids(demo_source_ids) + if not selected_demo_source_ids: + raise ValidationException( + user_message="At least one demo source must be selected.", + violations=[ + { + "field": "demo_source_ids", + "description": "Select one or more demo source IDs.", + } + ], + ) + + selected_sources = [ + _require_source_definition(demo_source_id) + for demo_source_id in selected_demo_source_ids + ] + results: list[MaterializedDemoSource] = [] + for source in selected_sources: + result = await self._materialize_source( + db, + user_id=user_id, + namespace=namespace, + source=source, + ) + results.append(result) + + await db.commit() + await invalidate_retrieval_cache_namespaces( + user_id=user_id, + namespaces=[namespace], + ) + return results + + async def _materialize_source( + self, + db: AsyncSession, + *, + user_id: str, + namespace: str, + source: DemoSourceDefinition, + ) -> MaterializedDemoSource: + await _lock_materialization_scope( + db, + user_id=user_id, + namespace=namespace, + demo_source_id=source.demo_source_id, + ) + existing = await self._get_existing_materialization( + db, + user_id=user_id, + namespace=namespace, + demo_source_id=source.demo_source_id, + ) + if existing is not None and await self._is_active_document( + db, + document_id=existing.document_id, + ): + return _materialized_source_payload( + source=source, + document_id=existing.document_id, + status="existing", + ) + + document_id = f"doc_{uuid4().hex[:12]}" + job_id = f"job_demo_{uuid4().hex[:12]}" + job_result_id = str(uuid4()) + timestamp = _utc_now() + result_bundle = _upload_demo_result_bundle(job_id=job_id, source=source) + + db.add( + Job( + job_id=job_id, + user_id=user_id, + job_type="demo_materialization", + status="done", + source_type="demo", + webhook_enabled=False, + job_metadata={ + "document_id": document_id, + "namespace": namespace, + "source_type": "demo", + "source_file_name": source.title, + "demo_source_id": source.demo_source_id, + }, + version=0, + created_at=timestamp, + updated_at=timestamp, + credits_charged=0, + billing_status="skipped", + ) + ) + db.add( + JobResult( + id=job_result_id, + job_id=job_id, + delivery_mode="inline", + document_metadata={ + "source_file_name": source.title, + "demo_source_id": source.demo_source_id, + }, + inline_payload={"source": "canonical_demo"}, + result_s3_key=result_bundle["zip_key"], + result_size=result_bundle["zip_size"], + created_at=timestamp, + updated_at=timestamp, + ) + ) + await db.flush() + chunks = _publication_chunks(source) + await db.run_sync( + lambda sync_db: self._publication_service.publish_document_state( + sync_db, + job_id=job_id, + job_result_id=job_result_id, + chunks=[dict(chunk) for chunk in chunks], + ) + ) + await db.run_sync( + lambda sync_db: self._publication_service.publish_document_graph( + sync_db, + job_id=job_id, + job_result_id=job_result_id, + ) + ) + await db.flush() + + if existing is None: + db.add( + DemoMaterialization( + user_id=user_id, + namespace=namespace, + demo_source_id=source.demo_source_id, + document_id=document_id, + created_at=timestamp, + updated_at=timestamp, + ) + ) + else: + existing.document_id = document_id + existing.updated_at = timestamp + await db.flush() + return _materialized_source_payload( + source=source, + document_id=document_id, + status="created", + ) + + async def _get_existing_materialization( + self, + db: AsyncSession, + *, + user_id: str, + namespace: str, + demo_source_id: str, + ) -> DemoMaterialization | None: + result = await db.execute( + select(DemoMaterialization) + .where(DemoMaterialization.user_id == user_id) + .where(DemoMaterialization.namespace == namespace) + .where(DemoMaterialization.demo_source_id == demo_source_id) + .with_for_update() + .limit(1) + ) + return result.scalar_one_or_none() + + async def _is_active_document( + self, + db: AsyncSession, + *, + document_id: str, + ) -> bool: + result = await db.execute( + select(Document.document_id) + .where(Document.document_id == document_id) + .where(Document.status == "active") + .limit(1) + ) + return result.scalar_one_or_none() is not None + + def _source_catalog_payload(self, source: DemoSourceDefinition) -> dict[str, Any]: + return { + "demo_source_id": source.demo_source_id, + "canonical_document_id": source.canonical_document_id, + "title": source.title, + "mime_type": source.mime_type, + "size_bytes": source.size_bytes, + "status": "ready", + "chunk_count": source.chunk_count, + "original_file": { + "url": f"/api/v1/demo/sources/{source.demo_source_id}/original", + "mime_type": source.mime_type, + "size_bytes": source.size_bytes, + "can_download": False, + }, + "examples": [ + self._example_payload(source=source, example=example) + for example in source.examples + ], + } + + def _example_payload( + self, + *, + source: DemoSourceDefinition, + example: DemoExampleDefinition, + ) -> dict[str, Any]: + return { + "id": example.id, + "question": example.question, + "answer": example.answer, + "citations": [ + _citation_payload(source=source, citation=citation) + for citation in example.citations + ], + } + + +def _deduplicate_source_ids(demo_source_ids: list[str]) -> list[str]: + selected: list[str] = [] + seen: set[str] = set() + for demo_source_id in demo_source_ids: + normalized = str(demo_source_id).strip() + if not normalized or normalized in seen: + continue + selected.append(normalized) + seen.add(normalized) + return selected + + +async def _lock_materialization_scope( + db: AsyncSession, + *, + user_id: str, + namespace: str, + demo_source_id: str, +) -> None: + lock_id = _materialization_lock_id( + user_id=user_id, + namespace=namespace, + demo_source_id=demo_source_id, + ) + await db.execute(select(func.pg_advisory_xact_lock(lock_id))) + + +def _materialization_lock_id( + *, + user_id: str, + namespace: str, + demo_source_id: str, +) -> int: + lock_key = f"{user_id}\0{namespace}\0{demo_source_id}" + digest = blake2b(lock_key.encode("utf-8"), digest_size=8).digest() + return int.from_bytes(digest, byteorder="big", signed=True) + + +def _materialized_source_payload( + *, + source: DemoSourceDefinition, + document_id: str, + status: str, +) -> MaterializedDemoSource: + return MaterializedDemoSource( + demo_source_id=source.demo_source_id, + document_id=document_id, + status=status, + title=source.title, + mime_type=source.mime_type, + size_bytes=source.size_bytes, + chunk_count=source.chunk_count, + ) + + +def _upload_demo_result_bundle( + *, + job_id: str, + source: DemoSourceDefinition, +) -> dict[str, int | str]: + """Upload canonical demo result files so copied media URLs resolve.""" + source_directory = _source_directory(source) + with tempfile.TemporaryDirectory(prefix="knowhere-demo-result-") as temp_directory: + zip_base_path = Path(temp_directory) / job_id + zip_file_path = Path( + shutil.make_archive( + str(zip_base_path), + "zip", + root_dir=source_directory, + ) + ) + zip_size = zip_file_path.stat().st_size + bundle = get_result_storage().upload( + job_id=job_id, + result_dir=str(source_directory), + zip_file_path=str(zip_file_path), + ) + + return { + "zip_key": bundle.zip_key, + "zip_size": zip_size, + } + + +def _publication_chunks(source: DemoSourceDefinition) -> list[dict[str, Any]]: + return [ + _publication_chunk(source=source, chunk=chunk) + for chunk in _load_source_chunks(source) + ] + + +def _publication_chunk( + *, + source: DemoSourceDefinition, + chunk: dict[str, Any], +) -> dict[str, Any]: + materialized_chunk = dict(chunk) + metadata = _metadata(materialized_chunk) + raw_path = _first_string(metadata.get("path"), materialized_chunk.get("path")) + publication_path = _publication_path(source=source, raw_path=raw_path) + file_path = _first_string( + metadata.get("file_path"), + metadata.get("filePath"), + materialized_chunk.get("file_path"), + materialized_chunk.get("path") if _is_media_chunk(materialized_chunk) else None, + ) + + metadata["path"] = publication_path + if file_path: + metadata["file_path"] = file_path + materialized_chunk["file_path"] = file_path + materialized_chunk["path"] = publication_path + materialized_chunk["metadata"] = metadata + return materialized_chunk + + +def _publication_path( + *, + source: DemoSourceDefinition, + raw_path: str | None, +) -> str: + prefix = f"Default_Root/{source.title}" + raw = str(raw_path or "").strip() + if not raw: + return prefix + + if "-->" in raw: + sections = [ + part.strip() + for part in raw.split("-->")[1:] + if part.strip() + ] + return "/".join([prefix, *sections]) if sections else prefix + + if raw.startswith("images/") or raw.startswith("tables/"): + return f"{prefix}/Assets/{raw}" + + parts = [part.strip() for part in raw.split("/") if part.strip()] + if len(parts) >= 2 and parts[0] == "Default_Root": + return raw + return prefix + + +def _normalize_asset_path(asset_path: str) -> Path | None: + normalized = str(asset_path or "").strip().replace("\\", "/").lstrip("/") + parts = [part for part in normalized.split("/") if part and part != "."] + if not parts or parts[0] not in _ASSET_DIRECTORY_NAMES: + return None + if any(part == ".." or part.startswith(".") for part in parts): + return None + return Path(*parts) + + +def _citation_payload( + *, + source: DemoSourceDefinition, + citation: DemoCitationDefinition, +) -> dict[str, Any]: + chunk = _resolve_citation_chunk(source=source, citation=citation) + return { + "demo_source_id": source.demo_source_id, + "canonical_document_id": source.canonical_document_id, + "canonical_chunk_id": _canonical_chunk_id(source, chunk), + "chunk_id": chunk["chunk_id"], + "chunk_type": _normalize_chunk_type(chunk.get("type")), + "content": citation.content, + "description": citation.description, + "source": { + "document_id": source.canonical_document_id, + "source_file_name": source.title, + "section_path": citation.section_path, + }, + } + + +def _resolve_citation_chunk( + *, + source: DemoSourceDefinition, + citation: DemoCitationDefinition, +) -> dict[str, Any]: + chunks = _load_source_chunks(source) + normalized_content = _normalize_text(citation.content) + if normalized_content: + for chunk in chunks: + if normalized_content in _normalize_text(str(chunk.get("content") or "")): + return chunk + + for chunk in chunks: + if str(chunk.get("path") or "") == citation.section_path: + return chunk + + raise ValueError( + "Demo citation does not resolve to a canonical chunk: " + f"demo_source_id={source.demo_source_id}, section_path={citation.section_path}" + ) + + +def _chunk_payload( + *, + source: DemoSourceDefinition, + chunk: dict[str, Any], +) -> dict[str, Any]: + metadata = _metadata(chunk) + file_path = _first_string( + metadata.get("file_path"), + metadata.get("filePath"), + chunk.get("file_path"), + chunk.get("path") if _is_media_chunk(chunk) else None, + ) + return { + "id": _canonical_chunk_id(source, chunk), + "chunk_id": chunk["chunk_id"], + "chunk_type": _normalize_chunk_type(chunk.get("type")), + "content": str(chunk.get("content") or ""), + "section_path": str(chunk.get("path") or "") or None, + "source_chunk_path": str(chunk.get("path") or "") or None, + "file_path": file_path, + "sort_order": _sort_order(source=source, chunk=chunk), + "metadata": metadata, + "asset_url": _asset_url(source=source, file_path=file_path), + "created_at": None, + } + + +def _sort_order( + *, + source: DemoSourceDefinition, + chunk: dict[str, Any], +) -> int: + try: + return _load_source_chunks(source).index(chunk) + except ValueError: + return 0 + + +def _metadata(chunk: dict[str, Any]) -> dict[str, Any]: + metadata = chunk.get("metadata") + return dict(metadata) if isinstance(metadata, dict) else {} + + +def _is_media_chunk(chunk: dict[str, Any]) -> bool: + return _normalize_chunk_type(chunk.get("type")) in {"image", "table"} + + +def _canonical_chunk_id( + source: DemoSourceDefinition, + chunk: dict[str, Any], +) -> str: + return f"{source.demo_source_id}:{chunk['chunk_id']}" + + +def _asset_url( + *, + source: DemoSourceDefinition, + file_path: str | None, +) -> str | None: + if not file_path: + return None + encoded_path = quote(file_path, safe="/") + return f"/api/v1/demo/sources/{source.demo_source_id}/assets/{encoded_path}" + + +def _normalize_chunk_type(value: object) -> str: + raw = str(value or "").strip().split("\n", 1)[0].lower() + return raw if raw in {"text", "image", "table"} else "text" + + +def _first_string(*values: object) -> str | None: + for value in values: + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _normalize_text(value: str) -> str: + return " ".join(value.lower().split()) + + +def _get_source_definition(demo_source_id: str) -> DemoSourceDefinition | None: + return next( + ( + source + for source in _DEMO_SOURCE_DEFINITIONS + if source.demo_source_id == demo_source_id + ), + None, + ) + + +def _require_source_definition(demo_source_id: str) -> DemoSourceDefinition: + source = _get_source_definition(demo_source_id) + if source is None: + raise KeyError(demo_source_id) + return source + + +def _source_directory(source: DemoSourceDefinition) -> Path: + return _DATA_ROOT / source.asset_directory + + +@lru_cache(maxsize=8) +def _load_source_chunks(source: DemoSourceDefinition) -> tuple[dict[str, Any], ...]: + chunks_path = _source_directory(source) / "chunks.json" + with chunks_path.open("r", encoding="utf-8") as file: + payload = json.load(file) + + chunks = payload.get("chunks") if isinstance(payload, dict) else None + if not isinstance(chunks, list): + return () + + return tuple( + dict(chunk) + for chunk in chunks + if isinstance(chunk, dict) and isinstance(chunk.get("chunk_id"), str) + ) + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None) diff --git a/apps/api/tests/contract/test_demo_documents_contract.py b/apps/api/tests/contract/test_demo_documents_contract.py new file mode 100644 index 000000000..abca22c68 --- /dev/null +++ b/apps/api/tests/contract/test_demo_documents_contract.py @@ -0,0 +1,405 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager +from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast + +import pytest +from httpx import AsyncClient +from pytest import MonkeyPatch + +from tests.support.contract_database import ContractDatabase + + +DEMO_SOURCE_ID = "demo-tsla-q4-2025" + + +class FakeResultStorage: + def __init__(self) -> None: + self.raw_files_by_job_id: dict[str, set[str]] = {} + + def upload( + self, + *, + job_id: str, + result_dir: str, + zip_file_path: str, + ) -> SimpleNamespace: + assert Path(zip_file_path).is_file() + result_path = Path(result_dir) + raw_files = { + file_path.relative_to(result_path).as_posix() + for file_path in result_path.rglob("*") + if file_path.is_file() + } + self.raw_files_by_job_id[job_id] = raw_files + return SimpleNamespace( + zip_key=f"results/{job_id}.zip", + raw_prefix=f"results/{job_id}/", + raw_files={ + raw_file: f"results/{job_id}/{raw_file}" + for raw_file in sorted(raw_files) + }, + ) + + def normalize_artifact_ref(self, artifact_ref: str | None) -> str | None: + if not artifact_ref: + return None + normalized = str(artifact_ref).strip().replace("\\", "/").lstrip("/") + if normalized.startswith("images/") or normalized.startswith("tables/"): + return normalized + return None + + def generate_artifact_url( + self, + *, + job_id: str, + artifact_ref: str, + expires_in: int = 3600, + ) -> str | None: + normalized = self.normalize_artifact_ref(artifact_ref) + if not normalized: + return None + if normalized not in self.raw_files_by_job_id.get(job_id, set()): + return None + return f"https://assets.example.test/{job_id}/{normalized}" + + +@pytest.mark.asyncio +async def test_should_return_demo_catalog_with_resolvable_canonical_citations( + api_client_factory: Callable[[], AbstractAsyncContextManager[AsyncClient]], +) -> None: + async with api_client_factory() as api_client: + catalog_response = await api_client.get("/api/v1/demo/catalog") + + assert catalog_response.status_code == 200 + catalog = cast(dict[str, Any], catalog_response.json()) + sources = cast(list[dict[str, Any]], catalog["sources"]) + source = sources[0] + examples = cast(list[dict[str, Any]], source["examples"]) + citations = cast(list[dict[str, Any]], examples[0]["citations"]) + citation = citations[0] + + assert source["demo_source_id"] == DEMO_SOURCE_ID + assert source["canonical_document_id"] == "demo-doc-tsla-q4-2025" + assert source["chunk_count"] == 70 + assert source["original_file"]["can_download"] is False + assert citation["canonical_document_id"] == "demo-doc-tsla-q4-2025" + assert citation["canonical_chunk_id"].startswith(f"{DEMO_SOURCE_ID}:") + + async with api_client_factory() as api_client: + chunks_response = await api_client.get( + f"/api/v1/demo/sources/{DEMO_SOURCE_ID}/chunks?page_size=200" + ) + chunk_response = await api_client.get( + "/api/v1/demo/sources/" + f"{DEMO_SOURCE_ID}/chunks/{citation['canonical_chunk_id']}" + ) + + assert chunks_response.status_code == 200 + assert chunk_response.status_code == 200 + chunks_body = cast(dict[str, Any], chunks_response.json()) + chunk_page = cast(list[dict[str, Any]], chunks_body["chunks"]) + asset_url = next( + str(chunk["asset_url"]) for chunk in chunk_page if chunk.get("asset_url") + ) + chunk_body = cast(dict[str, Any], chunk_response.json()) + chunk = cast(dict[str, Any], chunk_body["chunk"]) + + assert chunk["id"] == citation["canonical_chunk_id"] + assert citation["content"] in chunk["content"] + + async with api_client_factory() as api_client: + asset_response = await api_client.get(asset_url) + original_response = await api_client.get( + f"/api/v1/demo/sources/{DEMO_SOURCE_ID}/original" + ) + internal_asset_response = await api_client.get( + f"/api/v1/demo/sources/{DEMO_SOURCE_ID}/assets/full.md" + ) + + assert asset_response.status_code == 200 + assert asset_response.headers["content-disposition"].startswith("inline") + assert original_response.status_code == 200 + assert original_response.headers["content-disposition"].startswith("inline") + assert internal_asset_response.status_code == 404 + + +@pytest.mark.asyncio +async def test_should_materialize_demo_source_without_parse_or_credit_charge( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], + monkeypatch: MonkeyPatch, +) -> None: + fake_result_storage = FakeResultStorage() + monkeypatch.setattr( + "shared.services.storage.result_storage.get_result_storage", + lambda: fake_result_storage, + ) + + async with developer_api_client_factory() as api_client: + empty_cached_response = await api_client.post( + "/api/v1/retrieval/query", + json={ + "namespace": "contract-demo", + "query": "xAI investment", + "top_k": 5, + }, + ) + first_response = await api_client.post( + "/api/v1/demo/materializations", + json={ + "namespace": "contract-demo", + "demo_source_ids": [DEMO_SOURCE_ID], + }, + ) + retry_response = await api_client.post( + "/api/v1/demo/materializations", + json={ + "namespace": "contract-demo", + "demo_source_ids": [DEMO_SOURCE_ID], + }, + ) + retrieval_response = await api_client.post( + "/api/v1/retrieval/query", + json={ + "namespace": "contract-demo", + "query": "xAI investment", + "top_k": 5, + }, + ) + document_chunks_response = await api_client.get( + f"/api/v1/documents/{first_response.json()['sources'][0]['document_id']}" + "/chunks?page_size=200" + ) + + assert empty_cached_response.status_code == 200 + assert first_response.status_code == 200 + assert retry_response.status_code == 200 + assert retrieval_response.status_code == 200 + assert document_chunks_response.status_code == 200 + + empty_cached_body = cast(dict[str, Any], empty_cached_response.json()) + assert cast(list[dict[str, Any]], empty_cached_body["results"]) == [] + + first_source = cast(dict[str, Any], first_response.json()["sources"][0]) + retry_source = cast(dict[str, Any], retry_response.json()["sources"][0]) + document_id = str(first_source["document_id"]) + + assert first_source["status"] == "created" + assert retry_source["status"] == "existing" + assert retry_source["document_id"] == document_id + + materialization_rows = await ContractDatabase.fetch_all( + """ + SELECT demo_source_id, document_id + FROM demo_materializations + WHERE user_id = 'local-dev-user' + AND namespace = 'contract-demo' + AND demo_source_id = :demo_source_id + """, + {"demo_source_id": DEMO_SOURCE_ID}, + ) + document_row = await ContractDatabase.fetch_one( + """ + SELECT document_id, status, source_file_name + FROM documents + WHERE document_id = :document_id + """, + {"document_id": document_id}, + ) + chunk_rows = await ContractDatabase.fetch_all( + """ + SELECT id + FROM document_chunks + WHERE document_id = :document_id + """, + {"document_id": document_id}, + ) + job_rows = await ContractDatabase.fetch_all( + """ + SELECT job_id, status, job_type, credits_charged, billing_status + FROM jobs + WHERE user_id = 'local-dev-user' + AND job_metadata ->> 'demo_source_id' = :demo_source_id + """, + {"demo_source_id": DEMO_SOURCE_ID}, + ) + + assert materialization_rows == [ + {"demo_source_id": DEMO_SOURCE_ID, "document_id": document_id} + ] + assert document_row == { + "document_id": document_id, + "status": "active", + "source_file_name": "TSLA-Q4-2025-Update.pdf", + } + assert len(chunk_rows) == 70 + assert len(job_rows) == 1 + job_row = job_rows[0] + assert job_row["status"] == "done" + assert job_row["job_type"] == "demo_materialization" + assert job_row["credits_charged"] == 0 + assert job_row["billing_status"] == "skipped" + + retrieval_body = cast(dict[str, Any], retrieval_response.json()) + retrieval_results = cast(list[dict[str, Any]], retrieval_body["results"]) + chunk_page_body = cast(dict[str, Any], document_chunks_response.json()) + materialized_chunks = cast(list[dict[str, Any]], chunk_page_body["chunks"]) + media_chunks = [ + chunk + for chunk in materialized_chunks + if chunk["chunk_type"] in {"image", "table"} + ] + + assert retrieval_body["namespace"] == "contract-demo" + assert retrieval_results + assert retrieval_results[0]["source"]["document_id"] == document_id + assert retrieval_results[0]["source"]["section_path"] != "Root" + assert media_chunks + assert media_chunks[0]["file_path"] + uploaded_files = fake_result_storage.raw_files_by_job_id[str(job_row["job_id"])] + assert any(file_path.startswith("images/") for file_path in uploaded_files) + assert any(file_path.startswith("tables/") for file_path in uploaded_files) + + +@pytest.mark.asyncio +async def test_should_serialize_concurrent_first_demo_materialization( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], + monkeypatch: MonkeyPatch, +) -> None: + fake_result_storage = FakeResultStorage() + monkeypatch.setattr( + "shared.services.storage.result_storage.get_result_storage", + lambda: fake_result_storage, + ) + + async with developer_api_client_factory() as api_client: + first_response, second_response = await asyncio.gather( + api_client.post( + "/api/v1/demo/materializations", + json={ + "namespace": "contract-demo-race", + "demo_source_ids": [DEMO_SOURCE_ID], + }, + ), + api_client.post( + "/api/v1/demo/materializations", + json={ + "namespace": "contract-demo-race", + "demo_source_ids": [DEMO_SOURCE_ID], + }, + ), + ) + + assert first_response.status_code == 200 + assert second_response.status_code == 200 + + first_source = cast(dict[str, Any], first_response.json()["sources"][0]) + second_source = cast(dict[str, Any], second_response.json()["sources"][0]) + document_ids = { + str(first_source["document_id"]), + str(second_source["document_id"]), + } + statuses = {str(first_source["status"]), str(second_source["status"])} + + assert len(document_ids) == 1 + assert statuses == {"created", "existing"} + + materialization_rows = await ContractDatabase.fetch_all( + """ + SELECT demo_source_id, document_id + FROM demo_materializations + WHERE user_id = 'local-dev-user' + AND namespace = 'contract-demo-race' + AND demo_source_id = :demo_source_id + """, + {"demo_source_id": DEMO_SOURCE_ID}, + ) + job_rows = await ContractDatabase.fetch_all( + """ + SELECT job_id + FROM jobs + WHERE user_id = 'local-dev-user' + AND job_metadata ->> 'namespace' = 'contract-demo-race' + AND job_metadata ->> 'demo_source_id' = :demo_source_id + """, + {"demo_source_id": DEMO_SOURCE_ID}, + ) + + assert materialization_rows == [ + { + "demo_source_id": DEMO_SOURCE_ID, + "document_id": next(iter(document_ids)), + } + ] + assert len(job_rows) == 1 + + +@pytest.mark.asyncio +async def test_should_reject_blank_demo_materialization_selection( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], +) -> None: + async with developer_api_client_factory() as api_client: + response = await api_client.post( + "/api/v1/demo/materializations", + json={ + "namespace": "contract-demo-blank", + "demo_source_ids": [" ", "\t"], + }, + ) + + assert response.status_code == 400 + + +@pytest.mark.asyncio +async def test_should_reject_mixed_demo_materialization_selection_before_upload( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], + monkeypatch: MonkeyPatch, +) -> None: + fake_result_storage = FakeResultStorage() + monkeypatch.setattr( + "shared.services.storage.result_storage.get_result_storage", + lambda: fake_result_storage, + ) + + async with developer_api_client_factory() as api_client: + response = await api_client.post( + "/api/v1/demo/materializations", + json={ + "namespace": "contract-demo-mixed-invalid", + "demo_source_ids": [DEMO_SOURCE_ID, "missing-demo-source"], + }, + ) + + job_rows = await ContractDatabase.fetch_all( + """ + SELECT job_id + FROM jobs + WHERE user_id = 'local-dev-user' + AND job_metadata ->> 'namespace' = 'contract-demo-mixed-invalid' + """, + ) + materialization_rows = await ContractDatabase.fetch_all( + """ + SELECT demo_source_id, document_id + FROM demo_materializations + WHERE user_id = 'local-dev-user' + AND namespace = 'contract-demo-mixed-invalid' + """, + ) + + assert response.status_code == 404 + assert fake_result_storage.raw_files_by_job_id == {} + assert job_rows == [] + assert materialization_rows == [] diff --git a/apps/api/tests/contract/test_job_creation_contract.py b/apps/api/tests/contract/test_job_creation_contract.py index 7b4025b44..b6cfa1375 100644 --- a/apps/api/tests/contract/test_job_creation_contract.py +++ b/apps/api/tests/contract/test_job_creation_contract.py @@ -1,11 +1,12 @@ from collections.abc import Callable from contextlib import AbstractAsyncContextManager -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import json import socket from typing import cast from uuid import uuid4 +import jwt import pytest from httpx import AsyncClient from pytest import MonkeyPatch @@ -405,6 +406,53 @@ async def test_should_reject_a_malformed_authorization_header_when_creating_a_jo assert await _count_jobs() == 0 +@pytest.mark.asyncio +async def test_should_reject_authenticated_user_id_missing_from_user_table( + api_client_factory: Callable[[], AbstractAsyncContextManager[AsyncClient]], + monkeypatch: MonkeyPatch, +) -> None: + user_id = f"contract-missing-user-{uuid4().hex[:12]}" + jwt_secret = f"contract-jwt-secret-{uuid4().hex[:12]}" + token = jwt.encode( + { + "id": user_id, + "exp": datetime.now(timezone.utc) + timedelta(minutes=5), + }, + jwt_secret, + algorithm="HS256", + ) + payload: dict[str, str] = { + "namespace": "contract-jobs", + "source_type": "file", + "file_name": "contract-upload.pdf", + "data_id": "contract-job-missing-user", + } + + async with api_client_factory() as api_client: + from app.core import dependencies as auth_dependencies + + monkeypatch.setattr( + auth_dependencies, + "_get_verification_key", + lambda _token: jwt_secret, + ) + + api_client.headers.update({"Authorization": f"Bearer {token}"}) + response = await api_client.post("/api/v1/jobs", json=payload) + + assert response.status_code == 401 + assert response.headers["x-request-id"] + + response_json: dict[str, object] = response.json() + error = cast(dict[str, object], response_json["error"]) + + assert response_json["success"] is False + assert error["code"] == "UNAUTHENTICATED" + assert error["message"] == "Invalid authentication credentials" + assert "details" not in error + assert await _count_jobs() == 0 + + @pytest.mark.asyncio async def test_should_return_conflict_when_creating_a_job_for_a_document_with_an_active_ingestion_job( developer_api_client_factory: Callable[ diff --git a/apps/api/tests/contract/test_job_read_contract.py b/apps/api/tests/contract/test_job_read_contract.py index 58d07e048..136d20f89 100644 --- a/apps/api/tests/contract/test_job_read_contract.py +++ b/apps/api/tests/contract/test_job_read_contract.py @@ -1,5 +1,6 @@ from collections.abc import Callable from contextlib import AbstractAsyncContextManager +from datetime import datetime, timedelta, timezone from typing import cast from uuid import uuid4 @@ -67,6 +68,34 @@ async def test_should_list_created_jobs_for_the_authenticated_developer( assert job["credits_spent"] == 0.0 +@pytest.mark.asyncio +async def test_should_list_jobs_when_date_filters_use_iso_utc_timezone( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], +) -> None: + start_time = datetime.now(timezone.utc) - timedelta(days=1) + end_time = datetime.now(timezone.utc) + timedelta(days=1) + query_params = { + "start_time": start_time.isoformat().replace("+00:00", "Z"), + "end_time": end_time.isoformat().replace("+00:00", "Z"), + } + + async with developer_api_client_factory() as api_client: + created_job = await _create_waiting_file_job(api_client) + + response = await api_client.get("/api/v1/jobs/page", params=query_params) + + assert response.status_code == 200 + + response_json = cast(dict[str, object], response.json()) + jobs = cast(list[dict[str, object]], response_json["jobs"]) + + assert response_json["total"] == 1 + assert len(jobs) == 1 + assert jobs[0]["job_id"] == created_job["job_id"] + + @pytest.mark.asyncio async def test_should_return_job_details_for_an_existing_waiting_file_job( developer_api_client_factory: Callable[ diff --git a/apps/api/tests/contract/test_qstash_callback_contract.py b/apps/api/tests/contract/test_qstash_callback_contract.py index 73be267a5..9b6734d2d 100644 --- a/apps/api/tests/contract/test_qstash_callback_contract.py +++ b/apps/api/tests/contract/test_qstash_callback_contract.py @@ -11,7 +11,12 @@ from tests.support.contract_database import ContractDatabase -async def _insert_qstash_event() -> tuple[str, str]: +async def _insert_qstash_event( + *, + status: str = "pending", + attempts: int = 0, + qstash_message_id: str | None = None, +) -> tuple[str, str]: user_id = f"contract-qstash-user-{uuid4().hex[:12]}" job_id = f"job_{uuid4().hex[:12]}" event_id = str(uuid4()) @@ -29,6 +34,9 @@ async def _insert_qstash_event() -> tuple[str, str]: job_id=job_id, target_url="https://hooks.contract.test/qstash", payload={"job_id": job_id, "status": "done"}, + status=status, + attempts=attempts, + qstash_message_id=qstash_message_id, ) return job_id, event_id @@ -112,6 +120,133 @@ async def test_should_mark_the_matching_event_delivered_and_persist_a_webhook_lo } +@pytest.mark.asyncio +async def test_should_keep_the_matching_event_delivering_for_retry_callback_with_non_success_status( + api_client_factory: Callable[[], AbstractAsyncContextManager[AsyncClient]], + monkeypatch: MonkeyPatch, +) -> None: + job_id: str = "" + event_id: str = "" + + async with api_client_factory() as api_client: + job_id, event_id = await _insert_qstash_event( + status="delivering", + qstash_message_id="qstash-message-retry", + ) + qstash_module = importlib.import_module("app.api.v1.routes.qstash_callbacks") + monkeypatch.setattr(qstash_module, "_verify_qstash_signature", lambda *args: True) + response = await api_client.post( + "/api/v1/webhooks/qstash/callback", + json={ + "status": 503, + "body": "temporary unavailable", + "retried": 2, + "sourceMessageId": "qstash-message-retry", + "sourceHeader": {"X-Knowhere-Event-Id": event_id}, + }, + headers={"upstash-signature": "contract-valid"}, + ) + + assert response.status_code == 200 + assert response.text == "OK" + + event_row = await ContractDatabase.fetch_webhook_event(event_id) + log_rows = await ContractDatabase.fetch_all( + """ + SELECT + job_id, + event_id, + attempt_number, + response_status_code, + response_body, + error_message, + qstash_message_id + FROM webhook_logs + WHERE event_id = :event_id + """, + {"event_id": event_id}, + ) + + assert event_row is not None + assert event_row["status"] == "delivering" + assert event_row["attempts"] == 3 + + assert len(log_rows) == 1 + assert log_rows[0] == { + "job_id": job_id, + "event_id": event_id, + "attempt_number": 3, + "response_status_code": 503, + "response_body": "temporary unavailable", + "error_message": "temporary unavailable", + "qstash_message_id": "qstash-message-retry", + } + + +@pytest.mark.asyncio +async def test_should_not_downgrade_terminal_event_when_retry_callback_arrives_late( + api_client_factory: Callable[[], AbstractAsyncContextManager[AsyncClient]], + monkeypatch: MonkeyPatch, +) -> None: + job_id: str = "" + event_id: str = "" + + async with api_client_factory() as api_client: + job_id, event_id = await _insert_qstash_event( + status="delivered", + attempts=4, + qstash_message_id="qstash-message-late-retry", + ) + qstash_module = importlib.import_module("app.api.v1.routes.qstash_callbacks") + monkeypatch.setattr(qstash_module, "_verify_qstash_signature", lambda *args: True) + response = await api_client.post( + "/api/v1/webhooks/qstash/callback", + json={ + "status": 503, + "body": "late retry callback", + "retried": 1, + "sourceMessageId": "qstash-message-late-retry", + "sourceHeader": {"X-Knowhere-Event-Id": event_id}, + }, + headers={"upstash-signature": "contract-valid"}, + ) + + assert response.status_code == 200 + assert response.text == "OK" + + event_row = await ContractDatabase.fetch_webhook_event(event_id) + log_rows = await ContractDatabase.fetch_all( + """ + SELECT + job_id, + event_id, + attempt_number, + response_status_code, + response_body, + error_message, + qstash_message_id + FROM webhook_logs + WHERE event_id = :event_id + """, + {"event_id": event_id}, + ) + + assert event_row is not None + assert event_row["status"] == "delivered" + assert event_row["attempts"] == 4 + + assert len(log_rows) == 1 + assert log_rows[0] == { + "job_id": job_id, + "event_id": event_id, + "attempt_number": 2, + "response_status_code": 503, + "response_body": "late retry callback", + "error_message": "late retry callback", + "qstash_message_id": "qstash-message-late-retry", + } + + @pytest.mark.asyncio async def test_should_mark_the_matching_event_failed_and_persist_the_error_on_failure_callback( api_client_factory: Callable[[], AbstractAsyncContextManager[AsyncClient]], diff --git a/apps/worker/app/core/tasks/kb_tasks.py b/apps/worker/app/core/tasks/kb_tasks.py index b9fed9f19..19acc3605 100644 --- a/apps/worker/app/core/tasks/kb_tasks.py +++ b/apps/worker/app/core/tasks/kb_tasks.py @@ -644,26 +644,6 @@ def _parse(job_id: str, user_id: str | None): metadata_service.update_metadata(job_id, processing_timing_updates) job_metadata.update(processing_timing_updates) - # 1.5. Garbage Collection: Remove redundant local media files - try: - from shared.services.retrieval.publication_service import RetrievalPublicationService - - with get_sync_db_context() as db: - job_record = db.execute(select(Job).where(Job.job_id == job_id)).scalar_one_or_none() - if job_record: - gc_namespace = JobMetadataHelper.get_field(job_metadata, "namespace") or "default" - chunks, dedup_stats = RetrievalPublicationService.garbage_collect_and_dedup_local_media( - db, - job_id=job_id, - user_id=str(job_record.user_id), - namespace=gc_namespace, - add_dir=str(add_dir) if add_dir else "", - chunks=chunks, - ) - except Exception as e: - logger.error(f"[{job_id}] GC failed (non-fatal): {e}") - dedup_stats = None - # Generate ZIP package zip_service = ZipResultService() zip_file_path, checksum, statistics, zip_size = ( @@ -712,7 +692,6 @@ def _parse(job_id: str, user_id: str | None): stored_count=stored_count, delivery_mode="url", section_summaries=section_summaries, - chunk_dedup_stats=dedup_stats, ) logger.info( diff --git a/apps/worker/app/core/tasks/webhook_tasks.py b/apps/worker/app/core/tasks/webhook_tasks.py index 55c5b80c4..0e046eca7 100644 --- a/apps/worker/app/core/tasks/webhook_tasks.py +++ b/apps/worker/app/core/tasks/webhook_tasks.py @@ -5,6 +5,10 @@ published through QStash, which owns retries and callback handling. """ +from datetime import datetime, timezone +from typing import Any +from uuid import NAMESPACE_URL, uuid5 + from loguru import logger from shared.core.celery_app import get_celery_app @@ -15,16 +19,93 @@ # Matches the beat_schedule period in celery_app.py _WEBHOOK_RECOVERY_PERIOD_SECONDS = 1800 +_WEBHOOK_CALLBACK_TEXT_LIMIT = 4096 celery_app = get_celery_app() +def _build_reconciliation_log_idempotency_key( + qstash_message_id: str, + event_id: str, + status: str, +) -> str: + """Build a fixed-width idempotency key for reconstructed QStash logs.""" + return str(uuid5(NAMESPACE_URL, f"{qstash_message_id}:{event_id}:{status}")) + + +def _truncate_callback_text(value: str | None) -> str | None: + """Trim QStash log text to the database column limit used by callbacks.""" + if not value: + return None + + return value[:_WEBHOOK_CALLBACK_TEXT_LIMIT] + + +def _reconcile_stale_delivering_events( + db: Any, + publisher: Any, + cutoff_time: datetime, +) -> int: + """Reconcile stale delivering events whose QStash message is terminal.""" + from sqlalchemy import select as sa_select + + from shared.models.database.webhook import WebhookEvent, WebhookEventStatus + from shared.models.database.webhook_log import WebhookLog + + result = db.execute( + sa_select(WebhookEvent) + .where( + WebhookEvent.status == WebhookEventStatus.DELIVERING, + WebhookEvent.qstash_message_id.is_not(None), + WebhookEvent.updated_at < cutoff_time, + ) + .limit(100) + ) + stale_events = result.scalars().all() + reconciled = 0 + + for event in stale_events: + qstash_message_id = str(event.qstash_message_id) + delivery_status = publisher.get_terminal_delivery_status(qstash_message_id) + if delivery_status is None: + continue + + event.status = delivery_status.status + event.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) + + db.add( + WebhookLog( + job_id=event.job_id, + event_id=event.id, + webhook_url=event.target_url, + attempt_number=max(event.attempts, 1), + request_payload=event.payload, + signature="", + idempotency_key=_build_reconciliation_log_idempotency_key( + qstash_message_id, + event.id, + delivery_status.status, + ), + response_status_code=delivery_status.response_status_code, + response_body=_truncate_callback_text(delivery_status.response_body), + error_message=_truncate_callback_text(delivery_status.error_message), + duration_ms=0, + delivery_provider="qstash", + qstash_message_id=qstash_message_id, + ) + ) + reconciled += 1 + + return reconciled + + @celery_app.task(name="app.core.tasks.webhook_tasks.recover_orphaned_webhooks") def recover_orphaned_webhooks() -> dict: """Periodic task to recover orphaned webhook events. Finds PENDING events with attempts=0 older than 5 minutes and republishes - them via QStash. + them via QStash. Also reconciles stale DELIVERING events when QStash logs + show a terminal result but the callback did not update the database. """ from datetime import datetime, timedelta, timezone @@ -48,6 +129,7 @@ def recover_orphaned_webhooks() -> dict: minutes=age_minutes ) recovered = 0 + reconciled = 0 publisher = get_qstash_webhook_publisher() try: @@ -81,14 +163,27 @@ def recover_orphaned_webhooks() -> dict: except Exception as e: logger.error(f"Error recovering webhook event {event.id}: {e}") + reconciled = _reconcile_stale_delivering_events( + db, + publisher, + cutoff_time, + ) + if recovered > 0: logger.bind(event=LogEvent.WORKER_TASK_COMPLETE.value).info( f"Recovered {recovered} orphaned webhook events via QStash" ) - else: - logger.debug("No orphaned webhook events found") + if reconciled > 0: + logger.bind(event=LogEvent.WORKER_TASK_COMPLETE.value).info( + f"Reconciled {reconciled} stale webhook events from QStash logs" + ) + if recovered == 0 and reconciled == 0: + logger.debug("No orphaned or stale webhook events found") - return {"status": "success", "recovered": recovered, "provider": "qstash"} + result = {"status": "success", "recovered": recovered, "provider": "qstash"} + if reconciled: + result["reconciled"] = reconciled + return result except Exception as e: logger.error(f"Orphaned webhook recovery job failed: {e}", exc_info=True) diff --git a/apps/worker/tests/contract/test_parse_task_contract.py b/apps/worker/tests/contract/test_parse_task_contract.py index a5c0aee0a..12f672267 100644 --- a/apps/worker/tests/contract/test_parse_task_contract.py +++ b/apps/worker/tests/contract/test_parse_task_contract.py @@ -568,6 +568,395 @@ def upload(self, *, job_id: str, result_dir: str, zip_file_path: str) -> Any: ] +def test_should_export_full_result_when_publication_deduplicates_existing_chunks( + worker_contract_environment: None, + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("BILLING_ENABLED", "false") + ( + kb_tasks, + parse_service, + sync_storage_service, + engine, + sync_job_info_service_cls, + sync_job_metadata_service_cls, + sync_redis_service_factory, + ) = _load_parse_task_modules() + + user_id: str = f"worker-user-{uuid4().hex[:12]}" + existing_job_id: str = f"job_existing_{uuid4().hex[:12]}" + existing_result_id: str = str(uuid4()) + existing_document_id: str = f"doc_{uuid4().hex[:12]}" + job_id: str = f"job_parse_dedup_{uuid4().hex[:12]}" + source_file_name: str = "dedup-export.pdf" + s3_key: str = f"uploads/{job_id}.pdf" + job_metadata = _build_pending_file_job_metadata(source_file_name) + captured_artifacts: dict[str, Any] = {} + + with engine.begin() as connection: + insert_contract_user(connection, user_id=user_id) + insert_contract_job( + connection, + job_id=existing_job_id, + user_id=user_id, + status="done", + source_type="file", + s3_key=f"uploads/{existing_job_id}.pdf", + webhook_enabled=False, + job_metadata=_build_pending_file_job_metadata("existing.pdf"), + billing_status="skipped", + ) + connection.execute( + text( + """ + INSERT INTO job_results ( + id, + job_id, + delivery_mode, + inline_payload, + result_s3_key, + result_size, + created_at, + updated_at + ) VALUES ( + :result_id, + :job_id, + 'url', + CAST(:inline_payload AS JSON), + :result_s3_key, + :result_size, + NOW(), + NOW() + ) + """ + ), + { + "result_id": existing_result_id, + "job_id": existing_job_id, + "inline_payload": json.dumps({"checksum": "existing"}), + "result_s3_key": f"results/{existing_job_id}.zip", + "result_size": 123, + }, + ) + connection.execute( + text( + """ + INSERT INTO documents ( + document_id, + user_id, + namespace, + status, + current_job_result_id, + source_file_name, + created_at, + updated_at + ) VALUES ( + :document_id, + :user_id, + 'worker-contract', + 'active', + :result_id, + 'existing.pdf', + NOW(), + NOW() + ) + """ + ), + { + "document_id": existing_document_id, + "user_id": user_id, + "result_id": existing_result_id, + }, + ) + connection.execute( + text( + """ + UPDATE job_results + SET document_id = :document_id + WHERE id = :result_id + """ + ), + { + "document_id": existing_document_id, + "result_id": existing_result_id, + }, + ) + connection.execute( + text( + """ + INSERT INTO document_chunks ( + id, + chunk_id, + user_id, + namespace, + document_id, + job_result_id, + chunk_type, + content, + source_chunk_path, + file_path, + chunk_metadata, + sort_order, + created_at + ) VALUES + ( + :text_id, + 'duplicate-text', + :user_id, + 'worker-contract', + :document_id, + :result_id, + 'text', + 'already published text', + 'Default_Root/existing.pdf/Section/Duplicate text', + NULL, + CAST(:text_metadata AS JSON), + 0, + NOW() + ), + ( + :image_id, + 'duplicate-image', + :user_id, + 'worker-contract', + :document_id, + :result_id, + 'image', + 'already published image', + 'Default_Root/existing.pdf/images/duplicate.png', + 'images/duplicate.png', + CAST(:image_metadata AS JSON), + 1, + NOW() + ) + """ + ), + { + "text_id": f"dchk_{uuid4().hex[:12]}", + "image_id": f"dchk_{uuid4().hex[:12]}", + "user_id": user_id, + "document_id": existing_document_id, + "result_id": existing_result_id, + "text_metadata": json.dumps({}), + "image_metadata": json.dumps({"file_path": "images/duplicate.png"}), + }, + ) + insert_contract_job( + connection, + job_id=job_id, + user_id=user_id, + status="pending", + source_type="file", + s3_key=s3_key, + webhook_enabled=False, + job_metadata=job_metadata, + billing_status="pending", + ) + + _save_worker_task_cache( + job_id=job_id, + user_id=user_id, + s3_key=s3_key, + metadata=job_metadata, + sync_job_info_service_cls=sync_job_info_service_cls, + sync_job_metadata_service_cls=sync_job_metadata_service_cls, + sync_redis_service_factory=sync_redis_service_factory, + ) + + _bind_parse_task_to_current_module(monkeypatch, kb_tasks=kb_tasks) + monkeypatch.setattr(kb_tasks.settings, "TMP_PATH", str(tmp_path)) + monkeypatch.setattr(kb_tasks.settings, "BILLING_ENABLED", False) + + def fake_cleanup_task_workspace(workspace_dir: str | None) -> bool: + captured_artifacts["workspace_dir"] = workspace_dir + return True + + def fake_verify_s3_file_exists(storage_key: str) -> dict[str, Any]: + return { + "exists": storage_key == s3_key, + "size": _SAMPLE_PDF_PATH.stat().st_size, + } + + def fake_generate_download_url(storage_key: str, bucket: str | None) -> dict[str, str]: + return {"download_url": f"https://example.test/{storage_key}"} + + def fake_download_s3_file_to_temp( + file_url: str, file_ext: str, temp_dir: str + ) -> str: + downloaded_path = Path(temp_dir) / f"downloaded{file_ext}" + shutil.copy2(_SAMPLE_PDF_PATH, downloaded_path) + return str(downloaded_path) + + def fake_checkerboard_inject_parse(**kwargs: Any) -> tuple[str, pd.DataFrame]: + output_dir = ( + Path(str(kwargs["output_dir"])) + / str(kwargs["kb_dir"]) + / str(kwargs["internal_output_filename"]) + ) + images_dir = output_dir / "images" + images_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "full.md").write_text("body", encoding="utf-8") + (images_dir / "duplicate.png").write_bytes(b"png") + + file_root = str(kwargs["internal_output_filename"]) + parsed_rows: list[dict[str, Any]] = [ + { + "content": "duplicate text", + "path": f"Default_Root/{file_root}/Section/Duplicate text", + "type": "text", + "length": 14, + "keywords": "", + "summary": "", + "know_id": "duplicate-text", + "tokens": "", + "connectto": "", + "addtime": "now", + "page_nums": "1", + }, + { + "content": "duplicate image", + "path": f"Default_Root/{file_root}/images/duplicate.png", + "type": "image", + "length": 15, + "keywords": "", + "summary": "", + "know_id": "duplicate-image", + "tokens": "", + "connectto": "", + "addtime": "now", + "page_nums": "2", + }, + { + "content": "new text", + "path": f"Default_Root/{file_root}/Section/New text", + "type": "text", + "length": 8, + "keywords": "", + "summary": "", + "know_id": "new-text", + "tokens": "", + "connectto": "", + "addtime": "now", + "page_nums": "3", + }, + ] + return str(output_dir), pd.DataFrame(parsed_rows) + + class FakeResultStorage: + def upload(self, *, job_id: str, result_dir: str, zip_file_path: str) -> Any: + result_dir_path = Path(result_dir) + zip_path = Path(zip_file_path) + captured_artifacts["raw_entries"] = sorted( + path.relative_to(result_dir_path).as_posix() + for path in result_dir_path.rglob("*") + if path.is_file() + ) + with zipfile.ZipFile(zip_path) as zip_file: + captured_artifacts["zip_entries"] = sorted(zip_file.namelist()) + captured_artifacts["zip_chunks"] = json.loads( + zip_file.read("chunks.json") + )["chunks"] + captured_artifacts["manifest"] = json.loads( + zip_file.read("manifest.json") + ) + + return SimpleNamespace( + zip_key=f"results/{job_id}.zip", + raw_prefix=f"results/{job_id}/", + raw_files={}, + ) + + monkeypatch.setattr(kb_tasks, "verify_s3_file_exists", fake_verify_s3_file_exists) + monkeypatch.setattr( + sync_storage_service, + "verify_s3_file_exists", + fake_verify_s3_file_exists, + ) + monkeypatch.setattr(kb_tasks, "generate_download_url", fake_generate_download_url) + monkeypatch.setattr( + sync_storage_service, + "generate_download_url", + fake_generate_download_url, + ) + monkeypatch.setattr(kb_tasks, "download_s3_file_to_temp", fake_download_s3_file_to_temp) + monkeypatch.setattr(parse_service, "checkerboard_inject_parse", fake_checkerboard_inject_parse) + monkeypatch.setattr(kb_tasks, "get_result_storage", lambda: FakeResultStorage()) + monkeypatch.setattr(kb_tasks, "cleanup_task_workspace", fake_cleanup_task_workspace) + + result = kb_tasks.parse_task.run(job_id, user_id, "kb_management") + + assert result["contents_count"] == 3 + assert "images/duplicate.png" in captured_artifacts["zip_entries"] + assert "images/duplicate.png" in captured_artifacts["raw_entries"] + assert [chunk["chunk_id"] for chunk in captured_artifacts["zip_chunks"]] == [ + "duplicate-text", + "duplicate-image", + "new-text", + ] + assert captured_artifacts["manifest"]["statistics"] == { + "total_chunks": 3, + "text_chunks": 2, + "image_chunks": 1, + "table_chunks": 0, + "total_pages": None, + } + workspace_dir = Path(str(captured_artifacts["workspace_dir"])) + assert list(workspace_dir.rglob("images/duplicate.png")) + + with engine.begin() as connection: + job_result_row = ( + connection.execute( + text( + """ + SELECT id, document_metadata + FROM job_results + WHERE job_id = :job_id + """ + ), + {"job_id": job_id}, + ) + .mappings() + .one() + ) + job_chunk_ids = list( + connection.execute( + text( + """ + SELECT chunk_id + FROM job_chunks + WHERE job_result_id = :job_result_id + ORDER BY sort_order + """ + ), + {"job_result_id": job_result_row["id"]}, + ) + .scalars() + .all() + ) + document_chunk_ids = list( + connection.execute( + text( + """ + SELECT document_chunks.chunk_id + FROM document_chunks + JOIN documents + ON documents.document_id = document_chunks.document_id + WHERE documents.current_job_result_id = :job_result_id + ORDER BY document_chunks.sort_order + """ + ), + {"job_result_id": job_result_row["id"]}, + ) + .scalars() + .all() + ) + + assert job_chunk_ids == ["duplicate-text", "duplicate-image", "new-text"] + assert document_chunk_ids == ["duplicate-text", "duplicate-image", "new-text"] + assert "chunk_overlap" not in dict(job_result_row["document_metadata"] or {}) + + def test_should_initialize_billing_once_for_concurrent_parse_tasks( worker_contract_environment: None, monkeypatch: MonkeyPatch, diff --git a/apps/worker/tests/contract/test_webhook_recovery_contract.py b/apps/worker/tests/contract/test_webhook_recovery_contract.py index dc9266886..b30633b9f 100644 --- a/apps/worker/tests/contract/test_webhook_recovery_contract.py +++ b/apps/worker/tests/contract/test_webhook_recovery_contract.py @@ -34,6 +34,8 @@ def _insert_webhook_event( status: str, attempts: int, created_at: datetime, + updated_at: datetime | None = None, + qstash_message_id: str | None = None, ) -> None: connection.execute( text( @@ -71,9 +73,9 @@ def _insert_webhook_event( "status": status, "attempts": attempts, "next_retry_at": None, - "qstash_message_id": None, + "qstash_message_id": qstash_message_id, "created_at": created_at, - "updated_at": created_at, + "updated_at": updated_at or created_at, }, ) @@ -103,11 +105,13 @@ def test_should_republish_only_orphaned_pending_webhook_events_and_persist_qstas retried_event_id = str(uuid4()) terminal_event_id = str(uuid4()) published_message_ids: list[str] = [] + published_calls: list[dict[str, Any]] = [] publisher = qstash_publisher.QStashWebhookPublisher() class FakeMessageClient: def publish(self, **kwargs: Any) -> SimpleNamespace: + published_calls.append(kwargs) message_id = f"msg_{kwargs['headers']['X-Knowhere-Event-ID']}" published_message_ids.append(message_id) return SimpleNamespace(message_id=message_id) @@ -199,6 +203,8 @@ def publish(self, **kwargs: Any) -> SimpleNamespace: "provider": "qstash", } assert published_message_ids == [f"msg_{orphaned_event_id}"] + assert published_calls[0]["deduplication_id"] == orphaned_event_id + assert published_calls[0]["label"] == "knowhere-webhook" with engine.begin() as connection: event_rows = connection.execute( @@ -265,6 +271,133 @@ def publish(self, **kwargs: Any) -> SimpleNamespace: assert secrets_count_row["secrets_count"] == 1 +def test_should_reconcile_stale_delivering_webhook_events_from_qstash_logs( + worker_contract_environment: None, + monkeypatch: MonkeyPatch, +) -> None: + webhook_tasks, qstash_publisher, engine = _load_worker_modules() + + user_id = f"worker-user-{uuid4().hex[:12]}" + target_url = "https://hooks.contract.test/worker" + job_id = f"job_stale_{uuid4().hex[:12]}" + event_id = str(uuid4()) + qstash_message_id = f"msg_{event_id}" + + class FakePublisher: + def publish_event(self, event_id: str) -> None: + raise AssertionError(f"stale delivering event should not republish: {event_id}") + + def get_terminal_delivery_status( + self, + message_id: str, + ) -> Any: + assert message_id == qstash_message_id + return qstash_publisher.QStashDeliveryStatus( + status="delivered", + response_status_code=204, + response_body="", + error_message=None, + ) + + monkeypatch.setattr( + qstash_publisher, + "get_qstash_webhook_publisher", + lambda: FakePublisher(), + ) + + now = _utc_now() + with engine.begin() as connection: + insert_contract_user(connection, user_id=user_id) + insert_contract_job( + connection, + job_id=job_id, + user_id=user_id, + status="done", + source_type="file", + webhook_url=target_url, + webhook_enabled=True, + job_metadata=_build_file_job_metadata(), + billing_status="charged", + ) + _insert_webhook_event( + connection, + event_id=event_id, + job_id=job_id, + target_url=target_url, + status="delivering", + attempts=2, + created_at=now - timedelta(minutes=20), + updated_at=now - timedelta(minutes=10), + qstash_message_id=qstash_message_id, + ) + + result = webhook_tasks.recover_orphaned_webhooks() + + assert result == { + "status": "success", + "recovered": 0, + "provider": "qstash", + "reconciled": 1, + } + + with engine.begin() as connection: + event_row = ( + connection.execute( + text( + """ + SELECT id, status, attempts, qstash_message_id + FROM webhook_events + WHERE id = :event_id + """ + ), + {"event_id": event_id}, + ) + .mappings() + .one() + ) + log_row = ( + connection.execute( + text( + """ + SELECT + job_id, + event_id, + webhook_url, + attempt_number, + response_status_code, + response_body, + error_message, + delivery_provider, + qstash_message_id + FROM webhook_logs + WHERE event_id = :event_id + """ + ), + {"event_id": event_id}, + ) + .mappings() + .one() + ) + + assert dict(event_row) == { + "id": event_id, + "status": "delivered", + "attempts": 2, + "qstash_message_id": qstash_message_id, + } + assert dict(log_row) == { + "job_id": job_id, + "event_id": event_id, + "webhook_url": target_url, + "attempt_number": 2, + "response_status_code": 204, + "response_body": None, + "error_message": None, + "delivery_provider": "qstash", + "qstash_message_id": qstash_message_id, + } + + def test_should_skip_duplicate_beat_firing_for_webhook_recovery( worker_contract_environment: None, ) -> None: diff --git a/packages/shared-python/shared/core/config/qstash.py b/packages/shared-python/shared/core/config/qstash.py index 33dfb4bb1..04372e80a 100644 --- a/packages/shared-python/shared/core/config/qstash.py +++ b/packages/shared-python/shared/core/config/qstash.py @@ -12,6 +12,10 @@ class QStashConfig(BaseModel): # QStash API credentials (from Upstash console) QSTASH_TOKEN: Optional[str] = Field(default=None, description="QStash API token") + QSTASH_BASE_URL: Optional[str] = Field( + default=None, + description="QStash API base URL override for local development and tests", + ) QSTASH_CURRENT_SIGNING_KEY: Optional[str] = Field( default=None, description="QStash current signing key for callback verification" ) diff --git a/packages/shared-python/shared/models/database/__init__.py b/packages/shared-python/shared/models/database/__init__.py index 261dd9992..a0aba9d3d 100644 --- a/packages/shared-python/shared/models/database/__init__.py +++ b/packages/shared-python/shared/models/database/__init__.py @@ -20,6 +20,7 @@ RetrievalRun, RetrievalStep, ) +from .demo_materialization import DemoMaterialization from .guest_device import GuestDevice from .job import Job from .job_result import JobChunk, JobResult @@ -53,6 +54,7 @@ "Document", "DocumentSection", "DocumentChunk", + "DemoMaterialization", "GraphNode", "GraphEdge", "RetrievalHitStat", diff --git a/packages/shared-python/shared/models/database/demo_materialization.py b/packages/shared-python/shared/models/database/demo_materialization.py new file mode 100644 index 000000000..d4cc1a7a1 --- /dev/null +++ b/packages/shared-python/shared/models/database/demo_materialization.py @@ -0,0 +1,55 @@ +"""Canonical demo document materialization state.""" + +from __future__ import annotations + +from datetime import datetime +from uuid import uuid4 + +from sqlalchemy import DateTime, ForeignKey, Index, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from shared.core.database import Base +from shared.utils.utc_now import utc_now_naive + + +class DemoMaterialization(Base): + """Maps a canonical demo source to one user's retrieval document copy.""" + + __tablename__ = "demo_materializations" + + id: Mapped[str] = mapped_column( + String(36), + primary_key=True, + default=lambda: f"demo_mat_{uuid4().hex[:12]}", + ) + user_id: Mapped[str] = mapped_column( + Text, ForeignKey("user.id", ondelete="RESTRICT"), nullable=False + ) + namespace: Mapped[str] = mapped_column( + String(255), nullable=False, default="default" + ) + demo_source_id: Mapped[str] = mapped_column(String(128), nullable=False) + document_id: Mapped[str] = mapped_column( + String(36), + ForeignKey("documents.document_id", ondelete="CASCADE"), + nullable=False, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime, default=utc_now_naive, nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, + default=utc_now_naive, + onupdate=utc_now_naive, + nullable=False, + ) + + __table_args__ = ( + UniqueConstraint( + "user_id", + "namespace", + "demo_source_id", + name="uq_demo_materializations_scope_source", + ), + Index("idx_demo_materializations_document", "document_id"), + ) diff --git a/packages/shared-python/shared/services/billing/credits_service.py b/packages/shared-python/shared/services/billing/credits_service.py index e4c8e03c8..d0dc48fc4 100644 --- a/packages/shared-python/shared/services/billing/credits_service.py +++ b/packages/shared-python/shared/services/billing/credits_service.py @@ -16,7 +16,9 @@ from shared.core.billing import MicroDollar from shared.core.config import settings -from shared.core.exceptions.domain_exceptions import InsufficientCreditsException +from shared.core.exceptions.domain_exceptions import ( + InsufficientCreditsException, +) from shared.core.logging import logger from shared.models.database.credits_transaction import CreditsTransaction from shared.models.database.payment_record import PaymentRecord diff --git a/packages/shared-python/shared/services/billing/credits_sync_service.py b/packages/shared-python/shared/services/billing/credits_sync_service.py index ba6ddbe38..e6eb63446 100644 --- a/packages/shared-python/shared/services/billing/credits_sync_service.py +++ b/packages/shared-python/shared/services/billing/credits_sync_service.py @@ -11,7 +11,9 @@ from shared.core.billing import MicroDollar from shared.core.config import settings -from shared.core.exceptions.domain_exceptions import InsufficientCreditsException +from shared.core.exceptions.domain_exceptions import ( + InsufficientCreditsException, +) from shared.core.logging import logger from shared.models.database.credits_transaction import CreditsTransaction from shared.models.database.payment_record import PaymentRecord diff --git a/packages/shared-python/shared/services/job_lifecycle_sync.py b/packages/shared-python/shared/services/job_lifecycle_sync.py index b0b7b197d..cd9dcc8ef 100644 --- a/packages/shared-python/shared/services/job_lifecycle_sync.py +++ b/packages/shared-python/shared/services/job_lifecycle_sync.py @@ -60,16 +60,16 @@ def finalize_job_success( stored_count: int = 0, delivery_mode: str = "url", section_summaries: Optional[Dict[str, str]] = None, - chunk_dedup_stats: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Finalize a successful job in a single atomic transaction. Steps (all within one DB transaction): - 1. Upsert JobResult + replace chunks - 2. Mark job as DONE via state machine (CAS) - 3. Create WebhookEvent if webhook_enabled - 4. COMMIT - 5. Post-commit: enqueue webhook + 1. Upsert JobResult + replace full result chunks + 2. Publish document state from full result chunks + 3. Mark job as DONE via state machine (CAS) + 4. Create WebhookEvent if webhook_enabled + 5. COMMIT + 6. Post-commit: enqueue webhook """ logger.info(f"Finalizing job success: job_id={job_id}") @@ -83,7 +83,6 @@ def finalize_job_success( inline_payload=inline_payload, result_s3_key=result_s3_key, result_size=zip_size, - chunk_dedup_stats=chunk_dedup_stats, ) normalized_chunks = chunks or [] @@ -306,7 +305,6 @@ def _upsert_job_result( inline_payload: Optional[Dict[str, Any]] = None, result_s3_key: Optional[str] = None, result_size: Optional[int] = None, - chunk_dedup_stats: Optional[Dict[str, Any]] = None, ) -> JobResult: """Create or update JobResult row.""" result = db.execute(select(JobResult).where(JobResult.job_id == job_id)) @@ -314,24 +312,16 @@ def _upsert_job_result( if existing: existing.delivery_mode = delivery_mode - doc_meta = existing.document_metadata or {} - if chunk_dedup_stats: - doc_meta["chunk_dedup"] = chunk_dedup_stats - existing.document_metadata = doc_meta existing.inline_payload = inline_payload existing.result_s3_key = result_s3_key existing.result_size = result_size db.flush() return existing - doc_meta = {} - if chunk_dedup_stats: - doc_meta["chunk_dedup"] = chunk_dedup_stats - job_result = JobResult( job_id=job_id, delivery_mode=delivery_mode, - document_metadata=doc_meta, + document_metadata={}, inline_payload=inline_payload, result_s3_key=result_s3_key, result_size=result_size, diff --git a/packages/shared-python/shared/services/retrieval/agent_navigate.py b/packages/shared-python/shared/services/retrieval/agent_navigate.py index 824cf2e0c..3f7a2d316 100644 --- a/packages/shared-python/shared/services/retrieval/agent_navigate.py +++ b/packages/shared-python/shared/services/retrieval/agent_navigate.py @@ -21,6 +21,7 @@ _FILE_SELECT_PROMPT = """\ You are a document routing assistant. +{budget_block} Below is a knowledge base overview showing all available documents, their navigation summaries, chunk counts, and media counts. @@ -41,8 +42,9 @@ You are a document navigation assistant. Document: "{doc_name}" (id: {doc_id}) -{scope_header} +{budget_block} +{scope_header} Below is the document's section tree. Sections tagged [SELECT] are within the current scope and may be selected. Other sections are shown as structural context only (not selectable). @@ -58,6 +60,8 @@ - You may ONLY select sections marked with [SELECT]. Do NOT select any other sections. - Select sections whose content is needed to answer the query. - If the titles and summaries already visible are sufficient (e.g. the query asks for an outline or overview), return an EMPTY list []. +- When budget is TIGHT, prefer fewer high-confidence selections over broad exploration. +- When budget is CRITICAL, be very selective — only pick paths with strong relevance. Return [] if current evidence already suffices. Return ONLY a JSON object: {{"selections": [{{"path": "...", "confidence": }}, ...]}} @@ -70,6 +74,7 @@ Document: "{doc_name}" +{budget_block} After navigating the document's section tree, the following section paths were additionally discovered via keyword and semantic search. They may contain relevant evidence not found through hierarchical navigation. @@ -82,6 +87,8 @@ {revision_context} Select section paths whose content is needed to answer the query. If none are relevant, return an EMPTY list []. +When budget is TIGHT, prefer fewer high-confidence candidates. +When budget is CRITICAL, be very selective — only pick paths with strong relevance. Return [] if evidence suffices. Return ONLY a JSON object: {{"selections": [{{"path": "...", "confidence": }}, ...]}} @@ -89,6 +96,27 @@ """ +def _format_budget_block(snapshot: dict | None) -> str: + if not snapshot: + return "" + planning = snapshot.get("planning") or {} + context = snapshot.get("context") or {} + return ( + "=== Resource Status ===\n" + f"Planning Budget: {planning.get('status', 'HEALTHY')} " + f"({planning.get('used_pct', 0)}% used)\n" + f"Context Budget: {context.get('status', 'HEALTHY')} " + f"({context.get('used_pct', 0)}% used)\n" + f"KG Coverage: {snapshot.get('explored_chunks', 0)}/" + f"{snapshot.get('total_chunks', 0)} chunks explored\n" + f"Docs Explored: {snapshot.get('explored_docs', 0)}/" + f"{snapshot.get('total_docs', 0)}\n" + "When budget is TIGHT, prefer fewer high-confidence selections over broad exploration. " + "When CRITICAL, be very selective — only pick paths with strong relevance. Return empty if evidence suffices.\n" + "=== End Resource Status ===\n" + ) + + def _extract_json_array_payload(text: str) -> list[Any]: """Best-effort extraction of a JSON array payload from LLM response text.""" text = text.strip() @@ -473,6 +501,12 @@ async def _load_child_sections( scope_parts = split_section_path(scope) scope_depth = len(scope_parts) + logger.debug( + f' _load_child_sections: scope={scope!r} scope_parts={scope_parts} ' + f'scope_depth={scope_depth} exclude_paths={_excl if (_excl := exclude_paths or set()) else "none"} ' + f'total_sections={len(section_rows)}' + ) + # Build full section metadata index all_sections: dict[str, dict] = {} # path → {title, summary, sort_order, section_id, parts, depth} for section_id, title, path, summary, sort_order in section_rows: @@ -571,12 +605,16 @@ async def _load_child_sections( continue # Category 2: Descendants of scope_path (children to explore) - if parts[:scope_depth] == scope_parts and depth > scope_depth: + # depth > scope_depth is guaranteed by the continue at line above + is_descendant = parts[:scope_depth] == scope_parts + if is_descendant: # Skip excluded paths - if _excl and any( + is_excluded = _excl and any( path == ep or path.startswith(ep + ' / ') for ep in _excl - ): + ) + if is_excluded: + logger.debug(f' _load_child_sections: EXCLUDED descendant path={path!r}') continue scope_child_depths.add(depth) items_by_path[path] = { @@ -592,6 +630,11 @@ async def _load_child_sections( 'show_summary': True, } continue + else: + logger.debug( + f' _load_child_sections: NOT descendant path={path!r} ' + f'parts[:scope_depth]={parts[:scope_depth]} != scope_parts={scope_parts}' + ) # Category 3: Everything else → pruned (not added) @@ -944,57 +987,98 @@ def render_unified_doc_tree( if depth == 0: parts.append(f'【文档】{doc_name}\n') - # Track which paths have been rendered via outline_items - rendered_paths: set[str] = set() - # Collect children keys for path-hierarchy dedup: child_prefixes = set(node.children.keys()) + # Helper: min sort_order of a leaf_content entry + def _min_sort(path: str) -> float: + chunks = node.leaf_content.get(path, []) + return min((c.get('sort_order') or float('inf') for c in chunks), default=float('inf')) + + # ── Build a unified render queue ── + # Each entry: (sort_key, render_type, data) + # render_type: 'outline' | 'orphan_leaf' | 'orphan_child' + render_queue: list[tuple[float, str, dict | str]] = [] + + outline_paths: set[str] = set() + # Position counter for outline-only items (no leaf content) to preserve + # their relative ordering among themselves. + outline_position = 0.0 + for item in node.outline_items: path = item.get('path', '') - title = item.get('title', '') - is_leaf = item.get('is_leaf', False) - level = item.get('level', 1) - leaf_tag = ' [Leaf]' if is_leaf else '' - - # Skip items that belong to a drilled-into child's subtree + # Skip items belonging to a drilled-into child's subtree if any(path.startswith(cp + ' / ') for cp in child_prefixes): continue + outline_paths.add(path) - rendered_paths.add(path) - - # Section header (title only — summaries are navigation aids, not evidence) - level_tag = f'[L{level}] ' if level else '' - if level <= 1: - parts.append(f'{indent}▸ {level_tag}{title}{leaf_tag}') + # Determine sort_key: use chunk sort_order if content exists, + # else use a synthetic position to maintain outline ordering. + if path in node.leaf_content or path in node.children: + sort_key = _min_sort(path) if path in node.leaf_content else outline_position else: - parts.append(f'{indent}└ {level_tag}{title}{leaf_tag}') + sort_key = outline_position + outline_position = max(outline_position, sort_key) + 0.001 - sub_indent = indent + ' ' + render_queue.append((sort_key, 'outline', item)) - # Case 1: This section was drilled into → show child tree inline - if path in node.children: - child = node.children[path] - child_text = render_unified_doc_tree(child, doc_name, depth + 1, asset_lookup=asset_lookup) - if child_text.strip(): - parts.append(child_text) + # Add orphan leaf_content paths (not covered by outline_items) + for path in node.leaf_content: + if path not in outline_paths: + render_queue.append((_min_sort(path), 'orphan_leaf', path)) - # Case 2: This is a hydrated leaf → show chunk content inline - elif path in node.leaf_content: - _render_leaf_chunks(parts, node.leaf_content[path], sub_indent, asset_lookup=asset_lookup) + # Add orphan children (not covered by outline_items) + for path in node.children: + if path not in outline_paths: + render_queue.append((float('inf'), 'orphan_child', path)) + + # Sort by sort_key (stable sort preserves insertion order for ties) + render_queue.sort(key=lambda x: x[0]) + + from typing import cast + + # ── Render the unified queue ── + for _sort_key, rtype, data in render_queue: + if rtype == 'outline': + item = cast(dict, data) + path = item.get('path', '') + title = item.get('title', '') + is_leaf = item.get('is_leaf', False) + level = item.get('level', 1) + leaf_tag = ' [Leaf]' if is_leaf else '' + + level_tag = f'[L{level}] ' if level else '' + if level <= 1: + parts.append(f'{indent}▸ {level_tag}{title}{leaf_tag}') + else: + parts.append(f'{indent}└ {level_tag}{title}{leaf_tag}') - # Case 3: Unselected → title already rendered above, nothing more needed + sub_indent = indent + ' ' - # Render orphan paths: leaf_content and children not covered by outline_items - for path in node.leaf_content: - if path not in rendered_paths: + # Case 1: drilled-into child → render child tree + if path in node.children: + child = node.children[path] + if path in node.leaf_content: + _render_leaf_chunks(parts, node.leaf_content[path], sub_indent, asset_lookup=asset_lookup) + child_text = render_unified_doc_tree(child, doc_name, depth + 1, asset_lookup=asset_lookup) + if child_text.strip(): + parts.append(child_text) + + # Case 2: hydrated leaf → show chunk content + elif path in node.leaf_content: + _render_leaf_chunks(parts, node.leaf_content[path], sub_indent, asset_lookup=asset_lookup) + + # Case 3: unselected → title only (already rendered above) + + elif rtype == 'orphan_leaf': + path = cast(str, data) title = path.rsplit(' / ', 1)[-1] if ' / ' in path else path parts.append(f'{indent}▸ [Leaf] {title}') sub_indent = indent + ' ' _render_leaf_chunks(parts, node.leaf_content[path], sub_indent, asset_lookup=asset_lookup) - for path in node.children: - if path not in rendered_paths: + elif rtype == 'orphan_child': + path = cast(str, data) title = path.rsplit(' / ', 1)[-1] if ' / ' in path else path parts.append(f'{indent}▸ {title} [DrillDown]') child_text = render_unified_doc_tree(node.children[path], doc_name, depth + 1, asset_lookup=asset_lookup) diff --git a/packages/shared-python/shared/services/retrieval/agentic/budget.py b/packages/shared-python/shared/services/retrieval/agentic/budget.py new file mode 100644 index 000000000..1f4940bdd --- /dev/null +++ b/packages/shared-python/shared/services/retrieval/agentic/budget.py @@ -0,0 +1,199 @@ +"""Token budget ledger for agentic retrieval runs.""" +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import Literal + + +BudgetPoolName = Literal["bootstrap", "planning", "context"] +BudgetStatus = Literal["HEALTHY", "TIGHT", "CRITICAL", "EXHAUSTED"] + + +class BudgetExceeded(Exception): + """Raised when a planned LLM call cannot reserve budget.""" + + +@dataclass +class BudgetPool: + name: BudgetPoolName + capacity: int + used: int = 0 + reserved: int = 0 + + @property + def remaining(self) -> int: + return max(self.capacity - self.used - self.reserved, 0) + + @property + def used_pct(self) -> int: + if self.capacity <= 0: + return 100 + return min(100, int(round((self.used + self.reserved) * 100 / self.capacity))) + + +class BudgetLedger: + """Concurrency-safe ledger with bootstrap/planning/context pools.""" + + def __init__( + self, + *, + total: int, + planning_ratio: float, + bootstrap: int = 2000, + per_doc_min_share: int = 1500, + ) -> None: + total = max(int(total), 1) + bootstrap = max(0, min(int(bootstrap), total)) + remaining = max(total - bootstrap, 0) + planning_ratio = min(max(float(planning_ratio), 0.0), 1.0) + planning_capacity = int(remaining * planning_ratio) + context_capacity = remaining - planning_capacity + + self._lock = asyncio.Lock() + self._pools: dict[BudgetPoolName, BudgetPool] = { + "bootstrap": BudgetPool("bootstrap", bootstrap), + "planning": BudgetPool("planning", planning_capacity), + "context": BudgetPool("context", context_capacity), + } + self._doc_caps: dict[str, int] = {} + self._doc_used: dict[str, int] = {} + self._doc_reserved: dict[str, int] = {} + self._per_doc_min_share = max(int(per_doc_min_share), 0) + self.total_chunks = 0 + self.total_docs = 0 + self.explored_chunks = 0 + self.explored_docs = 0 + self.trimmed_paths: list[dict[str, str]] = [] + + def remaining(self, pool: BudgetPoolName) -> int: + return self._pools[pool].remaining + + def status(self, pool: BudgetPoolName) -> BudgetStatus: + pool_state = self._pools[pool] + if pool_state.remaining <= 0: + return "EXHAUSTED" + used_pct = pool_state.used_pct + if used_pct >= 80: + return "CRITICAL" + if used_pct >= 50: + return "TIGHT" + return "HEALTHY" + + async def allocate_doc_caps(self, doc_chunks: dict[str, int]) -> None: + """Allocate planning soft caps by document chunk counts.""" + async with self._lock: + self._doc_caps.clear() + self._doc_used.clear() + self._doc_reserved.clear() + if not doc_chunks: + return + + planning_capacity = self._pools["planning"].capacity + total_weight = sum(max(int(count), 1) for count in doc_chunks.values()) + for doc_id, count in doc_chunks.items(): + weight = max(int(count), 1) + weighted = int(planning_capacity * weight / total_weight) + self._doc_caps[doc_id] = min( + planning_capacity, + max(self._per_doc_min_share, weighted), + ) + + async def try_reserve( + self, + pool: BudgetPoolName, + est: int, + doc_id: str | None = None, + *, + priority: Literal["normal", "low"] = "normal", + ) -> bool: + est = max(int(est), 0) + if est == 0: + return True + + async with self._lock: + pool_state = self._pools[pool] + if priority == "low" and self.status(pool) == "CRITICAL": + return False + if pool_state.remaining < est: + return False + + pool_state.reserved += est + if pool == "planning" and doc_id: + self._doc_reserved[doc_id] = self._doc_reserved.get(doc_id, 0) + est + return True + + async def commit( + self, + pool: BudgetPoolName, + *, + actual: int, + est: int, + doc_id: str | None = None, + ) -> None: + actual = max(int(actual), 0) + est = max(int(est), 0) + async with self._lock: + pool_state = self._pools[pool] + reserved_delta = min(est, pool_state.reserved) + pool_state.reserved -= reserved_delta + pool_state.used = min(pool_state.capacity, pool_state.used + actual) + + if pool == "planning" and doc_id: + doc_reserved = min(est, self._doc_reserved.get(doc_id, 0)) + if doc_reserved: + self._doc_reserved[doc_id] -= doc_reserved + if self._doc_reserved[doc_id] <= 0: + self._doc_reserved.pop(doc_id, None) + self._doc_used[doc_id] = self._doc_used.get(doc_id, 0) + actual + + async def refund( + self, + pool: BudgetPoolName, + *, + est: int, + doc_id: str | None = None, + ) -> None: + est = max(int(est), 0) + async with self._lock: + pool_state = self._pools[pool] + pool_state.reserved = max(pool_state.reserved - est, 0) + if pool == "planning" and doc_id: + current = self._doc_reserved.get(doc_id, 0) + remaining = max(current - est, 0) + if remaining: + self._doc_reserved[doc_id] = remaining + else: + self._doc_reserved.pop(doc_id, None) + + def mark_explored( + self, + *, + chunks: int = 0, + docs: int = 0, + ) -> None: + self.explored_chunks += max(int(chunks), 0) + self.explored_docs += max(int(docs), 0) + + def snapshot(self) -> dict[str, object]: + snapshot: dict[str, object] = { + name: { + "capacity": pool.capacity, + "used": pool.used, + "reserved": pool.reserved, + "remaining": pool.remaining, + "used_pct": pool.used_pct, + "status": self.status(name), + } + for name, pool in self._pools.items() + } + snapshot.update({ + "total_chunks": self.total_chunks, + "total_docs": self.total_docs, + "explored_chunks": min(self.explored_chunks, self.total_chunks) + if self.total_chunks else self.explored_chunks, + "explored_docs": min(self.explored_docs, self.total_docs) + if self.total_docs else self.explored_docs, + "trimmed_paths": list(self.trimmed_paths), + }) + return snapshot diff --git a/packages/shared-python/shared/services/retrieval/agentic/orchestrator.py b/packages/shared-python/shared/services/retrieval/agentic/orchestrator.py index a757f87a0..b8b766327 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/orchestrator.py +++ b/packages/shared-python/shared/services/retrieval/agentic/orchestrator.py @@ -17,14 +17,16 @@ from __future__ import annotations import os -from typing import Any +import json +from typing import Any, cast from loguru import logger -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from shared.models.database.document import Document +from shared.models.database.document import Document, DocumentChunk, RetrievalHitStat +from shared.services.retrieval.agentic.budget import BudgetExceeded, BudgetLedger, BudgetPoolName from shared.services.retrieval.agentic.trace import TraceRecorder from shared.services.retrieval.agentic.types import ( AgentRunConfig, @@ -39,6 +41,9 @@ _is_client_result_artifact_ref, ) from shared.services.retrieval.llm_adapter import LLMFn +from shared.services.retrieval.llm_adapter import current_llm_usage +from shared.services.retrieval.hit_stats_service import compute_importance_score +from shared.utils.token_estimate import estimate_tokens @@ -101,9 +106,23 @@ def _build_config_from_env() -> AgentRunConfig: max_revisions=int(os.environ.get('RETRIEVAL_AGENTIC_MAX_REVISIONS', '2')), max_nav_depth=int(os.environ.get('RETRIEVAL_AGENTIC_MAX_NAV_DEPTH', '3')), latency_budget_ms=int(os.environ.get('RETRIEVAL_AGENTIC_LATENCY_BUDGET_MS', '12000')), + token_budget_total=int(os.environ.get('RETRIEVAL_AGENTIC_TOKEN_BUDGET_TOTAL', '40000')), + planning_ratio=float(os.environ.get('RETRIEVAL_AGENTIC_PLANNING_RATIO', '0.5')), + bootstrap_budget=int(os.environ.get('RETRIEVAL_AGENTIC_BOOTSTRAP_BUDGET', '2000')), + per_doc_min_share=int(os.environ.get('RETRIEVAL_AGENTIC_PER_DOC_MIN_SHARE', '1500')), + inventory_aware=os.environ.get('RETRIEVAL_AGENTIC_INVENTORY_AWARE', 'true') == 'true', ) +def _stringify_llm_input(prompt: Any) -> str: + if isinstance(prompt, str): + return prompt + try: + return json.dumps(prompt, ensure_ascii=False, default=str) + except Exception: + return str(prompt) + + async def _render_evidence( db: AsyncSession, doc_trees: dict[str, DocTreeNode], @@ -135,6 +154,154 @@ async def _render_evidence( return '\n\n'.join(evidence_parts) if evidence_parts else '(no evidence collected)' +async def _load_budget_inventory( + db: AsyncSession, + *, + user_id: str, + namespace: str, + exclude_document_ids: list[str], +) -> tuple[int, int, dict[str, int]]: + stmt = ( + select(Document.document_id, func.count(DocumentChunk.id)) + .join( + DocumentChunk, + (DocumentChunk.document_id == Document.document_id) + & (DocumentChunk.job_result_id == Document.current_job_result_id), + ) + .where(Document.user_id == user_id) + .where(Document.namespace == namespace) + .where(Document.status == 'active') + .group_by(Document.document_id) + ) + if exclude_document_ids: + stmt = stmt.where(Document.document_id.notin_(list(exclude_document_ids))) + + result = await db.execute(stmt) + doc_chunks = {str(doc_id): int(count or 0) for doc_id, count in result.all()} + return sum(doc_chunks.values()), len(doc_chunks), doc_chunks + + +def _iter_leaf_content(node: DocTreeNode): + for path, chunks in node.leaf_content.items(): + yield path, chunks + for child in node.children.values(): + yield from _iter_leaf_content(child) + + +def _collect_confidences(node: DocTreeNode) -> dict[str, float]: + values = dict(node.confidence) + for child in node.children.values(): + for path, score in _collect_confidences(child).items(): + values[path] = max(values.get(path, 0.0), score) + return values + + +def _pop_leaf_path(node: DocTreeNode, path: str) -> bool: + if path in node.leaf_content: + node.leaf_content.pop(path) + return True + for child in node.children.values(): + if _pop_leaf_path(child, path): + return True + return False + + +def _estimate_chunks_tokens(chunks: list[dict[str, Any]]) -> int: + text = '\n'.join(str(chunk.get('content') or '') for chunk in chunks) + return estimate_tokens(text) + + +async def _fetch_importance_norm_scores( + db: AsyncSession, + *, + user_id: str, + namespace: str, + chunk_ids: list[str], +) -> dict[str, float]: + if not chunk_ids: + return {} + stmt = ( + select( + RetrievalHitStat.chunk_id, + RetrievalHitStat.hit_count, + RetrievalHitStat.last_hit_at, + RetrievalHitStat.created_at, + ) + .where(RetrievalHitStat.user_id == user_id) + .where(RetrievalHitStat.namespace == namespace) + .where(RetrievalHitStat.hit_kind == 'chunk') + .where(RetrievalHitStat.chunk_id.in_(chunk_ids)) + ) + result = await db.execute(stmt) + scores: dict[str, float] = {} + for chunk_id, hit_count, last_hit_at, created_at in result.all(): + if chunk_id and last_hit_at and created_at: + scores[str(chunk_id)] = compute_importance_score(hit_count, last_hit_at, created_at) + return scores + + +async def _trim_evidence_to_budget( + db: AsyncSession, + *, + doc_trees: dict[str, DocTreeNode], + doc_id_to_name: dict[str, str], + context_remaining: int, + user_id: str, + namespace: str, + ledger: BudgetLedger | None, + safety_margin: float = 0.9, +) -> str: + full_text = await _render_evidence(db, doc_trees, doc_id_to_name) + target = int(max(context_remaining, 0) * safety_margin) + if target <= 0 or estimate_tokens(full_text) <= target: + return full_text + + candidates: list[tuple[str, str, tuple[float, float, float], int]] = [] + for doc_id, tree in doc_trees.items(): + confidence = _collect_confidences(tree) + for path, chunks in _iter_leaf_content(tree): + chunk_ids = [ + str(chunk.get('chunk_id')) + for chunk in chunks + if chunk.get('chunk_id') + ] + importance = 0.0 + importance_scores = await _fetch_importance_norm_scores( + db, + user_id=user_id, + namespace=namespace, + chunk_ids=chunk_ids, + ) + if importance_scores: + importance = max(importance_scores.values()) + discovery_score = ( + float(chunks[0].get('discovery_score', 0.0) or 0.0) + if chunks else 0.0 + ) + score = (float(confidence.get(path, 0.0) or 0.0), discovery_score, importance) + candidates.append((doc_id, path, score, _estimate_chunks_tokens(chunks))) + + current_estimate = estimate_tokens(full_text) + removed: list[dict[str, str]] = [] + for doc_id, path, _score, token_estimate in sorted( + candidates, + key=lambda item: (item[2], -item[3]), + ): + if current_estimate <= target: + break + if _pop_leaf_path(doc_trees[doc_id], path): + removed.append({'document_id': doc_id, 'path': path}) + current_estimate = max(current_estimate - token_estimate, 0) + + if ledger is not None: + ledger.trimmed_paths.extend(removed) + logger.info( + f' agentic.trim_evidence: removed={len(removed)} ' + f'est_tokens={current_estimate} target={target}' + ) + return await _render_evidence(db, doc_trees, doc_id_to_name) + + class RetrievalAgent: """Agentic retrieval orchestrator — navigate-then-answer loop. @@ -152,6 +319,82 @@ class RetrievalAgent: If ``llm_fn`` is None, the run returns discovery-only results. """ + async def _call_llm_with_budget( + self, + state: AgentState, + llm_fn: LLMFn, + prompt: Any, + *, + pool: BudgetPoolName, + doc_id: str | None = None, + priority: str = 'normal', + ) -> str: + ledger = state.ledger + if ledger is None: + return await llm_fn(prompt) + + prompt_text = _stringify_llm_input(prompt) + est = estimate_tokens(prompt_text) + reserved = await ledger.try_reserve( + pool, + est, + doc_id=doc_id, + priority='low' if priority == 'low' else 'normal', + ) + if not reserved: + raise BudgetExceeded(f'{pool} budget exhausted') + + try: + response = await llm_fn(prompt) + except Exception: + await ledger.refund(pool, est=est, doc_id=doc_id) + raise + + usage = current_llm_usage.get() or {} + actual = int(usage.get('prompt_tokens') or est) + await ledger.commit(pool, actual=actual, est=est, doc_id=doc_id) + return response + + def _budgeted_doc_llm_fn( + self, + state: AgentState, + llm_fn: LLMFn, + *, + doc_id: str, + depth: int, + ) -> LLMFn: + async def _call(prompt): + return await self._call_llm_with_budget( + state, + llm_fn, + prompt, + pool='planning', + doc_id=doc_id, + priority='low' if depth >= 2 else 'normal', + ) + + return _call + + def _budgeted_discovery_llm_fn( + self, + state: AgentState, + llm_fn: LLMFn, + *, + doc_id: str, + low_priority: bool, + ) -> LLMFn: + async def _call(prompt): + return await self._call_llm_with_budget( + state, + llm_fn, + prompt, + pool='planning', + doc_id=doc_id, + priority='low' if low_priority else 'normal', + ) + + return _call + async def run( self, db: AsyncSession, @@ -188,6 +431,22 @@ async def run( exclude_sections = exclude_sections or [] state = AgentState() + state.ledger = BudgetLedger( + total=config.token_budget_total, + planning_ratio=config.planning_ratio, + bootstrap=config.bootstrap_budget, + per_doc_min_share=config.per_doc_min_share, + ) + total_chunks, total_docs, chunks_count_by_doc = await _load_budget_inventory( + db, + user_id=user_id, + namespace=namespace, + exclude_document_ids=exclude_document_ids, + ) + state.kg_total_chunks = total_chunks + state.kg_total_docs = total_docs + state.ledger.total_chunks = total_chunks + state.ledger.total_docs = total_docs trace = TraceRecorder( db, user_id=user_id, namespace=namespace, query=query, config=config, top_k=top_k, data_type=data_type, @@ -204,12 +463,38 @@ async def run( logger.info( f'agentic retrieval START: query="{query[:60]}..." ' - f'top_k={top_k} budget={config.latency_budget_ms}ms' + f'top_k={top_k} latency_budget={config.latency_budget_ms}ms ' + f'token_budget={config.token_budget_total}' ) if llm_fn is None: logger.warning('agentic: no llm_fn provided — running discovery-only mode') + planning_llm_fn: LLMFn | None = None + bootstrap_llm_fn: LLMFn | None = None + context_llm_fn: LLMFn | None = None + if llm_fn is not None: + base_llm_fn = llm_fn + + async def _planning_llm_call(prompt): + return await self._call_llm_with_budget( + state, base_llm_fn, prompt, pool='planning' + ) + + async def _bootstrap_llm_call(prompt): + return await self._call_llm_with_budget( + state, base_llm_fn, prompt, pool='bootstrap' + ) + + async def _context_llm_call(prompt): + return await self._call_llm_with_budget( + state, base_llm_fn, prompt, pool='context' + ) + + planning_llm_fn = _planning_llm_call + bootstrap_llm_fn = _bootstrap_llm_call + context_llm_fn = _context_llm_call + # Shared kwargs for bottom_discovery discovery_kwargs: dict[str, Any] = { 'user_id': user_id, @@ -248,15 +533,25 @@ async def run( ) # 1b. KG document selection (requires LLM) - if llm_fn is not None: - kg_result = await tools.kg_document_select( - db, - user_id=user_id, - namespace=namespace, - query=query, - llm_fn=llm_fn, - exclude_document_ids=list(state.ever_explored_doc_ids | set(exclude_document_ids)), - ) + if bootstrap_llm_fn is not None: + try: + kg_result = await tools.kg_document_select( + db, + user_id=user_id, + namespace=namespace, + query=query, + llm_fn=bootstrap_llm_fn, + exclude_document_ids=list(state.ever_explored_doc_ids | set(exclude_document_ids)), + budget_snapshot=state.ledger.snapshot() if state.ledger else None, + ) + except BudgetExceeded: + logger.info(' agentic: bootstrap budget exhausted during document selection') + if trace_enabled: + trace.record_budget_stop('bootstrap_exhausted') + kg_result = ToolResult( + status='no_confident_doc', + payload={'reason': 'bootstrap budget exhausted'}, + ) state.step_count += 1 if trace_enabled: @@ -319,7 +614,11 @@ async def run( if r.get('chunk_id') ] if trace_enabled: - await trace.complete(discovery_rows, 'agentic_discovery_only') + await trace.complete( + discovery_rows, + 'agentic_discovery_only', + budget_snapshot=state.ledger.snapshot() if state.ledger else None, + ) return AgenticResult( evidence_text='', answer_text='', @@ -360,6 +659,12 @@ async def run( if jrid: state.doc_job_map[did] = jrid + if state.ledger is not None: + await state.ledger.allocate_doc_caps({ + doc.document_id: chunks_count_by_doc.get(doc.document_id, 1) + for doc in state.selected_docs + }) + # ══════════════════════════════════════════════════════════════════ # Phase 2 + 3 Loop: Navigate → Render → Attempt Answer → (Revise) # ══════════════════════════════════════════════════════════════════ @@ -418,21 +723,38 @@ async def run( if depth >= config.max_nav_depth: continue - if llm_fn is None: + if planning_llm_fn is None: + break + if state.ledger and state.ledger.status('planning') in ('CRITICAL', 'EXHAUSTED'): + logger.info(' agentic: planning budget critical, ending BFS for current doc') break - # ★ Step 1: Tool selection (agent decides which tool) - tool_choice = await tools.tool_select_step( - db, - document_id=doc.document_id, - job_result_id=job_result_id, - query=query, - llm_fn=llm_fn, - doc_name=doc_name, - scope_path=scope, - exclude_paths=doc_exclude, - revision_hint=revision_hint if depth == 0 else None, + doc_llm_fn = self._budgeted_doc_llm_fn( + state, + cast(LLMFn, llm_fn), + doc_id=doc.document_id, + depth=depth, ) + + # ★ Step 1: Tool selection (agent decides which asset tools) + try: + tool_choices = await tools.tool_select_step( + db, + document_id=doc.document_id, + job_result_id=job_result_id, + query=query, + llm_fn=doc_llm_fn, + doc_name=doc_name, + scope_path=scope, + exclude_paths=doc_exclude, + revision_hint=revision_hint if depth == 0 else None, + budget_snapshot=state.ledger.snapshot() if state.ledger else None, + ) + except BudgetExceeded: + logger.info(' agentic: planning budget exhausted during tool selection') + if trace_enabled: + trace.record_budget_stop('planning_exhausted') + break state.step_count += 1 if trace_enabled: @@ -443,7 +765,7 @@ async def run( 'document_id': doc.document_id, 'scope': scope or 'root', 'depth': depth, - 'tool_choice': tool_choice, + 'tool_choice': tool_choices or ['NAVIGATE'], }, ), decision_reason=f'tool_r{round_idx}_d{depth}_{doc.source_file_name}', @@ -452,12 +774,15 @@ async def run( logger.info( f' agentic step {state.step_count}: tool_select_step ' f'doc="{doc.source_file_name}" scope={scope or "root"} ' - f'depth={depth} tool={tool_choice}' + f'depth={depth} tools={tool_choices or ["NAVIGATE"]}' ) - if tool_choice in ('FIND_IMAGES', 'FIND_TABLES'): - # ★ Terminal: asset filter (no LLM, pure programmatic) - asset_type = 'image' if tool_choice == 'FIND_IMAGES' else 'table' + pending_scope_assets: list[dict] = [] + for asset_tool in tool_choices: + if asset_tool not in ('FIND_IMAGES', 'FIND_TABLES'): + continue + # ★ Asset collection (programmatic extraction) + asset_type = 'image' if asset_tool == 'FIND_IMAGES' else 'table' asset_chunks = await tools.asset_filter_step( db, document_id=doc.document_id, @@ -466,7 +791,7 @@ async def run( asset_type=asset_type, ) if asset_chunks: - parent_node.leaf_content[scope or 'root'] = asset_chunks + pending_scope_assets.extend(asset_chunks) if trace_enabled: trace.record_step( @@ -487,34 +812,74 @@ async def run( f'doc="{doc.source_file_name}" scope={scope or "root"} ' f'type={asset_type} chunks={len(asset_chunks) if asset_chunks else 0}' ) - # NO further drill-down — terminal action - continue + # ★ Fallthrough: always proceed to NAVIGATE. # ★ Step 2: NAVIGATE (existing scope_navigate_step) - step_node, drill_paths = await tools.scope_navigate_step( - db, - document_id=doc.document_id, - job_result_id=job_result_id, - query=query, - llm_fn=llm_fn, - user_id=user_id, - namespace=namespace, - doc_name=doc_name, - scope_path=scope, - exclude_paths=doc_exclude, - revision_hint=revision_hint if depth == 0 else None, - ) + try: + step_node, drill_paths = await tools.scope_navigate_step( + db, + document_id=doc.document_id, + job_result_id=job_result_id, + query=query, + llm_fn=doc_llm_fn, + user_id=user_id, + namespace=namespace, + doc_name=doc_name, + scope_path=scope, + exclude_paths=doc_exclude, + revision_hint=revision_hint if depth == 0 else None, + budget_snapshot=state.ledger.snapshot() if state.ledger else None, + ) + except BudgetExceeded: + logger.info(' agentic: planning budget exhausted during navigation') + if trace_enabled: + trace.record_budget_stop('planning_exhausted') + break state.step_count += 1 # Merge step result into parent node parent_node.outline_items = step_node.outline_items - parent_node.leaf_content.update(step_node.leaf_content) + for leaf_path, chunks in step_node.leaf_content.items(): + parent_node.add_leaf_chunks(leaf_path, chunks) parent_node.confidence = step_node.confidence + # ★ Step 2.5: Reconcile pending assets with navigated leaf content + if pending_scope_assets: + # Collect all chunk_ids already present in any leaf_content + existing_ids = { + str(row.get('chunk_id') or '') + for row in parent_node.flatten_chunk_rows() + if row.get('chunk_id') + } + # Filter out assets already inlined + supplementary = [ + a for a in pending_scope_assets + if str(a.get('chunk_id') or '') not in existing_ids + ] + if supplementary: + # Only place assets into sections already in the + # navigated tree. If the owner path isn't part of + # the tree, fall back to the current scope. + navigated_paths = set(parent_node.leaf_content.keys()) | set(parent_node.children.keys()) + for asset in supplementary: + owner_path = ( + asset.get('owner_section_path') + or asset.get('section_path') + or scope + ) + if owner_path and owner_path in navigated_paths: + parent_node.add_leaf_chunks(str(owner_path), [asset]) + elif scope: + parent_node.add_leaf_chunks(str(scope), [asset]) + # Accumulate hydrated leaf paths into doc_exclude - # so subsequent drill-downs don't re-show them as [SELECT] + # so subsequent drill-downs don't re-show them as [SELECT]. + # Skip paths that are pending drill-down (non-leaf hybrid nodes + # hydrated via self_only) — their children must remain selectable. + drill_path_set = {sel['path'] for sel in drill_paths} for leaf_path in step_node.leaf_content: - doc_exclude.add(leaf_path) + if leaf_path not in drill_path_set: + doc_exclude.add(leaf_path) # Queue non-leaf selections for further drill-down for sel in drill_paths: @@ -528,11 +893,7 @@ async def run( # leaf chunks are stored on the parent node. Move them # into the child node so render_unified_doc_tree nests # them correctly instead of rendering orphans at root. - for child_path in list(parent_node.children.keys()): - for leaf_path in list(parent_node.leaf_content.keys()): - if leaf_path.startswith(child_path + ' / '): - parent_node.children[child_path].leaf_content[leaf_path] = \ - parent_node.leaf_content.pop(leaf_path) + parent_node.reparent_leaf_content() if trace_enabled: trace.record_step( @@ -557,24 +918,41 @@ async def run( f'leaves={len(step_node.leaf_content)} ' f'drills={len(drill_paths)}' ) + if state.ledger is not None: + state.ledger.mark_explored( + chunks=sum(len(chunks) for chunks in step_node.leaf_content.values()), + ) else: # B-class: no BFS, create empty root root = DocTreeNode(scope_path=None) # ── Post-BFS: Discovery selection step ───────────────────── doc_hints = discovery_by_doc.get(doc.document_id, []) - if doc_hints and llm_fn is not None and state.elapsed_ms < config.latency_budget_ms: - discovery_node = await tools.discovery_select_step( - db, - document_id=doc.document_id, - query=query, - llm_fn=llm_fn, - user_id=user_id, - namespace=namespace, - doc_name=doc_name, - discovery_hints=doc_hints, - revision_hint=revision_hint, + if doc_hints and planning_llm_fn is not None and state.elapsed_ms < config.latency_budget_ms: + doc_discovery_llm_fn = self._budgeted_discovery_llm_fn( + state, + cast(LLMFn, llm_fn), + doc_id=doc.document_id, + low_priority=root.has_content(), ) + try: + discovery_node = await tools.discovery_select_step( + db, + document_id=doc.document_id, + query=query, + llm_fn=doc_discovery_llm_fn, + user_id=user_id, + namespace=namespace, + doc_name=doc_name, + discovery_hints=doc_hints, + revision_hint=revision_hint, + budget_snapshot=state.ledger.snapshot() if state.ledger else None, + ) + except BudgetExceeded: + logger.info(' agentic: planning budget exhausted during discovery selection') + if trace_enabled: + trace.record_budget_stop('planning_exhausted') + discovery_node = DocTreeNode(scope_path=None) state.step_count += 1 if trace_enabled: @@ -592,6 +970,10 @@ async def run( # Merge discovery results into BFS tree root.merge(discovery_node) + if state.ledger is not None: + state.ledger.mark_explored( + chunks=sum(len(chunks) for chunks in discovery_node.leaf_content.values()), + ) # Merge or store doc tree if doc.document_id in state.doc_trees: @@ -599,14 +981,21 @@ async def run( else: state.doc_trees[doc.document_id] = root state.ever_explored_doc_ids.add(doc.document_id) + if state.ledger is not None: + state.ledger.mark_explored(docs=1) # ── Phase 3: Render evidence + attempt_answer ──────────────── - evidence_text = await _render_evidence( + evidence_text = await _trim_evidence_to_budget( db, - state.doc_trees, state.doc_id_to_name, + doc_trees=state.doc_trees, + doc_id_to_name=state.doc_id_to_name, + context_remaining=state.ledger.remaining('context') if state.ledger else config.token_budget_total, + user_id=user_id, + namespace=namespace, + ledger=state.ledger, ) - if llm_fn is None: + if context_llm_fn is None: stop_reason = 'no_llm' break @@ -620,16 +1009,30 @@ async def run( url for url in asset_url_map.values() if url ] + async def vlm_context_call(prompt, _vlm_fn=vlm_fn): + return await self._call_llm_with_budget( + state, cast(LLMFn, _vlm_fn), prompt, pool='context' + ) + # Auto-trigger attempt_answer (VLM if images present) - status, answer_text, reason = await attempt_answer( - llm_fn, - query=query, - evidence_text=evidence_text, - state=state, - config=config, - vlm_fn=vlm_fn, - image_urls=evidence_image_urls or None, - ) + try: + status, answer_text, reason = await attempt_answer( + context_llm_fn, + query=query, + evidence_text=evidence_text, + state=state, + config=config, + vlm_fn=vlm_context_call if vlm_fn else None, + image_urls=evidence_image_urls or None, + budget_snapshot=state.ledger.snapshot() if state.ledger else None, + ) + except BudgetExceeded: + logger.info(' agentic: context budget exhausted before attempt_answer') + if trace_enabled: + trace.record_budget_stop('context_exhausted') + status, answer_text, reason = 'NOT_FOUND', '', 'context budget exhausted' + stop_reason = 'context_budget' + break state.step_count += 1 if trace_enabled: @@ -671,15 +1074,26 @@ async def run( state.selected_docs.clear() # Re-run KG select with revision hint - kg_result = await tools.kg_document_select( - db, - user_id=user_id, - namespace=namespace, - query=query, - llm_fn=llm_fn, - exclude_document_ids=list(set(exclude_document_ids)), - revision_hint=revision_hint, - ) + if bootstrap_llm_fn is None: + stop_reason = 'no_llm' + break + try: + kg_result = await tools.kg_document_select( + db, + user_id=user_id, + namespace=namespace, + query=query, + llm_fn=bootstrap_llm_fn, + exclude_document_ids=list(set(exclude_document_ids)), + revision_hint=revision_hint, + budget_snapshot=state.ledger.snapshot() if state.ledger else None, + ) + except BudgetExceeded: + logger.info(' agentic: bootstrap budget exhausted during revision doc selection') + if trace_enabled: + trace.record_budget_stop('bootstrap_exhausted') + stop_reason = 'bootstrap_budget' + break state.step_count += 1 if kg_result.status == 'selected_docs': @@ -699,6 +1113,12 @@ async def run( stop_reason = 'no_new_docs' break + if state.ledger is not None: + await state.ledger.allocate_doc_caps({ + doc.document_id: chunks_count_by_doc.get(doc.document_id, 1) + for doc in state.selected_docs + }) + # ══════════════════════════════════════════════════════════════════ # Final Assembly # ══════════════════════════════════════════════════════════════════ @@ -729,6 +1149,8 @@ async def run( answer_text=answer_text, referenced_chunks=all_refs, router_used=router_used, + budget_snapshot=state.ledger.snapshot() if state.ledger else None, + stop_reason=stop_reason, ) logger.info( @@ -741,6 +1163,10 @@ async def run( ) if trace_enabled: - await trace.complete(all_refs, router_used) + await trace.complete( + all_refs, + router_used, + budget_snapshot=state.ledger.snapshot() if state.ledger else None, + ) return result diff --git a/packages/shared-python/shared/services/retrieval/agentic/policy.py b/packages/shared-python/shared/services/retrieval/agentic/policy.py index ab0087973..6d63035e7 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/policy.py +++ b/packages/shared-python/shared/services/retrieval/agentic/policy.py @@ -12,10 +12,13 @@ import json import os import re +from ipaddress import ip_address from typing import Any +from urllib.parse import urlparse from loguru import logger +from shared.services.retrieval.agentic.budget import BudgetExceeded from shared.services.retrieval.agentic.types import AgentRunConfig, AgentState from shared.services.retrieval.llm_adapter import LLMFn @@ -36,6 +39,20 @@ def _parse_answer_response(text: str) -> dict[str, Any] | None: return None +def _is_external_http_url(url: str) -> bool: + parsed = urlparse(str(url)) + if parsed.scheme not in {'http', 'https'} or not parsed.hostname: + return False + host = parsed.hostname.strip().lower() + if host in {'localhost', 'ip6-localhost', 'ip6-loopback'} or host.endswith('.local'): + return False + try: + addr = ip_address(host) + except ValueError: + return True + return not (addr.is_private or addr.is_loopback or addr.is_link_local) + + async def attempt_answer( llm_fn: LLMFn, *, @@ -45,6 +62,7 @@ async def attempt_answer( config: AgentRunConfig, vlm_fn: LLMFn | None = None, image_urls: list[str] | None = None, + budget_snapshot: dict | None = None, ) -> tuple[str, str, str]: """Attempt to answer the query using collected evidence. @@ -67,6 +85,7 @@ async def attempt_answer( evidence_context=evidence_text, revision_count=state.revision_count, max_revisions=config.max_revisions, + context_status=((budget_snapshot or {}).get('context') or {}).get('status', 'HEALTHY'), ) verbose = os.environ.get('RETRIEVAL_AGENTIC_VERBOSE', '') == 'true' @@ -75,12 +94,19 @@ async def attempt_answer( effective_fn = llm_fn effective_input: Any = prompt_text - if vlm_fn and image_urls: + usable_image_urls = [url for url in image_urls or [] if _is_external_http_url(url)] + if image_urls and len(usable_image_urls) != len(image_urls): + logger.info( + f' [attempt_answer] skipped {len(image_urls) - len(usable_image_urls)} ' + 'non-public image URLs for VLM' + ) + + if vlm_fn and usable_image_urls: # Build multimodal message: text evidence + image_url parts content_parts: list[dict[str, Any]] = [ {'type': 'text', 'text': prompt_text}, ] - for url in image_urls[:20]: # Cap at 20 images to avoid token overflow + for url in usable_image_urls[:20]: # Cap at 20 images to avoid token overflow content_parts.append({ 'type': 'image_url', 'image_url': {'url': url}, @@ -88,8 +114,8 @@ async def attempt_answer( effective_input = [{'role': 'user', 'content': content_parts}] effective_fn = vlm_fn logger.info( - f' [attempt_answer] using VLM with {len(image_urls)} image URLs ' - f'(capped at {min(len(image_urls), 20)})' + f' [attempt_answer] using VLM with {len(usable_image_urls)} image URLs ' + f'(capped at {min(len(usable_image_urls), 20)})' ) if verbose: @@ -98,7 +124,15 @@ async def attempt_answer( f'{prompt_text}' ) - raw_response = await effective_fn(effective_input) + try: + raw_response = await effective_fn(effective_input) + except BudgetExceeded: + raise + except Exception as exc: + if effective_fn is llm_fn: + raise + logger.warning(f' [attempt_answer] VLM failed, falling back to text LLM: {exc}') + raw_response = await llm_fn(prompt_text) logger.info(f' [attempt_answer] raw={repr(raw_response[:300])}') if verbose: @@ -107,11 +141,8 @@ async def attempt_answer( f'{raw_response}' ) - # If VLM returned empty (e.g. URL not reachable), fall back to text LLM if not raw_response.strip() and effective_fn is not llm_fn: - logger.info(' [attempt_answer] VLM returned empty, falling back to text LLM') - raw_response = await llm_fn(prompt_text) - logger.info(f' [attempt_answer] text fallback raw={repr(raw_response[:300])}') + return 'NOT_FOUND', '', 'VLM returned empty response for multimodal evidence' parsed = _parse_answer_response(raw_response) if not parsed: @@ -145,6 +176,7 @@ async def attempt_answer( {evidence_context} REVISION: {revision_count} of {max_revisions} revisions used. +Context budget remaining is {context_status}; the evidence may have been trimmed. INSTRUCTIONS: 1. If the evidence contains enough information to answer the query, diff --git a/packages/shared-python/shared/services/retrieval/agentic/tools.py b/packages/shared-python/shared/services/retrieval/agentic/tools.py index 1098130a6..d11e4da7b 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/tools.py +++ b/packages/shared-python/shared/services/retrieval/agentic/tools.py @@ -15,18 +15,18 @@ from sqlalchemy.ext.asyncio import AsyncSession from shared.models.database.document import Document +from shared.services.retrieval.agentic.budget import BudgetExceeded from shared.services.retrieval.agentic.types import DocTreeNode, ToolResult from shared.services.retrieval.agent_navigate import ( _build_knowledge_map_overview, - _expand_by_edges, _format_items_for_llm, - _grep_discover_document_ids, _load_child_sections, _parse_json_array, _parse_scope_nav_response, _SCOPE_NAV_PROMPT, _DISCOVERY_SELECT_PROMPT, _FILE_SELECT_PROMPT, + _format_budget_block, ) from shared.services.retrieval.app_service import ( _CHANNEL_WEIGHT_CONTENT, @@ -43,6 +43,104 @@ from shared.services.retrieval.llm_adapter import LLMFn +# --------------------------------------------------------------------------- +# Helper: resolve connected asset → owner text chunk section_path +# --------------------------------------------------------------------------- + +def _build_connected_owner_map(text_chunks: list[dict[str, Any]]) -> dict[str, str]: + """Build target_chunk_id → owner text chunk section_path mapping. + + When text chunks reference images/tables via connect_to metadata, + the referenced assets live in Root section. This map lets us attribute + those assets back to the text chunk's section for correct tree placement. + """ + owner_map: dict[str, str] = {} + for chunk in text_chunks: + if (chunk.get('chunk_type') or 'text') != 'text': + continue + section_path = chunk.get('section_path') or '' + if not section_path: + continue + metadata = chunk.get('chunk_metadata') or {} + if not isinstance(metadata, dict): + continue + for conn in metadata.get('connect_to') or []: + if not isinstance(conn, dict): + continue + target_id = str(conn.get('target') or '').strip() + if target_id and target_id not in owner_map: + owner_map[target_id] = section_path + return owner_map + + +async def _resolve_root_asset_owners( + db: AsyncSession, + *, + document_id: str, + job_result_id: str, + chunks: list[dict[str, Any]], +) -> dict[str, str]: + """Resolve owner section_path for Root-stranded image/table chunks. + + When Root is hydrated directly (e.g. via discovery selection), the + batch contains standalone image/table chunks whose section_path is + 'Root'. ``_build_connected_owner_map`` cannot help because the + referencing text chunks live in other sections outside the batch. + + This function queries the *entire document* for text chunks with + connect_to metadata, using the same logic as + ``_build_connected_owner_map``, to resolve the true owner. + + Returns target_chunk_id → owner_section_path for Root assets only. + Returns empty dict when there are no Root assets (zero DB overhead). + """ + from shared.models.database.document import DocumentChunk, DocumentSection + + root_asset_ids = [ + str(c.get('chunk_id') or '') + for c in chunks + if not c.get('owner_section_path') # skip if already resolved by batch-level owner map + and (c.get('section_path') or '') == 'Root' + and (c.get('chunk_type') or '').lower() in ('image', 'table') + and c.get('chunk_id') + ] + if not root_asset_ids: + return {} + + root_asset_set = set(root_asset_ids) + + # Query all text chunks in this document for connect_to metadata + text_stmt = ( + select( + DocumentChunk.chunk_metadata, + DocumentSection.section_path, + ) + .outerjoin(DocumentSection, DocumentSection.section_id == DocumentChunk.section_id) + .where(DocumentChunk.document_id == document_id) + .where(DocumentChunk.job_result_id == job_result_id) + .where(DocumentChunk.chunk_type == 'text') + ) + result = await db.execute(text_stmt) + + owner_map: dict[str, str] = {} + for metadata, section_path in result.all(): + if not isinstance(metadata, dict) or not section_path: + continue + for conn in metadata.get('connect_to') or []: + if not isinstance(conn, dict): + continue + target_id = str(conn.get('target') or '').strip() + if target_id in root_asset_set and target_id not in owner_map: + owner_map[target_id] = section_path + + if owner_map: + logger.info( + f' _resolve_root_asset_owners: resolved {len(owner_map)}/{len(root_asset_ids)} ' + f'Root assets to their owner sections' + ) + return owner_map + + # --------------------------------------------------------------------------- # Tool: bottom_discovery # --------------------------------------------------------------------------- @@ -207,6 +305,7 @@ async def kg_document_select( file_prompt = _FILE_SELECT_PROMPT.format( overview=overview_text, query=query, revision_context=revision_context, + budget_block=_format_budget_block(_kwargs.get('budget_snapshot')), ) file_response = await llm_fn(file_prompt) selected_ids = _parse_json_array(file_response) @@ -255,127 +354,14 @@ async def kg_document_select( }, latency_ms=latency, ) + except BudgetExceeded: + raise except Exception as e: latency = int((time.monotonic() - t0) * 1000) logger.error(f' agentic.kg_document_select failed: {e}') return ToolResult(status='error', error=str(e), latency_ms=latency) -# --------------------------------------------------------------------------- -# Tool: grep_document_discover -# --------------------------------------------------------------------------- - -async def grep_document_discover( - db: AsyncSession, - *, - user_id: str, - namespace: str, - query: str, - exclude_document_ids: list[str], - **_kwargs: Any, -) -> ToolResult: - """Discover documents via term search (GREP).""" - t0 = time.monotonic() - try: - grep_doc_ids = await _grep_discover_document_ids( - db, user_id=user_id, namespace=namespace, query=query, - exclude_document_ids=exclude_document_ids, - ) - - if not grep_doc_ids: - latency = int((time.monotonic() - t0) * 1000) - return ToolResult( - status='no_docs_found', - payload={'reason': 'GREP found no matching documents'}, - latency_ms=latency, - ) - - doc_stmt = ( - select(Document.document_id, Document.source_file_name, Document.current_job_result_id) - .where(Document.document_id.in_(grep_doc_ids)) - ) - doc_result = await db.execute(doc_stmt) - doc_id_to_name: dict[str, str] = {} - doc_job_map: dict[str, str] = {} - for did, fname, jrid in doc_result.all(): - doc_id_to_name[did] = fname or did - if jrid: - doc_job_map[did] = jrid - - latency = int((time.monotonic() - t0) * 1000) - logger.info(f' agentic.grep_document_discover: {len(grep_doc_ids)} docs found, {latency}ms') - return ToolResult( - status='discovered_docs', - payload={ - 'document_ids': grep_doc_ids, - 'doc_id_to_name': doc_id_to_name, - 'doc_job_map': doc_job_map, - }, - latency_ms=latency, - ) - except Exception as e: - latency = int((time.monotonic() - t0) * 1000) - logger.error(f' agentic.grep_document_discover failed: {e}') - return ToolResult(status='error', error=str(e), latency_ms=latency) - - -# --------------------------------------------------------------------------- -# Tool: graph_expand_docs -# --------------------------------------------------------------------------- - -async def graph_expand_docs( - db: AsyncSession, - *, - user_id: str, - namespace: str, - document_ids: list[str], - **_kwargs: Any, -) -> ToolResult: - """Expand document set via KG edge traversal.""" - t0 = time.monotonic() - try: - expanded_ids = await _expand_by_edges( - db, document_ids=document_ids, user_id=user_id, namespace=namespace, - ) - new_ids = [did for did in expanded_ids if did not in document_ids] - - if not new_ids: - latency = int((time.monotonic() - t0) * 1000) - return ToolResult( - status='no_expansion', - payload={'reason': 'no new neighbors found via edges'}, - latency_ms=latency, - ) - - doc_stmt = ( - select(Document.document_id, Document.source_file_name, Document.current_job_result_id) - .where(Document.document_id.in_(new_ids)) - ) - doc_result = await db.execute(doc_stmt) - doc_id_to_name: dict[str, str] = {} - doc_job_map: dict[str, str] = {} - for did, fname, jrid in doc_result.all(): - doc_id_to_name[did] = fname or did - if jrid: - doc_job_map[did] = jrid - - latency = int((time.monotonic() - t0) * 1000) - logger.info(f' agentic.graph_expand_docs: {len(new_ids)} new docs from edges, {latency}ms') - return ToolResult( - status='expanded_docs', - payload={ - 'document_ids': new_ids, - 'doc_id_to_name': doc_id_to_name, - 'doc_job_map': doc_job_map, - }, - latency_ms=latency, - ) - except Exception as e: - latency = int((time.monotonic() - t0) * 1000) - logger.error(f' agentic.graph_expand_docs failed: {e}') - return ToolResult(status='error', error=str(e), latency_ms=latency) - - # --------------------------------------------------------------------------- # Tool: tool_select_step (lightweight LLM router) # --------------------------------------------------------------------------- @@ -384,73 +370,94 @@ async def graph_expand_docs( You are a document navigation agent. Document: "{doc_name}" -{scope_header} +{budget_block} +{scope_header} Below is a summary of the current scope's sections: {tree_summary} User query: {query} -=== Available Tools === +=== Available Actions === -NAVIGATE +NAVIGATE (always performed) Drill into specific sections to explore detailed content. - Use when the query requires reading text content in sub-sections. - -FIND_IMAGES - Extract all image/chart/diagram assets under this scope. - Use when the query asks about images, charts, figures, or visual content. - -FIND_TABLES - Extract all table/data assets under this scope. - Use when the query asks about tables, tabular data, or structured data. - -Choose exactly ONE tool. Return ONLY a JSON object: -{{"tool": "NAVIGATE"}} -or {{"tool": "FIND_IMAGES"}} -or {{"tool": "FIND_TABLES"}} + This action always runs — you do not need to select it. + +FIND_IMAGES (optional, additive) + Also extract image/chart/diagram assets under this scope. + Select this when the query asks about images, charts, figures, or visual content. + +FIND_TABLES (optional, additive) + Also extract table/data assets under this scope. + Select this when the query asks about tables, tabular data, or structured data. + +You may select ZERO, ONE, or BOTH optional actions. +Navigation always happens regardless of your selection. + +Return ONLY a JSON object: +{{"tools": []}} — navigate only, no extra assets +{{"tools": ["FIND_IMAGES"]}} — navigate + extract images +{{"tools": ["FIND_TABLES"]}} — navigate + extract tables +{{"tools": ["FIND_IMAGES", "FIND_TABLES"]}} — navigate + extract both +When budget is TIGHT, prefer fewer extra actions. +When budget is CRITICAL, return empty tools unless assets directly answer the query. Do not include any explanation. """ -def _parse_tool_choice(text: str) -> str: - """Parse tool choice from LLM response. Returns one of NAVIGATE/FIND_IMAGES/FIND_TABLES.""" +def _parse_tool_choice(text: str) -> list[str]: + """Parse tool choices from LLM response. + + Returns a list of selected tools (subset of FIND_IMAGES, FIND_TABLES). + NAVIGATE is always implicit — an empty list means "navigate only". + """ import json as _json import re as _re text = text.strip() - _VALID_TOOLS = {'NAVIGATE', 'FIND_IMAGES', 'FIND_TABLES'} + _ASSET_TOOLS = {'FIND_IMAGES', 'FIND_TABLES'} + + def _extract_from_data(data: dict) -> list[str]: + # New format: {"tools": [...]} + tools_val = data.get('tools') + if isinstance(tools_val, list): + return [str(t).strip().upper() for t in tools_val if str(t).strip().upper() in _ASSET_TOOLS] + # Legacy format: {"tool": "..."} + tool_val = str(data.get('tool', '')).strip().upper() + if tool_val in _ASSET_TOOLS: + return [tool_val] + if tool_val == 'NAVIGATE': + return [] + return [] # Try JSON parse try: data = _json.loads(text) if isinstance(data, dict): - tool = str(data.get('tool', '')).strip().upper() - if tool in _VALID_TOOLS: - return tool + return _extract_from_data(data) except (ValueError, _json.JSONDecodeError): pass - # Fallback: extract JSON from markdown wrapper + # Accept a JSON object wrapped in markdown match = _re.search(r'\{.*?\}', text, _re.DOTALL) if match: try: data = _json.loads(match.group()) if isinstance(data, dict): - tool = str(data.get('tool', '')).strip().upper() - if tool in _VALID_TOOLS: - return tool + return _extract_from_data(data) except (ValueError, _json.JSONDecodeError): pass - # Last resort: keyword match + # Fallback: scan for tool names in raw text upper = text.upper() + result: list[str] = [] if 'FIND_IMAGES' in upper: - return 'FIND_IMAGES' + result.append('FIND_IMAGES') if 'FIND_TABLES' in upper: - return 'FIND_TABLES' - return 'NAVIGATE' + result.append('FIND_TABLES') + return result async def tool_select_step( @@ -464,23 +471,22 @@ async def tool_select_step( scope_path: str | None = None, exclude_paths: set[str] | None = None, revision_hint: str | None = None, -) -> str: - """Route to the appropriate tool for the current scope. - - Returns one of: 'NAVIGATE', 'FIND_IMAGES', 'FIND_TABLES'. + budget_snapshot: dict | None = None, +) -> list[str]: + """Route to the appropriate tools for the current scope. - This is the top-level agent decision — it picks WHICH tool - to invoke. Each tool then handles its own internal logic. + Returns a list of asset tools to run (FIND_IMAGES, FIND_TABLES). + NAVIGATE always runs implicitly after any asset extraction. Optimization: if the current scope has no image or table chunks, - skips the LLM call and returns 'NAVIGATE' directly. + skips the LLM call and returns an empty list (navigate only). """ items = await _load_child_sections( db, document_id, job_result_id, scope_path, exclude_paths=exclude_paths, ) if not items: - return 'NAVIGATE' # fallback + return [] # Build lightweight tree summary (titles + counts only, no summaries) summary_lines = [] @@ -504,7 +510,7 @@ async def tool_select_step( total_images = sum(i.get('image_count', 0) for i in items) total_tables = sum(i.get('table_count', 0) for i in items) if total_images == 0 and total_tables == 0: - return 'NAVIGATE' # no assets → skip tool selection, go straight to navigate + return [] # no assets → skip tool selection, navigate only scope_header = ( f'Current scope: "{scope_path}"' if scope_path @@ -513,6 +519,7 @@ async def tool_select_step( prompt = _TOOL_SELECT_PROMPT.format( doc_name=doc_name or document_id, scope_header=scope_header, + budget_block=_format_budget_block(budget_snapshot), tree_summary=tree_summary, query=query, ) @@ -525,13 +532,13 @@ async def tool_select_step( ) response = await llm_fn(prompt) - # Parse tool choice - tool = _parse_tool_choice(response) + # Parse tool choices + asset_tools = _parse_tool_choice(response) logger.info( f' tool_select_step scope={scope_path or "root"}: ' - f'tool={tool} images={total_images} tables={total_tables}' + f'tools={asset_tools or ["NAVIGATE"]} images={total_images} tables={total_tables}' ) - return tool + return asset_tools # --------------------------------------------------------------------------- @@ -600,10 +607,15 @@ async def asset_filter_step( asset_result = await db.execute(asset_stmt) asset_rows = asset_result.all() - # 3. Also find assets referenced via connect_to from text chunks + section_path_by_id = {section_id: section_path for section_id, section_path in section_rows} + + # 3. Resolve media → owner text section via connect_to tracing text_stmt = ( select( + DocumentChunk.section_id, + DocumentChunk.chunk_type, DocumentChunk.chunk_metadata, + DocumentChunk.source_chunk_path, ) .where(DocumentChunk.document_id == document_id) .where(DocumentChunk.job_result_id == job_result_id) @@ -611,14 +623,20 @@ async def asset_filter_step( .where(DocumentChunk.chunk_type == 'text') ) text_result = await db.execute(text_stmt) - connected_target_ids: set[str] = set() - for (metadata,) in text_result.all(): - if not isinstance(metadata, dict): - continue - for conn in metadata.get('connect_to') or []: - target_id = conn.get('target', '') - if target_id: - connected_target_ids.add(target_id) + text_row_dicts = [ + { + 'chunk_type': chunk_type, + 'chunk_metadata': metadata or {}, + 'section_id': sid, + 'section_path': section_path_by_id.get(sid, ''), + 'source_chunk_path': scp, + } + for sid, chunk_type, metadata, scp in text_result.all() + ] + owner_by_target_id = _build_connected_owner_map(text_row_dicts) + + # Collect connected target IDs for batch-loading + connected_target_ids: set[str] = set(owner_by_target_id.keys()) # Load connected targets that match asset_type if connected_target_ids: @@ -663,6 +681,29 @@ async def asset_filter_step( if chunk_id in seen_ids: continue seen_ids.add(chunk_id) + + # Owner resolution: prefer connect_to-based owner + owner_section_path = owner_by_target_id.get(chunk_id) + + # Fallback: media's own section_id path, but guard against + # Root / top-level aggregation sections + if not owner_section_path: + own_section_path = section_path_by_id.get(row[4]) + if own_section_path and ' / ' not in own_section_path: + # Reject document-root level sections as fallback owners + logger.warning( + f' asset_filter_step: rejecting root-level owner fallback ' + f'chunk_id={chunk_id} section_path={own_section_path}' + ) + own_section_path = None + owner_section_path = own_section_path + + if not owner_section_path: + logger.warning( + f' asset_filter_step unresolved owner: chunk_id={chunk_id} ' + f'file_path={row[3]} scope={scope_path or "root"}' + ) + continue chunks.append({ 'document_id': document_id, 'chunk_id': chunk_id, @@ -670,6 +711,8 @@ async def asset_filter_step( 'content': row[2], 'file_path': row[3], 'section_id': row[4], + 'section_path': owner_section_path, + 'owner_section_path': owner_section_path, 'source_chunk_path': row[5], 'chunk_metadata': row[6] or {}, 'sort_order': row[7], @@ -707,6 +750,7 @@ async def scope_navigate_step( scope_path: str | None = None, exclude_paths: set[str] | None = None, revision_hint: str | None = None, + budget_snapshot: dict | None = None, ) -> tuple[DocTreeNode, list[dict]]: """Single navigation step — one LLM call, no recursion. @@ -743,6 +787,7 @@ async def scope_navigate_step( doc_name=doc_name or document_id, doc_id=document_id, scope_header=scope_header, + budget_block=_format_budget_block(budget_snapshot), items_overview=text, query=query, ) @@ -772,6 +817,7 @@ async def scope_navigate_step( ] pending: list[dict] = [] + path_selections = [] for sel in valid_selections: path = sel['path'] conf = sel.get('confidence', 0.7) @@ -779,33 +825,63 @@ async def scope_navigate_step( node.confidence[path] = conf if item.get('is_leaf'): - # Leaf → hydrate all chunk types - chunks = await _hydrate_paths_to_rows( + path_selections.append({'path': path, 'confidence': conf, 'hydrate_mode': 'chunks'}) + else: + pending.append(sel) + # ★ NEW: Also hydrate this node's OWN direct chunks (not descendants) + path_selections.append({'path': path, 'confidence': conf, 'hydrate_mode': 'self_only'}) + + if path_selections: + chunks = await _hydrate_paths_to_rows( + db, + path_selections=path_selections, + user_id=user_id, + namespace=namespace, + document_id=document_id, + ) + if chunks: + connected = await hydrate_connected_target_rows( + db=db, + rows=chunks, + exclude_document_ids=[], + exclude_sections=[], + ) + if connected: + # Resolve owner_section_path for connected assets: + # map target_chunk_id → the section_path of the text chunk + # that references it via connect_to. + _owner_map = _build_connected_owner_map(chunks) + for c in connected: + if not c.get('owner_section_path'): + c['owner_section_path'] = _owner_map.get(str(c.get('chunk_id') or '')) + chunks = chunks + connected + + # Resolve Root-stranded assets to their true owner sections + # via document-wide connect_to lookup + _root_map = await _resolve_root_asset_owners( db, - path_selections=[ - {'path': path, 'confidence': conf, 'hydrate_mode': 'chunks'} - ], - user_id=user_id, - namespace=namespace, document_id=document_id, + job_result_id=job_result_id, + chunks=chunks, ) - # Also hydrate connected targets (image/table chunks referenced via connect_to) - if chunks: - connected = await hydrate_connected_target_rows( - db=db, - rows=chunks, - exclude_document_ids=[], - exclude_sections=[], - ) - if connected: - chunks = chunks + connected - node.leaf_content[path] = chunks - else: - # Non-leaf → return as pending for orchestrator to queue - pending.append(sel) + if _root_map: + for c in chunks: + if c.get('owner_section_path'): + continue # already resolved by batch-level owner map + cid = str(c.get('chunk_id') or '') + if cid in _root_map: + c['owner_section_path'] = _root_map[cid] + + for chunk in chunks: + # Distribute chunk to its real path or fallback to the selection path + real_path = chunk.get('owner_section_path') or chunk.get('section_path') or chunk.get('source_chunk_path') + if real_path: + node.add_leaf_chunks(str(real_path), [chunk]) return node, pending + except BudgetExceeded: + raise except Exception as e: logger.error(f' scope_navigate_step failed for doc={document_id}: {e}') return empty, [] @@ -829,6 +905,7 @@ async def discovery_select_step( doc_name: str = '', discovery_hints: list[dict[str, Any]], revision_hint: str | None = None, + budget_snapshot: dict | None = None, ) -> DocTreeNode: """Post-navigation discovery selection step. @@ -855,7 +932,7 @@ async def discovery_select_step( hint_by_path: dict[str, dict] = {} for h in hints: sp = h.get('section_path', '') - if not sp: + if not sp or sp == 'Root': continue title = sp.rsplit(' / ', 1)[-1] if ' / ' in sp else sp summary = h.get('summary', '') or '' @@ -882,6 +959,7 @@ async def discovery_select_step( prompt = _DISCOVERY_SELECT_PROMPT.format( doc_name=doc_name or document_id, + budget_block=_format_budget_block(budget_snapshot), items=items_text, query=query, revision_context=revision_context, @@ -896,16 +974,17 @@ async def discovery_select_step( # 2. Hydrate selected paths valid_selections = [s for s in selections if s['path'] in hint_by_path] + path_selections = [] for sel in valid_selections: path = sel['path'] conf = sel.get('confidence', 0.7) node.confidence[path] = conf + path_selections.append({'path': path, 'confidence': conf}) + if path_selections: chunks = await _hydrate_paths_to_rows( db, - path_selections=[ - {'path': path, 'confidence': conf, 'hydrate_mode': 'chunks'} - ], + path_selections=path_selections, user_id=user_id, namespace=namespace, document_id=document_id, @@ -918,8 +997,36 @@ async def discovery_select_step( exclude_sections=[], ) if connected: + _owner_map = _build_connected_owner_map(chunks) + for c in connected: + if not c.get('owner_section_path'): + c['owner_section_path'] = _owner_map.get(str(c.get('chunk_id') or '')) chunks = chunks + connected - node.leaf_content[path] = chunks + + # Resolve Root-stranded assets to their true owner sections + _disc_job_result_id = next( + (str(c['job_result_id']) for c in chunks if c.get('job_result_id')), + None, + ) + _root_map = await _resolve_root_asset_owners( + db, + document_id=document_id, + job_result_id=_disc_job_result_id, + chunks=chunks, + ) if _disc_job_result_id else {} + if _root_map: + for c in chunks: + if c.get('owner_section_path'): + continue # already resolved by batch-level owner map + cid = str(c.get('chunk_id') or '') + if cid in _root_map: + c['owner_section_path'] = _root_map[cid] + + for chunk in chunks: + # Distribute chunk to its real path or fallback to the selection path + real_path = chunk.get('owner_section_path') or chunk.get('section_path') or chunk.get('source_chunk_path') + if real_path: + node.add_leaf_chunks(str(real_path), [chunk]) latency = int((time.monotonic() - t0) * 1000) logger.info( @@ -928,6 +1035,8 @@ async def discovery_select_step( ) return node + except BudgetExceeded: + raise except Exception as e: logger.error(f' discovery_select_step failed for doc={document_id}: {e}') return node diff --git a/packages/shared-python/shared/services/retrieval/agentic/trace.py b/packages/shared-python/shared/services/retrieval/agentic/trace.py index b504971eb..8a58a243c 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/trace.py +++ b/packages/shared-python/shared/services/retrieval/agentic/trace.py @@ -115,6 +115,7 @@ def record_step( 'observation_payload_keys': list(result.payload.keys()) if result.payload else [], 'latency_ms': result.latency_ms, 'error': result.error, + 'tokens_used': result.tokens_used, 'created_at': _now_utc(), }) @@ -128,6 +129,7 @@ def record_budget_stop(self, reason: str) -> None: 'observation_payload_keys': [], 'latency_ms': 0, 'error': None, + 'tokens_used': 0, 'created_at': _now_utc(), }) @@ -135,6 +137,7 @@ async def complete( self, ranked_rows: list[dict[str, Any]], router_used: str, + budget_snapshot: dict[str, Any] | None = None, ) -> None: """Flush all step records and update the run row. Best-effort.""" if not self._created: @@ -155,6 +158,7 @@ async def complete( observation={ 'status': step_data['observation_status'], 'payload_keys': step_data['observation_payload_keys'], + 'tokens_used': step_data.get('tokens_used', 0), }, latency_ms=step_data['latency_ms'], error=step_data.get('error'), @@ -173,6 +177,8 @@ async def complete( 'step_count': len(self._steps), 'final_doc_ids': doc_ids_in_result, } + if budget_snapshot is not None: + provenance['budget_snapshot'] = budget_snapshot stmt = ( update(RetrievalRun) diff --git a/packages/shared-python/shared/services/retrieval/agentic/types.py b/packages/shared-python/shared/services/retrieval/agentic/types.py index e9d5f404c..498bf08b5 100644 --- a/packages/shared-python/shared/services/retrieval/agentic/types.py +++ b/packages/shared-python/shared/services/retrieval/agentic/types.py @@ -10,6 +10,8 @@ from dataclasses import dataclass, field from typing import Any +from shared.services.retrieval.agentic.budget import BudgetLedger + @dataclass class AgentRunConfig: @@ -17,6 +19,11 @@ class AgentRunConfig: max_revisions: int = 2 # max attempt_answer → revision cycles max_nav_depth: int = 3 # max scope_navigate recursion depth latency_budget_ms: int = 12000 + token_budget_total: int = 40000 + planning_ratio: float = 0.5 + bootstrap_budget: int = 2000 + per_doc_min_share: int = 1500 + inventory_aware: bool = True @dataclass @@ -30,6 +37,7 @@ class ToolResult: payload: dict[str, Any] = field(default_factory=dict) latency_ms: int = 0 error: str | None = None + tokens_used: int = 0 @dataclass @@ -99,6 +107,33 @@ def flatten_chunk_rows(self) -> list[dict[str, Any]]: rows.extend(child.flatten_chunk_rows()) return rows + def add_leaf_chunks(self, path: str, chunks: list[dict[str, Any]]) -> None: + """Merge chunks into a leaf path, deduplicating by (chunk_id, path).""" + if not path or not chunks: + return + existing = self.leaf_content.setdefault(path, []) + seen: set[tuple[str, str]] = { + (str(row.get('chunk_id') or ''), path) + for row in existing + if row.get('chunk_id') + } + for chunk in chunks: + chunk_id = str(chunk.get('chunk_id') or '') + key = (chunk_id, path) + if chunk_id and key in seen: + continue + if chunk_id: + seen.add(key) + existing.append(chunk) + + def reparent_leaf_content(self) -> None: + """Move descendant leaf paths into matching child nodes.""" + for child_path, child in list(self.children.items()): + for leaf_path in list(self.leaf_content.keys()): + if leaf_path == child_path or leaf_path.startswith(child_path + ' / '): + child.add_leaf_chunks(leaf_path, self.leaf_content.pop(leaf_path)) + child.reparent_leaf_content() + def collect_referenced_ids(self) -> list[dict[str, str]]: """Extract minimal chunk references from all hydrated leaves. @@ -133,8 +168,7 @@ def merge(self, other: 'DocTreeNode') -> None: if item.get('path', '') not in existing_paths: self.outline_items.append(item) for path, chunks in other.leaf_content.items(): - if path not in self.leaf_content: - self.leaf_content[path] = chunks + self.add_leaf_chunks(path, chunks) for path, child in other.children.items(): if path in self.children: self.children[path].merge(child) @@ -142,6 +176,7 @@ def merge(self, other: 'DocTreeNode') -> None: self.children[path] = child for path, conf in other.confidence.items(): self.confidence[path] = max(self.confidence.get(path, 0), conf) + self.reparent_leaf_content() @dataclass @@ -166,11 +201,16 @@ class AgenticResult: - ``referenced_chunks``: minimal chunk references for hit stats and frontend display (chunk_id, document_id, chunk_type, etc.) - ``router_used``: routing path identifier + - ``budget_snapshot``: final budget ledger state at run completion + - ``stop_reason``: why the run terminated (answer_done / max_revisions / + latency_budget / context_budget / no_llm / etc.) """ evidence_text: str answer_text: str = '' referenced_chunks: list[dict[str, str]] = field(default_factory=list) router_used: str = 'agentic_discovery_only' + budget_snapshot: dict[str, Any] | None = None + stop_reason: str = '' @dataclass @@ -201,6 +241,12 @@ class AgentState: ever_explored_doc_ids: set[str] = field(default_factory=set) seen_section_keys: set[str] = field(default_factory=set) # "{doc_id}::{section_path}" + # Token budget + KG inventory + ledger: BudgetLedger | None = None + kg_total_chunks: int = 0 + kg_total_docs: int = 0 + explored_chunks: int = 0 + @property def elapsed_ms(self) -> int: return int((time.monotonic() - self.start_time) * 1000) diff --git a/packages/shared-python/shared/services/retrieval/app_service.py b/packages/shared-python/shared/services/retrieval/app_service.py index 93f6d6d03..c066a1c16 100644 --- a/packages/shared-python/shared/services/retrieval/app_service.py +++ b/packages/shared-python/shared/services/retrieval/app_service.py @@ -850,9 +850,13 @@ async def _hydrate_paths_to_rows( # ── Chunk modes: load real chunks with optional type filters ───────── if chunk_paths: section_path_filters = [] + # Separate self_only paths (exact match only, no descendant LIKE) + # from regular chunk paths (exact + descendant subtree match) + self_only_paths = {p for p in chunk_paths if mode_by_path.get(p) == 'self_only'} for path in chunk_paths: section_path_filters.append(DocumentSection.section_path == path) - section_path_filters.append(DocumentSection.section_path.like(f'{path} / %')) + if path not in self_only_paths: + section_path_filters.append(DocumentSection.section_path.like(f'{path} / %')) stmt = ( select(Document, DocumentChunk, DocumentSection, JobResult) @@ -877,6 +881,7 @@ async def _hydrate_paths_to_rows( # Build a map of path → allowed chunk_types based on hydrate_mode _MODE_ALLOWED_TYPES: dict[str, set[str] | None] = { 'chunks': None, # all types + 'self_only': None, # all types, but without descendant filtering 'assets_only': {'image', 'table'}, 'image_only': {'image'}, 'table_only': {'table'}, diff --git a/packages/shared-python/shared/services/retrieval/llm_adapter.py b/packages/shared-python/shared/services/retrieval/llm_adapter.py index ba5093701..c6566d5f3 100644 --- a/packages/shared-python/shared/services/retrieval/llm_adapter.py +++ b/packages/shared-python/shared/services/retrieval/llm_adapter.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +from contextvars import ContextVar from typing import Any, Callable, Coroutine, Union, Sequence, cast from loguru import logger @@ -15,6 +16,11 @@ # LLMFn accepts either a plain string or a list of ChatCompletionMessageParam LLMFnInput = Union[str, Sequence[dict[str, Any]]] LLMFn = Callable[[LLMFnInput], Coroutine[Any, Any, str]] +LLMUsage = dict[str, int] +current_llm_usage: ContextVar[LLMUsage | None] = ContextVar( + 'current_llm_usage', + default=None, +) _RETRIEVAL_LLM_TEMPERATURE = 0.1 _RETRIEVAL_LLM_MAX_TOKENS = 2048 @@ -69,24 +75,16 @@ async def llm_fn(prompt: LLMFnInput) -> str: from shared.utils.OpenAICompatibleClientSync import get_openai_client client = get_openai_client(model=effective_model) - try: - result = await asyncio.to_thread( - client.chat_completion, - cast(Any, prompt), - model=effective_model, - temperature=temperature, - max_tokens=max_tokens, - ) - return result - except Exception as exc: - logger.warning( - "retrieval: agent LLM call failed (degrading gracefully): " - "model={} error_type={} error={}", - effective_model, - type(exc).__name__, - exc, - ) - return '' + current_llm_usage.set(None) + result, usage = await asyncio.to_thread( + client.chat_completion_with_usage, + cast(Any, prompt), + model=effective_model, + temperature=temperature, + max_tokens=max_tokens, + ) + current_llm_usage.set(usage) + return result return llm_fn @@ -118,23 +116,15 @@ async def vlm_fn(prompt: LLMFnInput) -> str: from shared.utils.OpenAICompatibleClientSync import get_openai_client client = get_openai_client(model=effective_model) - try: - result = await asyncio.to_thread( - client.chat_completion, - cast(Any, prompt), - model=effective_model, - temperature=temperature, - max_tokens=max_tokens, - ) - return result - except Exception as exc: - logger.warning( - "retrieval: VLM call failed (degrading gracefully): " - "model={} error_type={} error={}", - effective_model, - type(exc).__name__, - exc, - ) - return '' + current_llm_usage.set(None) + result, usage = await asyncio.to_thread( + client.chat_completion_with_usage, + cast(Any, prompt), + model=effective_model, + temperature=temperature, + max_tokens=max_tokens, + ) + current_llm_usage.set(usage) + return result return vlm_fn diff --git a/packages/shared-python/shared/services/retrieval/publication_service.py b/packages/shared-python/shared/services/retrieval/publication_service.py index 5322fa809..80aaa4ae0 100644 --- a/packages/shared-python/shared/services/retrieval/publication_service.py +++ b/packages/shared-python/shared/services/retrieval/publication_service.py @@ -8,7 +8,6 @@ from __future__ import annotations -from collections import defaultdict from datetime import datetime, timezone from typing import Any, Dict, List, Optional from uuid import uuid4 @@ -36,139 +35,6 @@ def utc_now_naive() -> datetime: class RetrievalPublicationService: - - # ── Chunk-level content-hash dedup ────────────────────────────────── - # Mirrors graph_builder._dedup_chunks_by_content but operates on - # the DB document_chunks table instead of local knowledge_graph.json. - - @staticmethod - def _collect_existing_chunk_id_map( - db: Session, - *, - user_id: str, - namespace: str, - ) -> Dict[str, str]: - """Return {chunk_id -> document_id} for all active document chunks - in the given (user_id, namespace) scope. - - Only considers chunks belonging to the *current* revision of each - active document (Document.current_job_result_id == DocumentChunk.job_result_id). - """ - rows = db.execute( - select(DocumentChunk.chunk_id, DocumentChunk.document_id) - .join( - Document, - (Document.document_id == DocumentChunk.document_id) - & (Document.current_job_result_id == DocumentChunk.job_result_id), - ) - .where( - Document.user_id == user_id, - Document.namespace == namespace, - Document.status == "active", - ) - ).all() - return {row[0]: row[1] for row in rows} - - @staticmethod - def _dedup_chunks_by_content( - new_chunks: List[Dict[str, Any]], - existing_chunk_map: Dict[str, str], - ) -> tuple[List[Dict[str, Any]], Dict[str, int]]: - """Filter new_chunks: discard any whose chunk_id already exists. - - Uses the same deterministic know_id (content-hash) comparison as - graph_builder._dedup_chunks_by_content. - - Returns: - (deduped_chunks, overlap_by_document) - - deduped_chunks: chunks whose chunk_id is NOT in existing_chunk_map - - overlap_by_document: {document_id: count} of skipped chunks per - existing document (for observability logging) - """ - overlap_by_document: Dict[str, int] = defaultdict(int) - deduped: List[Dict[str, Any]] = [] - skipped = 0 - - for chunk in new_chunks: - cid = str(chunk.get("chunk_id") or chunk.get("know_id", "")) - if cid and cid in existing_chunk_map: - skipped += 1 - overlap_by_document[existing_chunk_map[cid]] += 1 - else: - deduped.append(chunk) - - if skipped > 0: - logger.warning( - f"📊 DB chunk dedup: {skipped}/{len(new_chunks)} duplicate chunks " - f"skipped (by chunk_id), {len(deduped)} new chunks to insert. " - f"Overlap by document: {dict(overlap_by_document)}" - ) - return deduped, dict(overlap_by_document) - - @classmethod - def garbage_collect_and_dedup_local_media( - cls, - db: Session, - *, - job_id: str, - user_id: str, - namespace: str, - add_dir: str, - chunks: List[Dict[str, Any]], - ) -> tuple[List[Dict[str, Any]], Dict[str, Any]]: - """ - Deduplicates chunks against the DB and physically deletes associated redundant - media files (images/tables) from the local add_dir before ZIP packaging. - Returns the deduplicated chunks. - """ - import os - - logger.info(f"[{job_id}] Starting local GC for redundant media files in namespace: {namespace}...") - try: - existing_map = cls._collect_existing_chunk_id_map( - db, user_id=user_id, namespace=namespace - ) - deduped_chunks, overlap = cls._dedup_chunks_by_content(chunks, existing_map) - - stats = { - "total_incoming": len(chunks), - "duplicates_skipped": len(chunks) - len(deduped_chunks), - "new_chunks_inserted": len(deduped_chunks), - "overlap_by_document": overlap, - } - - if len(deduped_chunks) < len(chunks): - active_paths = set() - for c in deduped_chunks: - fp = c.get("metadata", {}).get("file_path") or c.get("file_path") - if fp: - active_paths.add(fp) - - deleted_count = 0 - if add_dir and os.path.exists(add_dir): - for c in chunks: - fp = c.get("metadata", {}).get("file_path") or c.get("file_path") - if fp and fp not in active_paths: - abs_path = os.path.join(add_dir, fp) - if os.path.exists(abs_path): - os.remove(abs_path) - deleted_count += 1 - - logger.info(f"[{job_id}] GC complete: permanently removed {deleted_count} redundant local media files.") - return deduped_chunks, stats - else: - logger.info(f"[{job_id}] GC complete: no redundant chunks found.") - return chunks, stats - except Exception as e: - logger.error(f"[{job_id}] GC failed (non-fatal): {e}") - stats = { - "total_incoming": len(chunks), - "duplicates_skipped": 0, - "new_chunks_inserted": len(chunks), - "overlap_by_document": {}, - } - return chunks, stats - # ── Public API ────────────────────────────────────────────────────── def get_existing_document_scope( diff --git a/packages/shared-python/shared/services/webhook/qstash_publisher.py b/packages/shared-python/shared/services/webhook/qstash_publisher.py index a3c52659c..8150100b2 100644 --- a/packages/shared-python/shared/services/webhook/qstash_publisher.py +++ b/packages/shared-python/shared/services/webhook/qstash_publisher.py @@ -15,6 +15,7 @@ import hmac import json import time +from dataclasses import dataclass from typing import Any, Dict, Optional from loguru import logger @@ -27,6 +28,16 @@ ) +@dataclass(frozen=True) +class QStashDeliveryStatus: + """Terminal delivery status observed from QStash logs.""" + + status: str + response_status_code: Optional[int] + response_body: Optional[str] + error_message: Optional[str] + + class QStashWebhookPublisher: """Publishes webhook events to customer endpoints via QStash.""" @@ -55,7 +66,7 @@ def _get_client(self) -> Any: operation="initialize_client", ) - self._client = QStash(token) + self._client = QStash(token, base_url=app_config.QSTASH_BASE_URL) return self._client def publish_event(self, event_id: str) -> Optional[str]: @@ -189,6 +200,8 @@ def _publish_to_qstash( "retry_delay": retry_delay_expression, "callback": callback_url, "failure_callback": failure_callback_url, + "deduplication_id": event_id, + "label": "knowhere-webhook", } response = client.message.publish(**publish_kwargs) @@ -199,6 +212,45 @@ def _publish_to_qstash( return message_id + def get_terminal_delivery_status( + self, + qstash_message_id: str, + ) -> Optional[QStashDeliveryStatus]: + """Read QStash logs for a terminal destination delivery state.""" + try: + from qstash.log import LogState + + response = self._get_client().log.list( + filter={"message_id": qstash_message_id}, + count=20, + ) + except Exception as exc: + logger.warning( + f"QStash delivery status lookup failed: " + f"message_id={qstash_message_id}, error={exc}" + ) + return None + + terminal_logs = sorted(response.logs, key=lambda log: log.time, reverse=True) + for log in terminal_logs: + if log.state == LogState.DELIVERED: + return QStashDeliveryStatus( + status=WebhookEventStatus.DELIVERED, + response_status_code=log.response_status, + response_body=log.response_body, + error_message=log.error, + ) + + if log.state == LogState.FAILED: + return QStashDeliveryStatus( + status=WebhookEventStatus.FAILED, + response_status_code=log.response_status, + response_body=log.response_body, + error_message=log.error, + ) + + return None + def _enrich_payload(self, db: Any, event: Any) -> Dict[str, Any]: """Enrich the webhook payload (e.g., generate fresh presigned S3 URL).""" from sqlalchemy import select diff --git a/packages/shared-python/shared/utils/OpenAICompatibleClientSync.py b/packages/shared-python/shared/utils/OpenAICompatibleClientSync.py index a1c5cc3e5..66da76a6d 100644 --- a/packages/shared-python/shared/utils/OpenAICompatibleClientSync.py +++ b/packages/shared-python/shared/utils/OpenAICompatibleClientSync.py @@ -22,6 +22,7 @@ from shared.utils.security_utils import mask_api_key LOCAL_DEBUG = os.getenv("LOCAL_DEBUG", "0") == "1" +LLMUsage = dict[str, int] _client_cache: Dict[tuple, "OpenAICompatibleClientSync"] = {} _client_cache_lock = threading.Lock() @@ -37,6 +38,21 @@ def _should_mock_llm_calls() -> bool: return bool(getattr(settings, "LLM_MOCK_ENABLED", False)) +def _empty_usage() -> LLMUsage: + return {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + +def _extract_usage(response: Any) -> LLMUsage: + usage = getattr(response, "usage", None) + if usage is None: + return _empty_usage() + return { + "prompt_tokens": int(getattr(usage, "prompt_tokens", 0) or 0), + "completion_tokens": int(getattr(usage, "completion_tokens", 0) or 0), + "total_tokens": int(getattr(usage, "total_tokens", 0) or 0), + } + + def _summarize_exception_chain(exc: Exception, *, max_depth: int = 4) -> str: parts: list[str] = [] current: BaseException | None = exc @@ -167,7 +183,7 @@ def _make_ali_pool_call( temperature: float, max_tokens: int, api_kwargs: Dict[str, Any], - ) -> str: + ) -> tuple[str, LLMUsage]: """Acquire a token, make the call, and retry inline on 429.""" from shared.utils.ali_quota_manager import get_ali_quota_manager @@ -198,7 +214,7 @@ def _make_ali_pool_call( internal_message="AI returned empty result", provider=self.default_model, ) - return choices[0].message.content or "" + return choices[0].message.content or "", _extract_usage(response) except openai.RateLimitError as exc: retry_after = _parse_retry_after(exc) quota_manager.mark_rate_limited(lease.token_id, retry_after) @@ -239,7 +255,7 @@ def _make_ali_pool_call( # ------------------------------------------------------------------ - def chat_completion( + def chat_completion_with_usage( self, messages: Union[str, List[ChatCompletionMessageParam]], model: Optional[str] = None, @@ -248,7 +264,7 @@ def chat_completion( top_p: Optional[float] = None, timeout: Optional[int] = None, **kwargs, - ) -> str: + ) -> tuple[str, LLMUsage]: all_messages: List[ChatCompletionMessageParam] if isinstance(messages, list): all_messages = messages # type: ignore[assignment] @@ -285,7 +301,7 @@ def chat_completion( return build_mock_chat_completion_response( messages=all_messages, model_name=effective_model, - ) + ), _empty_usage() # Route through Ali token pool when applicable if self._should_use_ali_pool(): @@ -332,7 +348,7 @@ def chat_completion( ) content = choices[0].message.content or "" - return content + return content, _extract_usage(response) except LLMServiceException: raise except Exception as exc: @@ -348,6 +364,27 @@ def chat_completion( original_exception=exc, ) from exc + def chat_completion( + self, + messages: Union[str, List[ChatCompletionMessageParam]], + model: Optional[str] = None, + temperature: float = 0.1, + max_tokens: int = 4096, + top_p: Optional[float] = None, + timeout: Optional[int] = None, + **kwargs, + ) -> str: + content, _usage = self.chat_completion_with_usage( + messages=messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + top_p=top_p, + timeout=timeout, + **kwargs, + ) + return content + def _parse_retry_after(exc: openai.RateLimitError) -> int: """Extract Retry-After seconds from a RateLimitError, with sane bounds.""" diff --git a/packages/shared-python/shared/utils/token_estimate.py b/packages/shared-python/shared/utils/token_estimate.py new file mode 100644 index 000000000..9d3c635d2 --- /dev/null +++ b/packages/shared-python/shared/utils/token_estimate.py @@ -0,0 +1,66 @@ +"""Token estimation helpers for retrieval budgeting. + +The estimator intentionally keeps ``tiktoken`` optional. Production +environments that install it get model-aware counts; other environments use a +conservative mixed Chinese/English heuristic with no extra dependency. +""" +from __future__ import annotations + +import re +from functools import lru_cache + + +_CJK_RE = re.compile(r"[\u4e00-\u9fff]") +_ASCII_WORD_RE = re.compile(r"[A-Za-z0-9_]+") + + +@lru_cache(maxsize=32) +def _get_tiktoken_encoding(model_hint: str | None): + try: + import tiktoken # type: ignore[import-not-found] + except Exception: + return None + + try: + if model_hint: + return tiktoken.encoding_for_model(model_hint) + except Exception: + # Model hint lookup failed; fall through to cl100k_base default + pass + + try: + return tiktoken.get_encoding("cl100k_base") + except Exception: + return None + + +def _heuristic_estimate(text: str) -> int: + if not text: + return 0 + + zh_chars = len(_CJK_RE.findall(text)) + ascii_chars = sum(len(match.group(0)) for match in _ASCII_WORD_RE.finditer(text)) + other_chars = max(len(text) - zh_chars - ascii_chars, 0) + + mixed_estimate = (zh_chars / 1.5) + (ascii_chars / 4.0) + (other_chars / 3.0) + conservative_floor = len(text) / 2.5 + return max(1, int(max(mixed_estimate, conservative_floor))) + + +def estimate_tokens(text: str, model_hint: str | None = None) -> int: + """Estimate input tokens for ``text``. + + ``model_hint`` is advisory. If no compatible tokenizer is available, the + function falls back to a deterministic heuristic. + """ + if not text: + return 0 + + encoding = _get_tiktoken_encoding(model_hint) + if encoding is not None: + try: + return len(encoding.encode(text)) + except Exception: + pass + + return _heuristic_estimate(text)