fix(llm): honor legacy llm_override endpoints behind an opt-in flag

infra/compose/.env.example

@@ -14,6 +14,21 @@ BASE_URL=
 API_KEY=
 MODEL=
 LLM_SEMAPHORE=10
+# Legacy escape hatch for pre-refactor clients. Clients may always override the
+# model name via metadata.llm_override; set this to also honor base_url/api_key.
+#
+# DANGER: the target HOST is unrestricted. With this on, any caller who can reach
+# /v1/chat/completions can make the server POST to any host it can route to — a
+# server-side request forgery primitive. It is constrained to https and to the
+# fixed /chat/completions path (no client-controlled query, fragment, or path
+# traversal), which keeps it off plaintext internal infra and out of arbitrary
+# internal endpoints — but a host that itself serves /chat/completions is still
+# reachable. Enable only where every API caller is already trusted with that.
+# The server's own API key is never forwarded to an overridden endpoint.
+#
+# Prefer registering a named endpoint under /model-endpoints and binding it to
+# the partition (chat_llm), which needs no such trade-off.
+# LLM_OVERRIDE_ALLOW_CUSTOM_ENDPOINT=true
 
 # ── VLM (vision model, used for image understanding) ────────────────────────
 # Can reuse the LLM values above if that model accepts images.

openrag/api/routers/user/chat.py

@@ -30,6 +30,7 @@
 from api.routers.user.source_links import build_document_source_link
 from api.schemas.user.chat import OpenAIChatCompletionRequest, OpenAICompletionRequest
 from core.config import load_config
+from core.config.endpoints import custom_endpoint_override_enabled
 from core.models.preset import resolve_partition_chat_llm
 from core.utils.exceptions import OpenRAGError
 from core.utils.logging import get_logger
@@ -401,9 +402,22 @@ def _apply_default_max_tokens(
     consistent with the endpoint that serves the request.
 
     An explicit client-supplied value is always honoured.
+
+    Skipped entirely when the request routes to a client-supplied endpoint: the
+    budget resolved here belongs to the *server's* endpoint and says nothing
+    about the client's provider, which may not even accept the parameter —
+    OpenAI's newer models reject ``max_tokens`` outright in favour of
+    ``max_completion_tokens``. Leaving it unset drops it from the payload
+    (``model_dump(exclude_none=True)``) and lets the provider apply its own
+    default. The preflight below is unaffected: ``validate_tokens_limit`` falls
+    back to the configured default when ``max_tokens`` is ``None``.
     """
-    if request.max_tokens is None:
-        request.max_tokens = _effective_max_output_tokens(config, partitions)
+    if request.max_tokens is not None:
+        return
+    llm_override = (getattr(request, "metadata", None) or {}).get("llm_override") or {}
+    if llm_override.get("base_url") and custom_endpoint_override_enabled():
+        return
+    request.max_tokens = _effective_max_output_tokens(config, partitions)
 
 
 def check_tokens_limit(

openrag/core/config/endpoints.py

@@ -2,10 +2,35 @@
 
 from __future__ import annotations
 
+import os
+
 from pydantic import Field
 
 from .base import ConfigMixin
 
+LLM_OVERRIDE_ENDPOINT_ENV = "LLM_OVERRIDE_ALLOW_CUSTOM_ENDPOINT"
+
+
+def custom_endpoint_override_enabled() -> bool:
+    """Is a client-supplied ``llm_override.base_url`` honored at all?
+
+    Off by default, and deliberately not host-restricted when on: enabling it
+    lets any authenticated caller make the server issue requests to an arbitrary
+    host, which is a server-side request forgery primitive. The request itself is
+    constrained (https only, fixed ``/chat/completions`` path — see
+    ``VLLMClient._resolve_endpoint_override``), but the host is not. Turn it on
+    only where every API caller is already trusted with that.
+
+    Lives here rather than beside its main consumer in ``services.inference``
+    because the API layer needs it too — the chat router skips its server-side
+    ``max_tokens`` default for overridden endpoints — and ``api -> services`` is
+    a forbidden import direction.
+
+    Read on demand rather than at import so a test — or a reloaded worker — sees
+    the current environment.
+    """
+    return os.environ.get(LLM_OVERRIDE_ENDPOINT_ENV, "").strip().lower() in ("1", "true", "yes", "on")
+
 
 class LLMParamsConfig(ConfigMixin):
     """Shared parameters for LLM/VLM endpoints."""

openrag/services/inference/vllm_client.py

@@ -16,8 +16,10 @@
 import base64
 import re
 from collections.abc import AsyncIterator, Mapping
+from urllib.parse import urlsplit
 
 import httpx
+from core.config.endpoints import LLM_OVERRIDE_ENDPOINT_ENV, custom_endpoint_override_enabled
 from core.embeddings import Embedder, embedder_registry
 from core.llm import LLM, llm_registry
 from core.utils.exceptions import (
@@ -148,10 +150,14 @@ def __init__(
         self._api_key = api_key
         self._enable_thinking = enable_thinking
         self._defaults: dict = kwargs
-        headers: dict[str, str] = {"Content-Type": "application/json"}
-        if api_key:
-            headers["Authorization"] = f"Bearer {api_key}"
-        self._client = httpx.AsyncClient(timeout=timeout, headers=headers)
+        self._allow_custom_endpoint = custom_endpoint_override_enabled()
+        # Authorization is sent per request, not set on the client: httpx merges
+        # client-level headers into every request and offers no way to drop one,
+        # so a client-level Authorization would ride along to an overridden
+        # endpoint even when the override carries no key of its own — handing the
+        # server's credential to a third-party host.
+        self._auth_headers: dict[str, str] = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+        self._client = httpx.AsyncClient(timeout=timeout, headers={"Content-Type": "application/json"})
         # Same construction breadcrumb as VLLMEmbedder: the component factories
         # cache instances per endpoint name, so this fires once per configured
         # endpoint and shows which base URL/model a preset name resolved to.
@@ -162,32 +168,143 @@ def __init__(
             enable_thinking=self._enable_thinking,
         ).debug(f"{type(self).__name__} ready")
 
-    def _resolve_overrides(self, kwargs: dict) -> tuple[str, str, dict[str, str] | None]:
+    def _resolve_overrides(self, kwargs: dict) -> tuple[str, str, dict[str, str]]:
         """Read ``metadata.llm_override`` from *kwargs* without mutating caller data.
 
         Pure read: ``kwargs`` is untouched so retries see the original override on
         every attempt. The caller strips ``metadata`` from the outbound payload via
         ``_payload_kwargs`` — every ``metadata`` key is OpenRAG-internal and never
         belongs on the wire.
+
+        Returns the headers to send, never ``None``: the server's Authorization
+        lives here rather than on the shared client precisely so an overridden
+        endpoint can be given different credentials — or none.
         """
         base_url = self._endpoint
         model = self._model
-        override_headers: dict[str, str] | None = None
+        headers = self._auth_headers
 
-        # Only `model` may be overridden by the client. `base_url` / `api_key`
-        # are deliberately NOT read from the request: honoring a client-supplied
-        # endpoint enables SSRF (the server would issue requests to an arbitrary
-        # host, e.g. cloud metadata) and would leak the server's API key to that
-        # host. The endpoint and credentials always come from server config.
+        # `model` is always client-overridable. `base_url` / `api_key` are honored
+        # only when the operator sets LLM_OVERRIDE_ALLOW_CUSTOM_ENDPOINT — see
+        # `_resolve_endpoint_override` for what that opens up.
         llm_override = (kwargs.get("metadata") or {}).get("llm_override") or {}
         if llm_override.get("model"):
             model = llm_override["model"]
 
-        return base_url, model, override_headers
+        if llm_override.get("base_url"):
+            if self._allow_custom_endpoint:
+                base_url, headers = self._resolve_endpoint_override(llm_override)
+            else:
+                # The failure this warning exists for: `model` is applied while
+                # `base_url` is dropped, so the request goes to the *server's*
+                # endpoint carrying a *third party's* model name. The provider
+                # then answers "invalid model name", which reads as a client bug
+                # and says nothing about the override having been ignored.
+                logger.bind(
+                    requested_endpoint=llm_override.get("base_url"),
+                    used_endpoint=base_url,
+                    model=model,
+                ).warning(
+                    f"Ignoring llm_override.base_url — {LLM_OVERRIDE_ENDPOINT_ENV} is not enabled. "
+                    f"The request goes to the configured endpoint with model={model!r}."
+                )
+        elif llm_override:
+            logger.bind(keys=sorted(llm_override), model=model).warning(
+                "llm_override carries no base_url; only the model name is overridden"
+            )
+
+        return base_url, model, headers
+
+    def _resolve_endpoint_override(self, llm_override: Mapping) -> tuple[str, dict[str, str]]:
+        """Honor a legacy ``llm_override`` endpoint (opt-in, host-unrestricted).
+
+        Restores the pre-refactor contract — ``base_url`` + ``api_key`` + ``model``
+        in ``metadata.llm_override`` — for deployments whose clients still send it.
+
+        With the flag on, *which host* is reached is deliberately unconstrained:
+        any caller who can reach ``/v1/chat/completions`` can make the server POST
+        to any host it can route to. What is constrained is *what the request looks
+        like*, and that is what keeps this from being a read primitive against
+        internal HTTP APIs:
+
+        * **https only.** Refusing ``http`` keeps the override off plaintext
+          internal infra — Milvus, Postgres-over-HTTP, admin panels — which is
+          where the SSRF payoff lives, those services trusting anything on their
+          network.
+        * **No client-controlled path.** The endpoint is always hit as
+          ``{base_url}/chat/completions``. A ``#`` or ``?`` in ``base_url`` would
+          truncate that appended suffix once concatenated
+          (``…/collections/list#/chat/completions`` reaches ``…/collections/list``),
+          and a ``..`` segment would traverse out of it — either turns the override
+          into a path-picker aimed at arbitrary internal endpoints. Rejected here,
+          the suffix is always appended literally, so only a host that genuinely
+          serves ``…/chat/completions`` can answer. A real LLM base URL
+          (``https://api.openai.com/v1``) trips none of these.
+        * The server's own API key is never sent to an overridden endpoint. The
+          override's ``api_key`` is used, or no ``Authorization`` at all — never a
+          fallback to ``self._api_key``.
+
+        Note the ``#``/``?`` test is on the raw string, not ``urlsplit`` parts: a
+        trailing ``#`` parses as an *empty* fragment on ``base_url`` alone and only
+        becomes meaningful after ``/chat/completions`` is concatenated, so the
+        parsed view would miss exactly the truncation being defended against.
+
+        Rejections are 4xx so ``with_retry`` (429/502/503/504 only) does not
+        re-attempt a request that can never succeed.
+        """
+        candidate = str(llm_override["base_url"]).strip()
 
-    def _chat_payload_kwargs(self, kwargs: dict) -> dict:
-        payload_kwargs = {**self._defaults, **kwargs}
-        enable_thinking = payload_kwargs.pop("enable_thinking", self._enable_thinking)
+        if "#" in candidate or "?" in candidate:
+            raise InferenceError(
+                "llm_override.base_url must not carry a query string or fragment",
+                code="LLM_OVERRIDE_REJECTED",
+                status_code=400,
+            )
+        parts = urlsplit(candidate)
+        scheme = parts.scheme.lower()
+        if scheme != "https":
+            raise InferenceError(
+                f"llm_override.base_url scheme {scheme!r} is not allowed (https only)",
+                code="LLM_OVERRIDE_REJECTED",
+                status_code=400,
+            )
+        if ".." in parts.path.split("/"):
+            raise InferenceError(
+                "llm_override.base_url must not contain a '..' path segment",
+                code="LLM_OVERRIDE_REJECTED",
+                status_code=400,
+            )
+
+        candidate = candidate.rstrip("/")
+        api_key = llm_override.get("api_key")
+        headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+        logger.bind(endpoint=candidate, configured=self._endpoint).debug("Honoring legacy llm_override endpoint")
+        return candidate, headers
+
+    def _has_endpoint_override(self, kwargs: dict) -> bool:
+        """Is this request being routed to a client-supplied endpoint?
+
+        Read before ``kwargs.pop("metadata")``. Decides whether the server's
+        sampling defaults apply — see ``_chat_payload_kwargs``.
+        """
+        if not self._allow_custom_endpoint:
+            return False
+        llm_override = (kwargs.get("metadata") or {}).get("llm_override") or {}
+        return bool(llm_override.get("base_url"))
+
+    def _chat_payload_kwargs(self, kwargs: dict, *, use_defaults: bool = True) -> dict:
+        """Merge server sampling defaults under the request's own params.
+
+        ``use_defaults=False`` for a client-supplied endpoint: the defaults
+        (``temperature``, ``logprobs``, ``enable_thinking``) describe the server's
+        *configured* model, and a different provider may reject them outright —
+        e.g. Gemini 400s on an unsolicited ``logprobs``. The pre-refactor client
+        had the same rule, and dropping it is what still made a legacy
+        ``llm_override`` fail after the endpoint itself was restored.
+        """
+        payload_kwargs = {**self._defaults, **kwargs} if use_defaults else dict(kwargs)
+        fallback_thinking = self._enable_thinking if use_defaults else None
+        enable_thinking = payload_kwargs.pop("enable_thinking", fallback_thinking)
         if enable_thinking is not None and enable_thinking is True:
             chat_template_kwargs = dict(payload_kwargs.get("chat_template_kwargs") or {})
             chat_template_kwargs.setdefault("enable_thinking", enable_thinking)
@@ -198,8 +315,9 @@ def _chat_payload_kwargs(self, kwargs: dict) -> dict:
     @with_retry(max_attempts=3)
     async def generate(self, prompt: str, **kwargs) -> dict:
         base_url, model, headers = self._resolve_overrides(kwargs)
+        overridden = self._has_endpoint_override(kwargs)
         kwargs.pop("metadata", None)
-        payload = {**self._defaults, **kwargs, "model": model, "prompt": prompt}
+        payload = {**({} if overridden else self._defaults), **kwargs, "model": model, "prompt": prompt}
         try:
             resp = await self._client.post(f"{base_url}/completions", json=payload, headers=headers)
             resp.raise_for_status()
@@ -218,8 +336,14 @@ async def generate(self, prompt: str, **kwargs) -> dict:
     @with_retry(max_attempts=3)
     async def chat(self, messages: list[dict[str, str]], **kwargs) -> dict:
         base_url, model, headers = self._resolve_overrides(kwargs)
+        overridden = self._has_endpoint_override(kwargs)
         kwargs.pop("metadata", None)
-        payload = {**self._chat_payload_kwargs(kwargs), "model": model, "messages": messages, "stream": False}
+        payload = {
+            **self._chat_payload_kwargs(kwargs, use_defaults=not overridden),
+            "model": model,
+            "messages": messages,
+            "stream": False,
+        }
         try:
             resp = await self._client.post(f"{base_url}/chat/completions", json=payload, headers=headers)
             resp.raise_for_status()
@@ -236,8 +360,14 @@ async def chat(self, messages: list[dict[str, str]], **kwargs) -> dict:
 
     async def stream_chat(self, messages: list[dict[str, str]], **kwargs) -> AsyncIterator[str]:
         base_url, model, headers = self._resolve_overrides(kwargs)
+        overridden = self._has_endpoint_override(kwargs)
         kwargs.pop("metadata", None)
-        payload = {**self._chat_payload_kwargs(kwargs), "model": model, "messages": messages, "stream": True}
+        payload = {
+            **self._chat_payload_kwargs(kwargs, use_defaults=not overridden),
+            "model": model,
+            "messages": messages,
+            "stream": True,
+        }
         try:
             async with self._client.stream(
                 "POST", f"{base_url}/chat/completions", json=payload, headers=headers
@@ -510,6 +640,10 @@ async def caption_image(self, image_bytes: bytes, prompt: str | None = None) ->
             resp = await self._client.post(
                 f"{self._endpoint}/chat/completions",
                 json=payload,
+                # Explicit because VLLMClient moved Authorization off the shared
+                # httpx client; captioning always uses the configured endpoint,
+                # so it always carries the server's key.
+                headers=self._auth_headers,
             )
             resp.raise_for_status()
         except httpx.ConnectError as exc: