From f991f726ac1a527a6fd60fda4b3ea9b3fc1cb76f Mon Sep 17 00:00:00 2001 From: NikolaI Baakh Date: Sat, 19 Sep 2026 10:50:03 +0000 Subject: [PATCH 1/4] feat(core-app): the local runtime may simply not exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `OLLAMA_URL=""` now means *there is no local runtime*, deliberately — distinct from "the runtime is unreachable" and from "misconfigured". Those three facts used to be one, so each call site guessed: some caught and degraded, some propagated to a 500, one polled for three minutes. - `CoreAppSettings.local_runtime_enabled` is the fact; every path reads it. - `GET /platform/v1/llm/local-runtime` reports `absent` | `unreachable` | `ok` plus `url_configured`. A separate endpoint: `GET /llm/models` stays a bare `list[ModelInfo]` (twelve consumers) and never 500s again — `[]` in both non-serving states, which is the regression fix for the 500 the Models page collected every ten seconds. - pull / pull-stream / delete / unload refuse with 409 when absent and 502 when a configured runtime is unreachable, never a bare 500; the SSE pull refuses before the response starts. The KV-cache apply refuses with 409 when absent and persists nothing — it never talks to Ollama, so "unreachable" is not a state it can observe. - `_ensure_can_serve` refuses a local model id with the existing `ModelCapabilityError` (ADR-0140), asked before the pause rule, since "resume to run inference" is an instruction an operator with no runtime cannot follow. A hosted embedding model keeps working; a bare one fails with the fix in the message instead of a connection error. The chat fallback chain skips local candidates for the same reason. - Readiness reports the model `n/a` and ready, instead of warming forever. - The bootstrap returns in one log line, with no 180s poll, and now waits on the runtime *state* rather than on `models()` raising. - The KV-cache apply returns its existing unavailable result and asks neither arm of the container seam to find a workload. Two test fixes ride along: the tool-rejection warning and the api-key-redaction assertions moved off `capture_logs` onto the `_RecordingLog` recorder the file already documents, because structlog freezes a module logger on first use once the app has configured logging — a capture that intercepts nothing made the redaction assertion vacuously true. core-app 0.128.0 -> 0.129.0 (MINOR). Part of #962. --- services/core-app/pyproject.toml | 2 +- .../core-app/src/epicurus_core_app/app.py | 3 + .../src/epicurus_core_app/llm/bootstrap.py | 40 ++- .../src/epicurus_core_app/llm/errors.py | 38 +++ .../src/epicurus_core_app/llm/gateway.py | 195 +++++++++-- .../src/epicurus_core_app/llm/models.py | 36 +- .../epicurus_core_app/llm/ollama_runtime.py | 20 +- .../src/epicurus_core_app/llm/routes.py | 127 ++++++- .../src/epicurus_core_app/readiness.py | 24 +- .../src/epicurus_core_app/settings.py | 16 + services/core-app/tests/test_llm_bootstrap.py | 40 ++- services/core-app/tests/test_llm_gateway.py | 317 ++++++++++++++++-- services/core-app/tests/test_llm_routes.py | 149 +++++++- .../core-app/tests/test_ollama_runtime.py | 21 ++ services/core-app/tests/test_readiness.py | 41 ++- services/core-app/tests/test_settings.py | 32 ++ uv.lock | 2 +- 17 files changed, 1002 insertions(+), 101 deletions(-) diff --git a/services/core-app/pyproject.toml b/services/core-app/pyproject.toml index 06246703..138e30eb 100644 --- a/services/core-app/pyproject.toml +++ b/services/core-app/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "epicurus-core-app" -version = "0.128.0" +version = "0.129.0" description = "The epicurus core runtime: hosts the agent loop, LLM gateway, memory, power states, and the module-facing platform API." requires-python = ">=3.11" dependencies = [ diff --git a/services/core-app/src/epicurus_core_app/app.py b/services/core-app/src/epicurus_core_app/app.py index 4e698df6..e83d9c66 100644 --- a/services/core-app/src/epicurus_core_app/app.py +++ b/services/core-app/src/epicurus_core_app/app.py @@ -507,6 +507,9 @@ async def embed(texts: list[str]) -> list[list[float]]: docker, env_path=settings.ollama_runtime_env_path, service=settings.ollama_service_name, + # A hosted-only deployment has no Ollama workload on either runtime arm (#962, + # ADR-0144), so the KV-cache apply neither writes nor restarts anything. + local_runtime_enabled=settings.local_runtime_enabled, ) # Agent tool discovery scans only enabled modules (#126); wired after the registry # exists (the host needs the registry and the registry needs the host). diff --git a/services/core-app/src/epicurus_core_app/llm/bootstrap.py b/services/core-app/src/epicurus_core_app/llm/bootstrap.py index cb1a6de0..e5e223de 100644 --- a/services/core-app/src/epicurus_core_app/llm/bootstrap.py +++ b/services/core-app/src/epicurus_core_app/llm/bootstrap.py @@ -34,13 +34,19 @@ from epicurus_core import get_logger from epicurus_core_app.llm import providers as registry +from epicurus_core_app.llm.errors import LocalRuntimeState from epicurus_core_app.llm.models import ModelInfo log = get_logger("epicurus_core_app.llm.bootstrap") class _Gateway(Protocol): - """The four gateway methods the bootstrap needs (kept narrow for tests).""" + """The four gateway methods (and one property) the bootstrap needs (narrow, for tests).""" + + @property + def local_runtime_enabled(self) -> bool: ... + + async def local_runtime_state(self) -> LocalRuntimeState: ... async def models( self, tenant_id: str | None = None, *, with_capabilities: bool = False @@ -100,6 +106,15 @@ async def run(self) -> None: log.error("model bootstrap failed unexpectedly", error=str(exc)) async def _run(self) -> None: + if not self._gateway.local_runtime_enabled: + # No local runtime to seed, by the operator's choice (#962, ADR-0144). One line + # and out: before this, a hosted-only deployment spent 180 s of every process + # start polling an address that was never going to answer — the poll loop's own + # docstring said "a hosted-only deployment may run no Ollama at all" while having + # no way to know that it was one. + log.info("model bootstrap skipped: this deployment runs no local LLM runtime") + return + spec = self._models_spec.strip() if not spec: log.info("model bootstrap disabled (LLM_BOOTSTRAP_MODELS is blank)") @@ -138,18 +153,25 @@ async def _run(self) -> None: async def _wait_for_runtime(self) -> set[str] | None: """Poll the runtime until it answers, returning the installed (tagged) model names. - ``None`` after the deadline — a hosted-only deployment may run no Ollama at all, - and that must cost one warning, not a crash loop. + ``None`` after the deadline — a configured runtime that is still down when the clock + runs out costs one warning, not a crash loop. Only reached when a runtime *is* + configured: the absent case returns from :meth:`_run` before any of this. + + It polls the **state**, not a raised exception. ``models()`` no longer raises for a + runtime that cannot answer (#962) — it reports an empty list, which is the truth for + a caller asking what is installed and a trap for a caller asking whether the runtime + is up: an Ollama container thirty seconds into its own start-up would have read as + "an empty runtime", and the bootstrap would have raced it with a pull instead of + waiting the few seconds it needed. """ deadline = asyncio.get_running_loop().time() + self._ready_timeout_s while True: - try: + if await self._gateway.local_runtime_state() == "ok": return {info.name for info in await self._gateway.models()} - except Exception as exc: - if asyncio.get_running_loop().time() >= deadline: - log.debug("runtime still unreachable at deadline", error=str(exc)) - return None - await asyncio.sleep(self._poll_interval_s) + if asyncio.get_running_loop().time() >= deadline: + log.debug("runtime still unreachable at deadline", waited_s=self._ready_timeout_s) + return None + await asyncio.sleep(self._poll_interval_s) async def _resolve_wanted(self, spec: str) -> list[str]: """The models to ensure: the effective defaults for ``auto``, else the explicit list. diff --git a/services/core-app/src/epicurus_core_app/llm/errors.py b/services/core-app/src/epicurus_core_app/llm/errors.py index 407fddda..c30c82fa 100644 --- a/services/core-app/src/epicurus_core_app/llm/errors.py +++ b/services/core-app/src/epicurus_core_app/llm/errors.py @@ -7,6 +7,16 @@ from __future__ import annotations +from typing import Literal + +LocalRuntimeState = Literal["absent", "unreachable", "ok"] +"""What the deployment's local LLM runtime is doing (#962, ADR-0144). + +``absent`` — none is configured (``OLLAMA_URL`` blank), deliberately. ``unreachable`` — one is +configured and did not answer. ``ok`` — it answered. Every local-runtime surface branches on +exactly these three; nothing anywhere may re-derive them from a caught exception. +""" + class ModelCapabilityError(RuntimeError): """The selected model cannot serve this request (#944, #947). @@ -29,3 +39,31 @@ def __init__(self, *, model: str, capability: str, message: str, hint: str) -> N def __str__(self) -> str: return self.message + + +class LocalRuntimeUnavailableError(RuntimeError): + """A local-runtime-only action was asked of a deployment that cannot serve it (#962). + + Carries which of the two non-serving states applies, because they are different facts + with different answers, and collapsing them is what made every one of these paths a bare + 500 (ADR-0144): + + * ``absent`` — no local runtime is configured (``OLLAMA_URL`` is blank). A deliberate + hosted-only deployment. Nothing is wrong; the action simply does not exist here, so the + surface answers **409**. + * ``unreachable`` — a runtime *is* configured and did not answer. That is an error, and + the surface answers **502**. + + Args: + state: ``"absent"`` or ``"unreachable"``. + message: Operator-readable explanation naming the mode. No payloads, no URLs with + credentials in them. + """ + + def __init__(self, *, state: LocalRuntimeState, message: str) -> None: + super().__init__(message) + self.state = state + self.message = message + + def __str__(self) -> str: + return self.message diff --git a/services/core-app/src/epicurus_core_app/llm/gateway.py b/services/core-app/src/epicurus_core_app/llm/gateway.py index d484daa8..c559c0db 100644 --- a/services/core-app/src/epicurus_core_app/llm/gateway.py +++ b/services/core-app/src/epicurus_core_app/llm/gateway.py @@ -33,7 +33,11 @@ estimate_tools_tokens, reply_reserve, ) -from epicurus_core_app.llm.errors import ModelCapabilityError +from epicurus_core_app.llm.errors import ( + LocalRuntimeState, + LocalRuntimeUnavailableError, + ModelCapabilityError, +) from epicurus_core_app.llm.model_settings import ModelSettings, ModelSettingsStore from epicurus_core_app.llm.models import ( ChatMessage, @@ -42,6 +46,7 @@ ModelDetails, ModelInfo, ModelRole, + ModelWarmth, ProviderInfo, StreamEvent, ToolCallFragment, @@ -259,7 +264,11 @@ def __init__( model_settings: ModelSettingsStore | None = None, saved_models: SavedHostedModelStore | None = None, ) -> None: - self._ollama_url = ollama_url.rstrip("/") + # Blank (after stripping) is the *absent* state, not a bad URL: this deployment runs no + # local runtime at all (#962, ADR-0144). Stored normalised so every call site can ask + # one cheap question — :attr:`local_runtime_enabled` — instead of guessing from a + # caught connection error. + self._ollama_url = ollama_url.strip().rstrip("/") self._default_model = default_model self._default_embed_model = default_embed_model self._keep_alive = keep_alive @@ -297,6 +306,56 @@ def __init__( # blind for the rest of the process. Bounded, like ``_unmapped_models``. self._local_roles: dict[str, ModelRole] = {} + # ── the local runtime: present, absent, or unreachable (#962, ADR-0144) ────── + + @property + def local_runtime_enabled(self) -> bool: + """Whether this deployment has a local LLM runtime configured at all. + + False is *absent*: a deliberate hosted-only deployment (``OLLAMA_URL`` blank). It is a + fact about the configuration, costs no I/O, and is therefore what every hot path asks + — the role gate on each call, the readiness probe, the model list — before it considers + touching the network. + """ + return bool(self._ollama_url) + + def require_local_runtime(self, action: str) -> None: + """Refuse ``action`` when there is no local runtime to perform it on. + + The single place the *absent* refusal is worded, so the 409 an operator sees names the + deployment mode rather than leaking a connection error from four different call sites. + Public because two callers cannot rely on the in-method guard: the SSE pull route (an + async generator's body does not run until it is iterated, by which time the response + has started) and the KV-cache route (which must refuse *before* persisting the choice). + """ + if self._ollama_url: + return + raise LocalRuntimeUnavailableError( + state="absent", + message=( + f"cannot {action}: this deployment runs no local LLM runtime " + "(OLLAMA_URL is blank — hosted models only)" + ), + ) + + async def local_runtime_state(self) -> LocalRuntimeState: + """Probe the local runtime and report which of the three states applies. + + ``absent`` costs nothing (no URL, no call). Otherwise ``/api/tags`` is asked with a + short timeout — the same endpoint :meth:`models` uses, so "the list is empty because + the runtime is down" and "the state is unreachable" can never disagree. + """ + if not self._ollama_url: + return "absent" + try: + async with httpx.AsyncClient(base_url=self._ollama_url, timeout=10) as client: + response = await client.get("/api/tags") + response.raise_for_status() + except (httpx.HTTPError, ValueError) as exc: + log.warning("local runtime unreachable", error=str(exc)) + return "unreachable" + return "ok" + async def effective_default(self, tenant_id: str | None = None) -> str: """The active default model: the stored pref if set, else the env default.""" if self._prefs is not None: @@ -371,29 +430,36 @@ async def _settings_for(self, model: str, tenant_id: str | None) -> ModelSetting async def model_readiness( self, model: str | None = None, *, tenant_id: str | None = None - ) -> tuple[str, bool | None]: + ) -> ModelWarmth: """Report whether a model is ready to answer *now* (ADR-0027). - Returns ``(resolved_model, warm)``. ``warm`` is ``None`` for hosted providers — they - need no local warm-up, so they are always ready; for the local runtime it is ``True`` - only when the model is already loaded in memory (``False`` while paused, or cold). - Best-effort: a runtime probe failure reports the model as cold rather than raising. + Returns ``(resolved_model, warm, runtime)``. ``warm`` is ``None`` whenever local + warm-up is not a question that applies, and ``runtime`` says why: ``hosted`` (a + provider needs no warm-up, so it is always ready) or ``absent`` (this deployment runs + no local runtime, #962 — the model can never warm up, and saying "warming" forever is + what made the chat progress bar lie on every turn of a hosted-only install). For a + local model on a real runtime it is ``True`` only when the model is already loaded in + memory (``False`` while paused, or cold). Best-effort: a probe failure reports the + model as cold rather than raising — *unreachable* is an error, unlike *absent*, and + cold is the honest reading of it. """ resolved = model or await self.effective_default(tenant_id) _, provider = registry.resolve(resolved) if not provider.is_local: - return resolved, None + return ModelWarmth(resolved, None, "hosted") + if not self._ollama_url: + return ModelWarmth(resolved, None, "absent") if self._power.paused: - return resolved, False + return ModelWarmth(resolved, False, "local") target = resolved.split("/", 1)[-1] # a bare local name has no prefix; this is a no-op try: loaded = {info.name for info in await self.models(tenant_id) if info.loaded} except Exception: # runtime unreachable — treat as cold, never raise into readiness log.warning("model readiness probe failed; reporting cold", model=resolved) - return resolved, False + return ModelWarmth(resolved, False, "local") # The runtime tags loaded models (e.g. "llama3.2:latest"); match the bare name too. warm = target in loaded or any(name.split(":", 1)[0] == target for name in loaded) - return resolved, warm + return ModelWarmth(resolved, warm, "local") def _candidates(self, model: str) -> list[str]: """The chosen model followed by the configured fallback chain (deduped).""" @@ -404,12 +470,18 @@ def _candidates(self, model: str) -> list[str]: return ordered def _is_available(self, model: str) -> bool: - """Unavailable only if local while paused — running it would wake the GPU. + """Whether ``model`` can be *tried* — the filter the chat fallback chain applies. - Hosted providers stay available when paused (they use no local GPU). + A local model is unavailable while the runtime is paused (running it would wake the + GPU) and on a deployment that runs no local runtime at all (#962) — there is nothing + to run it on, and a fallback chain that walked into one would trade a clean refusal + for a connection error. Hosted providers are available in both cases: they use no + local GPU and need no local runtime. """ _, provider = registry.resolve(model) - return not (self._power.paused and provider.is_local) + if not provider.is_local: + return True + return bool(self._ollama_url) and not self._power.paused async def _ensure_can_serve( self, @@ -421,10 +493,18 @@ async def _ensure_can_serve( ) -> None: """The one gate every inference entry point passes through (ADR-0140). - Two rules today, and the place the third goes — the system-level Local AI / Hosted AI - switches (#945) add a clause *here*, not a fourth copy scattered across the call sites: + Three rules today, and the place the fourth goes — the system-level Local AI / Hosted + AI switches (#945) add a clause *here*, not a fifth copy scattered across the call + sites: * **Paused** (ADR-0005): a local model cannot run while the runtime is paused. + * **No local runtime** (#962, ADR-0144): a *local* model id on a deployment that runs + no local runtime is refused here, before any provider call. This is the rule the + quiet degradation of :meth:`show` used to defeat — an unreachable ``/api/show`` + returns empty details, the role reads ``unknown``, and ``unknown`` is waved through + to fail at the provider with a connection error instead of a sentence. Absence is a + fact about the deployment, not a catalogue miss, so it is asked first and answered + definitively. * **Role** (#944): a model whose role is *known* and is not ``want`` is refused before any provider call, with a :class:`ModelCapabilityError` naming the one action that fixes it. ``unknown`` is refused nothing — a catalogue miss must never lock the @@ -437,6 +517,19 @@ async def _ensure_can_serve( :meth:`embed` has no fallback chain, so it takes the whole gate — which is what retires the inline copy of the pause rule that used to live there. """ + # Absence is asked **before** the pause rule, and the order is load-bearing: both make + # a local model unavailable, but "resume to run inference" is an instruction an + # operator with no runtime cannot follow. The more specific fact answers first. + if not self._ollama_url and registry.resolve(model)[1].is_local: + raise ModelCapabilityError( + model=model, + capability=want, + message=( + f"{model} runs on the local LLM runtime, and this deployment has none " + "configured." + ), + hint="No local runtime is configured — choose a hosted model.", + ) if check_pause and not self._is_available(model): raise GatewayPausedError("LLM gateway is paused; resume to run inference") role = await self.model_role(model, tenant_id) @@ -1111,18 +1204,31 @@ async def models( model, concurrently. It costs one extra call per model, so it is **opt-in** — the chat picker lists without it; the Models page asks for it to badge what each model can do and show its context window. + + **Never raises for a runtime that cannot answer** (#962, ADR-0144). An empty list is + the truthful answer to "which models does the local runtime hold" in both non-serving + states: *absent* (there is no runtime, and no call is made at all) and *unreachable* + (one is configured and did not answer). This is the regression fix for the 500 the + Models page collected every ten seconds; *which* state applies is a different + question, answered by :meth:`local_runtime_state` and its own endpoint. """ - async with httpx.AsyncClient(base_url=self._ollama_url, timeout=10) as client: - response = await client.get("/api/tags") - response.raise_for_status() - payload = response.json() - loaded: set[str] = set() - try: # /api/ps lists running models; best-effort decoration only - ps = await client.get("/api/ps") - ps.raise_for_status() - loaded = {m["name"] for m in ps.json().get("models", [])} - except (httpx.HTTPError, KeyError): - log.warning("ollama /api/ps failed; loaded-state unknown") + if not self._ollama_url: + return [] + try: + async with httpx.AsyncClient(base_url=self._ollama_url, timeout=10) as client: + response = await client.get("/api/tags") + response.raise_for_status() + payload = response.json() + loaded: set[str] = set() + try: # /api/ps lists running models; best-effort decoration only + ps = await client.get("/api/ps") + ps.raise_for_status() + loaded = {m["name"] for m in ps.json().get("models", [])} + except (httpx.HTTPError, KeyError): + log.warning("ollama /api/ps failed; loaded-state unknown") + except (httpx.HTTPError, ValueError) as exc: + log.warning("ollama /api/tags failed; reporting no local models", error=str(exc)) + return [] hidden: set[str] = set() if self._prefs is not None: hidden = set(await self._prefs.get_hidden(tenant_id or self._default_tenant)) @@ -1241,6 +1347,10 @@ async def show(self, model: str, tenant_id: str | None = None) -> ModelDetails: unreachable, so the model-settings sheet degrades to "unknown". The trained context length lives under ``model_info`` keyed by the architecture (e.g. ``llama.context_length``); fall back to any ``*.context_length`` if the arch is absent. + With **no** local runtime the same empty answer is returned without a call — but note + that empty details are not a licence to run the model: absence is refused by + :meth:`_ensure_can_serve` before any role question is asked (#962), precisely because + "no reported capabilities" and "no runtime to report them" used to look identical here. Hosted: LiteLLM's cost/context map is the source of truth for both capabilities and context length — no provider call, and no fake default when the model isn't in the map @@ -1250,6 +1360,9 @@ async def show(self, model: str, tenant_id: str | None = None) -> ModelDetails: _, provider = registry.resolve(model) if not provider.is_local: return await self._hosted_details(model, tenant_id) + if not self._ollama_url: + log.debug("no local runtime; reporting empty model details", model=model) + return ModelDetails() try: async with httpx.AsyncClient(base_url=self._ollama_url, timeout=10) as client: response = await client.post("/api/show", json={"model": model}) @@ -1376,7 +1489,13 @@ def _forget_local_role(self, model: str) -> None: self._local_roles.pop(model, None) async def pull(self, model: str) -> None: - """Pull a model into the local runtime (blocks until complete).""" + """Pull a model into the local runtime (blocks until complete). + + Raises :class:`LocalRuntimeUnavailableError` (``absent``) when there is no local + runtime to pull into — the route turns that into a 409 naming the mode, rather than + the 500 a connection error to a blank URL used to produce (#962). + """ + self.require_local_runtime("pull a model") self._forget_local_role(model) async with httpx.AsyncClient(base_url=self._ollama_url, timeout=None) as client: response = await client.post("/api/pull", json={"model": model, "stream": False}) @@ -1387,7 +1506,11 @@ async def pull_stream(self, model: str) -> AsyncIterator[dict[str, Any]]: Each item is Ollama's progress shape (``status``, and ``total``/``completed`` while a layer downloads) — the model-manager UI renders these directly. + + Refuses before the response starts when there is no local runtime (#962), so the + caller gets a 409 rather than an SSE stream whose only event is an error. """ + self.require_local_runtime("pull a model") self._forget_local_role(model) async with ( httpx.AsyncClient(base_url=self._ollama_url, timeout=None) as client, @@ -1400,7 +1523,11 @@ async def pull_stream(self, model: str) -> AsyncIterator[dict[str, Any]]: yield item async def delete_model(self, model: str) -> None: - """Remove a model from the local runtime.""" + """Remove a model from the local runtime. + + Raises :class:`LocalRuntimeUnavailableError` (``absent``) with no runtime (#962). + """ + self.require_local_runtime("delete a model") self._forget_local_role(model) async with httpx.AsyncClient(base_url=self._ollama_url, timeout=30) as client: response = await client.request("DELETE", "/api/delete", json={"model": model}) @@ -1411,8 +1538,16 @@ async def unload(self, model: str | None = None) -> None: With ``model`` set, unload just that one (the on-demand per-model Unload, #331); otherwise unload every installed model (the power-pause path). Never raises — a - runtime hiccup is logged, not surfaced. + runtime hiccup is logged, not surfaced, and with **no** local runtime there is + nothing loaded to drop, so it returns at once (#962). That silence is deliberate and + is why the *route* checks for absence itself and answers 409: this method is also on + the power-pause path (``PUT /platform/v1/power``), which must keep working on a + hosted-only deployment — pausing there is about the GPU it does not have, and + refusing it would break a control that has nothing to do with the local runtime. """ + if not self._ollama_url: + log.debug("no local runtime; nothing to unload", model=model) + return try: targets = [model] if model is not None else [info.name for info in await self.models()] async with httpx.AsyncClient(base_url=self._ollama_url, timeout=10) as client: diff --git a/services/core-app/src/epicurus_core_app/llm/models.py b/services/core-app/src/epicurus_core_app/llm/models.py index dd94c8ee..f5c60092 100644 --- a/services/core-app/src/epicurus_core_app/llm/models.py +++ b/services/core-app/src/epicurus_core_app/llm/models.py @@ -9,19 +9,22 @@ from __future__ import annotations from enum import StrEnum -from typing import Literal +from typing import Literal, NamedTuple from pydantic import BaseModel from epicurus_core import ChatMessage, ChatResult, Role +from epicurus_core_app.llm.errors import LocalRuntimeState __all__ = [ "ChatMessage", "ChatResult", "KeyState", + "LocalRuntimeStatus", "ModelDetails", "ModelInfo", "ModelRole", + "ModelWarmth", "PowerState", "ProviderInfo", "Role", @@ -30,6 +33,37 @@ "UsageEvent", ] + +class LocalRuntimeStatus(BaseModel): + """Whether this deployment has a local LLM runtime, and whether it answers (#962). + + The body of ``GET /platform/v1/llm/local-runtime``. A **separate** endpoint rather than an + envelope around ``GET /llm/models``, which stays a bare ``list[ModelInfo]``: five web + consumers and seven internal callers read that list, and wrapping it would be a breaking + change bought for nothing (ADR-0144). + + ``url_configured`` is the *why* behind the state: false means ``OLLAMA_URL`` is blank — a + deliberate hosted-only deployment — and no surface should draw a local-runtime control at + all. It is false exactly when ``state`` is ``absent``. + """ + + state: LocalRuntimeState + url_configured: bool + + +class ModelWarmth(NamedTuple): + """What :meth:`LlmGateway.model_readiness` answers (ADR-0027, extended by #962). + + ``warm`` is ``None`` whenever local warm-up is not a question that applies — a hosted + model, or a deployment with no local runtime at all — and ``runtime`` says which of the + two, so the readiness probe reports ``n/a`` for the second instead of "warming" forever. + """ + + model: str + warm: bool | None + runtime: Literal["local", "hosted", "absent"] = "local" + + ModelRole = Literal["chat", "embedding", "unknown"] """What a model is *for*, as the gateway resolved it (#944, ADR-0140). diff --git a/services/core-app/src/epicurus_core_app/llm/ollama_runtime.py b/services/core-app/src/epicurus_core_app/llm/ollama_runtime.py index 15711d35..a6fc36c8 100644 --- a/services/core-app/src/epicurus_core_app/llm/ollama_runtime.py +++ b/services/core-app/src/epicurus_core_app/llm/ollama_runtime.py @@ -55,11 +55,17 @@ class OllamaRuntime: """Writes Ollama's start-up env file and restarts the container to apply it.""" def __init__( - self, docker: ContainerController | None, *, env_path: str, service: str = "ollama" + self, + docker: ContainerController | None, + *, + env_path: str, + service: str = "ollama", + local_runtime_enabled: bool = True, ) -> None: self._docker = docker self._env_path = Path(env_path) self._service = service + self._local_runtime_enabled = local_runtime_enabled def apply_kv_cache_type(self, kv_cache_type: str | None) -> KvCacheApplyResult: """Apply ``kv_cache_type`` to the live Ollama runtime; report how far it got (#709). @@ -67,7 +73,19 @@ def apply_kv_cache_type(self, kv_cache_type: str | None) -> KvCacheApplyResult: Writes (or clears) the shared env file, then restarts Ollama so it re-reads it. Degrades instead of failing the request, in two distinct ways the caller must be able to tell apart — see :class:`KvCacheApplyResult`. Never raises. + + With **no local runtime** (#962, ADR-0144) it does neither and reports neither: the + same "unavailable"-shaped result the two degraded modes use, ``applied=False, + staged=False``. The route refuses such a call with 409 before it ever arrives here, + so this is belt-and-braces for an internal caller — but it matters that the answer is + the existing shape rather than a new one, on *both* arms of the container seam + (ADR-0134): with nothing to restart, neither Docker nor Kubernetes should be asked to + look, and neither should report a failed restart of a workload that was never meant + to exist. """ + if not self._local_runtime_enabled: + log.info("no local LLM runtime; KV-cache choice recorded but nothing to apply") + return KvCacheApplyResult(applied=False, staged=False) try: self._write_env_file(kv_cache_type) except OSError as exc: # volume not mounted / not writable — degrade, don't fail diff --git a/services/core-app/src/epicurus_core_app/llm/routes.py b/services/core-app/src/epicurus_core_app/llm/routes.py index e82cef7d..14499256 100644 --- a/services/core-app/src/epicurus_core_app/llm/routes.py +++ b/services/core-app/src/epicurus_core_app/llm/routes.py @@ -4,16 +4,25 @@ import asyncio import json -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Awaitable +from typing import TypeVar +import httpx from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from epicurus_core_app.llm.catalog import CatalogResponse, ModelCatalog +from epicurus_core_app.llm.errors import LocalRuntimeUnavailableError from epicurus_core_app.llm.gateway import LlmGateway, UnknownProviderError from epicurus_core_app.llm.model_settings import ModelSettings, ModelSettingsStore -from epicurus_core_app.llm.models import ModelDetails, ModelInfo, PowerState, ProviderInfo +from epicurus_core_app.llm.models import ( + LocalRuntimeStatus, + ModelDetails, + ModelInfo, + PowerState, + ProviderInfo, +) from epicurus_core_app.llm.ollama_runtime import KvCacheApplyResult, OllamaRuntime from epicurus_core_app.llm.power import PowerController from epicurus_core_app.llm.prefs import LlmPrefsStore @@ -34,6 +43,61 @@ "X-Accel-Buffering": "no", } +_T = TypeVar("_T") + +# What a local-runtime state means as an HTTP status (#962, ADR-0144). `absent` is **409**, +# not 503 or 404: the deployment is in a state that makes the request meaningless, nothing is +# broken, and retrying will never help — the operator either configures a runtime or stops +# asking. `unreachable` is **502**: one *is* configured and did not answer, which is an error, +# and the core is a gateway in front of it. Neither is ever a bare 500 again. Read with a +# default rather than indexed: `LocalRuntimeState` also has an `ok`, which cannot reach here +# (nothing raises the error for a runtime that is serving) and must not become a KeyError — +# a 500 — if a later change ever makes it. +_LOCAL_RUNTIME_STATUS = {"absent": 409, "unreachable": 502} + + +async def _through_local_runtime(action: str, call: Awaitable[_T]) -> _T: + """Await a local-runtime call, mapping both non-serving states onto their status. + + One wrapper so the four local-only write paths (pull, delete, unload, the KV-cache apply) + cannot each invent their own answer — which is exactly how three of them came to 500 and + one to succeed silently. ``httpx.RequestError`` is *unreachable* (we never got an answer); + an ``HTTPStatusError`` means the runtime answered and refused, which is still a 502 from + the core's position in front of it, but its detail names the status so a typo'd model name + reads as one rather than as an outage. + """ + try: + return await call + except LocalRuntimeUnavailableError as exc: + raise HTTPException( + status_code=_LOCAL_RUNTIME_STATUS.get(exc.state, 502), detail=exc.message + ) from exc + except httpx.HTTPStatusError as exc: + raise HTTPException( + status_code=502, + detail=(f"the local LLM runtime refused to {action} (HTTP {exc.response.status_code})"), + ) from exc + except httpx.HTTPError as exc: + raise HTTPException( + status_code=502, + detail=f"the local LLM runtime is unreachable, so it cannot {action}: {exc}", + ) from exc + + +def _refuse_without_local_runtime(gateway: LlmGateway, action: str) -> None: + """409 before doing anything when this deployment runs no local runtime (#962). + + For the two callers that cannot let the gateway raise from inside the call: the SSE pull + (the response has already started by the time an async generator's body runs) and the + KV-cache apply (the preference must not be persisted for a runtime that does not exist). + """ + try: + gateway.require_local_runtime(action) + except LocalRuntimeUnavailableError as exc: + raise HTTPException( + status_code=_LOCAL_RUNTIME_STATUS.get(exc.state, 502), detail=exc.message + ) from exc + class PullRequest(BaseModel): model: str @@ -199,9 +263,30 @@ def create_llm_router( async def list_models(capabilities: bool = False) -> list[ModelInfo]: """List local models. ``?capabilities=true`` additionally fills each model's reported capabilities (tools/vision/…) from ``/api/show`` — opt-in, one call per model, so the - Models page can badge them while the chat picker stays light.""" + Models page can badge them while the chat picker stays light. + + **Always 200** (#962, ADR-0144). An empty list is the answer when the runtime is + absent *or* unreachable; this used to 500, and the Models page polls it every ten + seconds. The shape is deliberately still a bare ``list[ModelInfo]`` — the state lives + at ``GET /llm/local-runtime`` rather than in an envelope here.""" return await gateway.models(with_capabilities=capabilities) + @router.get("/local-runtime", response_model=LocalRuntimeStatus) + async def local_runtime() -> LocalRuntimeStatus: + """Whether this deployment has a local LLM runtime, and whether it answers (#962). + + ``absent`` — none is configured (``OLLAMA_URL`` blank): a deliberate hosted-only + deployment, and every local-runtime control should be collapsed rather than shown + broken. ``unreachable`` — one is configured and did not answer: an error, and it + should still look like one. ``ok`` — it answered. + + Separate from ``GET /llm/models`` on purpose (ADR-0144): that list has twelve + consumers and stays a bare array, so nothing has to be rewritten to learn the state. + Costs one ``/api/tags`` round trip when a runtime is configured, none when it is not. + """ + state = await gateway.local_runtime_state() + return LocalRuntimeStatus(state=state, url_configured=gateway.local_runtime_enabled) + @router.get("/catalog", response_model=CatalogResponse) async def get_catalog() -> CatalogResponse: """The browsable model catalog the core parses from upstream (#269). @@ -234,8 +319,11 @@ async def get_variants(model: str) -> ModelVariantsResponse: async def delete_model(name: str) -> dict[str, str]: """Remove a local model. ``name`` is a query param — model names contain ``:`` and ``/`` (e.g. ``hf.co/org/model:tag``), which proxies may mangle - in a path.""" - await gateway.delete_model(name) + in a path. + + **409** when this deployment runs no local runtime, **502** when one is configured + and does not answer (#962) — never a bare 500.""" + await _through_local_runtime("delete a model", gateway.delete_model(name)) return {"status": "ok", "model": name} @router.get("/models/details", response_model=ModelDetails) @@ -249,7 +337,13 @@ async def model_details(model: str) -> ModelDetails: async def unload_models(request: UnloadRequest) -> dict[str, str]: """Drop model(s) from memory now (``keep_alive=0``) **without** changing power state (#331) — the standalone unload the Models page calls. ``model`` omitted unloads every - loaded model; the ``loaded`` badge refreshes on success / the next poll.""" + loaded model; the ``loaded`` badge refreshes on success / the next poll. + + **409** with no local runtime (#962). The gateway's own ``unload`` stays silent there + — it is on the power-pause path, which must keep working on a hosted-only deployment — + so the refusal is made here, where the caller is an operator clicking Unload and + deserves to be told why nothing happened.""" + _refuse_without_local_runtime(gateway, "unload a model") await gateway.unload(request.model) return {"status": "ok", "model": request.model or "all"} @@ -280,12 +374,20 @@ async def clear_provider_key(alias: str) -> dict[str, str]: @router.post("/pull") async def pull(request: PullRequest) -> dict[str, str]: - await gateway.pull(request.model) + """Pull a model into the local runtime. + + **409** with no local runtime, **502** when one is configured and unreachable (#962).""" + await _through_local_runtime("pull a model", gateway.pull(request.model)) return {"status": "ok", "model": request.model} @router.post("/pull/stream") async def pull_stream(request: PullRequest) -> StreamingResponse: - """Pull a model, streaming the runtime's progress as SSE.""" + """Pull a model, streaming the runtime's progress as SSE. + + The no-runtime refusal is made **before** the response starts (#962), so the caller + gets a real 409 instead of a 200 whose single event is an error — the one thing an + SSE endpoint cannot say once it has begun.""" + _refuse_without_local_runtime(gateway, "pull a model") async def events() -> AsyncIterator[str]: try: @@ -398,9 +500,18 @@ async def set_kv_cache_type(request: SetKvCacheTypeRequest) -> dict[str, str | b that's left — the usual case without Docker access, since the entrypoint re-sources the file on every start. Only ``staged: false`` calls for editing environment variables by hand, which is what the UI used to say in every degraded case. + + **409** when this deployment runs no local runtime (#962): the setting describes how + Ollama *starts*, so there is nothing to stage and nothing to restart, and the refusal + comes before the write so a hosted-only deployment cannot accumulate a preference for + a server it will never run. A runtime that is merely **unreachable** is deliberately + not refused: this path never talks to Ollama — it writes an env file and asks the + container runtime to bounce the workload — so setting the value while the server is + down is legitimate, and ``applied``/``staged`` already says how far it got. """ if prefs is None: raise HTTPException(status_code=503, detail="preferences store not available") + _refuse_without_local_runtime(gateway, "apply a KV-cache setting") await prefs.set_kv_cache_type(default_tenant, request.value) result = ( ollama_runtime.apply_kv_cache_type(request.value) diff --git a/services/core-app/src/epicurus_core_app/readiness.py b/services/core-app/src/epicurus_core_app/readiness.py index 010dc97f..9c5d3c4f 100644 --- a/services/core-app/src/epicurus_core_app/readiness.py +++ b/services/core-app/src/epicurus_core_app/readiness.py @@ -101,16 +101,30 @@ async def _modules(self) -> ReadinessComponent: ) async def _model(self, model: str | None, tenant: str) -> ReadinessComponent: - """Whether the turn's model is warm; hosted models report ready (no local warm-up).""" + """Whether the turn's model is warm; hosted models report ready (no local warm-up). + + Three ways to be ready without warming up, and they must not be confused (#962): + a hosted model (``· hosted``), a deployment with no local runtime at all + (``· n/a``), and a local model already loaded (``· warm``). Before ADR-0144 the + second reported ``warming`` forever — the probe asked a runtime that was not there, + caught the failure, and read it as "cold" — so the chat progress bar never completed + on a single turn of a hosted-only install. None of these drags ``ready`` to false: + readiness is advisory and never blocks a turn, and a component that can never become + ready has no business claiming the system is not. + """ try: - name, warm = await self._gateway.model_readiness(model, tenant_id=tenant) + warmth = await self._gateway.model_readiness(model, tenant_id=tenant) except Exception as exc: # gateway trouble must not block the chat log.warning("model readiness probe failed", error=str(exc)) return ReadinessComponent(name="model", ready=True, detail="unknown") - if warm is None: - return ReadinessComponent(name="model", ready=True, detail=f"{name} · hosted") + if warmth.runtime == "absent": + return ReadinessComponent(name="model", ready=True, detail=f"{warmth.model} · n/a") + if warmth.warm is None: + return ReadinessComponent(name="model", ready=True, detail=f"{warmth.model} · hosted") return ReadinessComponent( - name="model", ready=warm, detail=f"{name} · {'warm' if warm else 'warming'}" + name="model", + ready=warmth.warm, + detail=f"{warmth.model} · {'warm' if warmth.warm else 'warming'}", ) diff --git a/services/core-app/src/epicurus_core_app/settings.py b/services/core-app/src/epicurus_core_app/settings.py index 1b79dec6..1a710e19 100644 --- a/services/core-app/src/epicurus_core_app/settings.py +++ b/services/core-app/src/epicurus_core_app/settings.py @@ -17,6 +17,9 @@ class CoreAppSettings(CoreSettings): """Adds the LLM-gateway configuration to the shared settings.""" # Ollama, the local LLM runtime. On the internal Docker network: http://ollama:11434. + # **Blank means this deployment runs no local runtime at all** (#962, ADR-0144) — a + # deliberate hosted-only install, not a misconfiguration; see `local_runtime_enabled`. + # The default stays local-first: an operator opts *out* by blanking the value. ollama_url: str = "http://localhost:11434" # KV-cache apply (#307): the core writes Ollama's start-up env file here (a named volume # both containers share; the Ollama entrypoint sources it), then restarts this service so it @@ -378,6 +381,19 @@ def _blank_timeout_to_default(cls, value: object) -> object: return cls.model_fields["llm_timeout"].default return value + @property + def local_runtime_enabled(self) -> bool: + """Whether this deployment has a local LLM runtime at all (#962, ADR-0144). + + The three states a call site used to collapse into one are *absent* (no runtime is + configured — this property), *unreachable* (one is configured and does not answer) and + *ok*. A blank ``OLLAMA_URL`` is the operator saying "hosted only": every local-runtime + surface then refuses with a reason (409 / a capability refusal) instead of timing out, + 500ing, or polling for three minutes. Whitespace is blank — ``OLLAMA_URL=" "`` from a + hand-edited env file means the same thing as empty. + """ + return bool(self.ollama_url.strip()) + @property def fallback_models(self) -> list[str]: """The fallback chain parsed from ``llm_fallbacks``.""" diff --git a/services/core-app/tests/test_llm_bootstrap.py b/services/core-app/tests/test_llm_bootstrap.py index 6f1ca1f7..ceb10f91 100644 --- a/services/core-app/tests/test_llm_bootstrap.py +++ b/services/core-app/tests/test_llm_bootstrap.py @@ -13,6 +13,7 @@ from contextlib import suppress from epicurus_core_app.llm.bootstrap import ModelBootstrap +from epicurus_core_app.llm.errors import LocalRuntimeState from epicurus_core_app.llm.models import ModelInfo @@ -27,6 +28,7 @@ def __init__( embed_default: str = "nomic-embed-text", pull_failures: dict[str, int] | None = None, reachable: bool = True, + local_runtime_enabled: bool = True, ) -> None: self.installed = list(installed or []) self.default = default @@ -34,8 +36,20 @@ def __init__( # Model → number of times pull raises before succeeding. self.pull_failures = dict(pull_failures or {}) self.reachable = reachable + self._local_runtime_enabled = local_runtime_enabled self.pull_calls: list[str] = [] self.models_calls = 0 + self.state_calls = 0 + + @property + def local_runtime_enabled(self) -> bool: + return self._local_runtime_enabled + + async def local_runtime_state(self) -> LocalRuntimeState: + self.state_calls += 1 + if not self._local_runtime_enabled: + return "absent" + return "ok" if self.reachable else "unreachable" async def models( self, tenant_id: str | None = None, *, with_capabilities: bool = False @@ -146,7 +160,11 @@ async def test_explicit_list_still_ensures_on_a_non_empty_runtime() -> None: async def test_unreachable_runtime_gives_up_quietly() -> None: gateway = FakeGateway(reachable=False) await make_bootstrap(gateway, models_spec="auto").run() - assert gateway.models_calls >= 1 + # It waits on the runtime's *state*, not on `models()` raising: since #962 an + # unreachable runtime reports an empty list rather than an error, and polling that + # would read a still-starting Ollama as "an empty runtime" and race it with a pull. + assert gateway.state_calls >= 1 + assert gateway.models_calls == 0 assert gateway.pull_calls == [] @@ -214,3 +232,23 @@ async def test_cancellation_propagates_for_shutdown() -> None: with suppress(asyncio.CancelledError): await asyncio.wait_for(task, timeout=1.0) assert task.cancelled() + + +async def test_no_local_runtime_returns_at_once_without_polling() -> None: + """A hosted-only deployment spent 180 s of every process start polling nothing (#962). + + Asserted on behaviour, not on a clock: the bootstrap must not probe the runtime's state, + must not list models, and must not pull — it knows from the configuration alone. + """ + gateway = FakeGateway(local_runtime_enabled=False) + await make_bootstrap(gateway, models_spec="auto", ready_timeout_s=30.0).run() + assert gateway.state_calls == 0 + assert gateway.models_calls == 0 + assert gateway.pull_calls == [] + + +async def test_no_local_runtime_wins_over_an_explicit_model_list() -> None: + """An explicit pin is a standing instruction — but not one this deployment can carry out.""" + gateway = FakeGateway(local_runtime_enabled=False) + await make_bootstrap(gateway, models_spec="llama3.2,nomic-embed-text").run() + assert gateway.pull_calls == [] diff --git a/services/core-app/tests/test_llm_gateway.py b/services/core-app/tests/test_llm_gateway.py index 227e43ab..40b37489 100644 --- a/services/core-app/tests/test_llm_gateway.py +++ b/services/core-app/tests/test_llm_gateway.py @@ -5,15 +5,18 @@ import asyncio import json from collections.abc import AsyncIterator -from typing import Any, cast +from typing import Any, ClassVar, cast +import httpx import pytest from sqlalchemy.ext.asyncio import create_async_engine from sqlalchemy.pool import StaticPool -from structlog.testing import capture_logs from epicurus_core import EventBus, SecretError, SecretNotFoundError, SecretStore -from epicurus_core_app.llm.errors import ModelCapabilityError +from epicurus_core_app.llm.errors import ( + LocalRuntimeUnavailableError, + ModelCapabilityError, +) from epicurus_core_app.llm.gateway import ( _CONNECT_TIMEOUT_S, _UNBOUNDED_READ_S, @@ -24,7 +27,7 @@ with_no_tools_note, ) from epicurus_core_app.llm.model_settings import ModelSettings, ModelSettingsStore -from epicurus_core_app.llm.models import ChatMessage, ModelInfo, PowerState +from epicurus_core_app.llm.models import ChatMessage, ModelInfo, ModelWarmth, PowerState from epicurus_core_app.llm.power import GatewayPausedError, PowerController from epicurus_core_app.llm.prefs import LlmPrefsStore from epicurus_core_app.llm.saved_models import SavedHostedModelStore, SavedModelOverride @@ -96,9 +99,10 @@ def _gateway( prefs: LlmPrefsStore | None = None, model_settings: ModelSettingsStore | None = None, saved_models: SavedHostedModelStore | None = None, + ollama_url: str = "http://ollama:11434", ) -> LlmGateway: return LlmGateway( - ollama_url="http://ollama:11434", + ollama_url=ollama_url, default_model="llama3.2", keep_alive="5m", power=power or PowerController(), @@ -203,11 +207,13 @@ async def fake_acompletion(**kwargs: Any) -> _Response: monkeypatch.setattr("epicurus_core_app.llm.gateway.litellm.acompletion", fake_acompletion) secrets = _FakeSecrets({"llm/anthropic": {"api_key": "fixture-redaction-sentinel"}}) - with capture_logs() as logs: - await _gateway(secrets=secrets).chat( - [ChatMessage(role="user", content="hi")], model="claude/c" - ) - assert not any("fixture-redaction-sentinel" in str(entry) for entry in logs) + # Recorded, not captured — see ``_RecordingLog``. A ``capture_logs`` block that silently + # intercepts nothing makes this particular assertion *vacuously* true, which is the worst + # possible failure mode for a test whose whole job is to prove a key never gets logged. + recorder = _RecordingLog() + monkeypatch.setattr("epicurus_core_app.llm.gateway.log", recorder) + await _gateway(secrets=secrets).chat([ChatMessage(role="user", content="hi")], model="claude/c") + assert not any("fixture-redaction-sentinel" in str(call) for call in recorder.calls) async def test_providers_reports_configured() -> None: @@ -1284,7 +1290,7 @@ async def fake_models(tenant_id: str | None = None) -> list[ModelInfo]: ] monkeypatch.setattr(gw, "models", fake_models) - assert await gw.model_readiness("llama3.2") == ("llama3.2", True) + assert await gw.model_readiness("llama3.2") == ModelWarmth("llama3.2", True, "local") async def test_model_readiness_local_cold(monkeypatch: pytest.MonkeyPatch) -> None: @@ -1294,21 +1300,22 @@ async def fake_models(tenant_id: str | None = None) -> list[ModelInfo]: return [ModelInfo(name="llama3.2:latest", loaded=False)] monkeypatch.setattr(gw, "models", fake_models) - assert await gw.model_readiness("llama3.2") == ("llama3.2", False) + assert await gw.model_readiness("llama3.2") == ModelWarmth("llama3.2", False, "local") async def test_model_readiness_hosted_is_always_ready() -> None: # Hosted providers need no local warm-up — warm is None (always ready), no runtime probe. - name, warm = await _gateway().model_readiness("claude/claude-sonnet-4-6") - assert name == "claude/claude-sonnet-4-6" and warm is None + warmth = await _gateway().model_readiness("claude/claude-sonnet-4-6") + assert warmth.model == "claude/claude-sonnet-4-6" + assert warmth.warm is None and warmth.runtime == "hosted" async def test_model_readiness_paused_local_is_cold_without_probing() -> None: power = PowerController() power.pause() # While paused the runtime is never probed (that would wake the GPU): cold by definition. - name, warm = await _gateway(power=power).model_readiness("llama3.2") - assert name == "llama3.2" and warm is False + warmth = await _gateway(power=power).model_readiness("llama3.2") + assert warmth.model == "llama3.2" and warmth.warm is False async def test_model_readiness_runtime_error_reports_cold(monkeypatch: pytest.MonkeyPatch) -> None: @@ -1318,7 +1325,7 @@ async def boom(tenant_id: str | None = None) -> list[ModelInfo]: raise RuntimeError("ollama unreachable") monkeypatch.setattr(gw, "models", boom) - assert await gw.model_readiness("llama3.2") == ("llama3.2", False) + assert await gw.model_readiness("llama3.2") == ModelWarmth("llama3.2", False, "local") async def test_model_readiness_defaults_to_effective_default( @@ -1330,7 +1337,7 @@ async def fake_models(tenant_id: str | None = None) -> list[ModelInfo]: return [ModelInfo(name="llama3.2:latest", loaded=True)] monkeypatch.setattr(gw, "models", fake_models) - assert await gw.model_readiness() == ("llama3.2", True) + assert await gw.model_readiness() == ModelWarmth("llama3.2", True, "local") # ── context window (num_ctx) pref resolution ────────────────────────────────────── @@ -2079,9 +2086,21 @@ def warning(self, event: str, **fields: Any) -> None: def debug(self, event: str, **fields: Any) -> None: self.calls.append(("debug", event, fields)) + def info(self, event: str, **fields: Any) -> None: + self.calls.append(("info", event, fields)) + + def error(self, event: str, **fields: Any) -> None: + self.calls.append(("error", event, fields)) + def levels(self) -> list[str]: return [level for level, _, _ in self.calls] + def find(self, prefix: str) -> tuple[str, str, dict[str, Any]]: + """The first call whose event starts with ``prefix`` — asserts that there is one.""" + found = next((call for call in self.calls if call[1].startswith(prefix)), None) + assert found is not None, f"no log line starting {prefix!r}; recorded {self.calls}" + return found + async def test_an_unmapped_model_warns_once_per_process(monkeypatch: pytest.MonkeyPatch) -> None: """An operator-saved alias outside the map is expected — worth one warning, not a stream.""" @@ -2433,19 +2452,23 @@ async def fake_acompletion(**kwargs: Any) -> _Response: _hosted_map(monkeypatch, {}) store = await _saved_store({"openrouter/some/model": SavedModelOverride()}) gw = _gateway(secrets=_FakeSecrets({"llm/openrouter": {"api_key": "k"}}), saved_models=store) - with capture_logs() as logs: - await gw.chat( - [ChatMessage(role="user", content="hi")], - model="openrouter/some/model", - tools=[{"type": "function", "function": {"name": "now"}}], - ) - learned = next(e for e in logs if e["event"].startswith("model rejected the tool list")) - assert learned["log_level"] == "warning" - assert learned["provider"] == "openrouter" - assert learned["upstream"] == "NextBit" # the aggregator's readable provider name - assert learned["matched"] == "tool choice requires" + # Recorded directly rather than through ``capture_logs`` — see ``_RecordingLog``: once any + # test in the run has booted the app, the gateway's module logger is frozen and a capture + # here intercepts nothing, so this assertion passed alone and failed in a full run. + recorder = _RecordingLog() + monkeypatch.setattr("epicurus_core_app.llm.gateway.log", recorder) + await gw.chat( + [ChatMessage(role="user", content="hi")], + model="openrouter/some/model", + tools=[{"type": "function", "function": {"name": "now"}}], + ) + level, _, fields = recorder.find("model rejected the tool list") + assert level == "warning" + assert fields["provider"] == "openrouter" + assert fields["upstream"] == "NextBit" # the aggregator's readable provider name + assert fields["matched"] == "tool choice requires" # The raw body — and the account identifier riding in it — never reaches the log line. - assert "user_2abcDEF" not in json.dumps(learned, default=str) + assert "user_2abcDEF" not in json.dumps(fields, default=str) async def test_a_learned_no_disables_tools_on_the_next_turn( @@ -2494,3 +2517,237 @@ def test_the_no_tools_note_lands_inside_the_protected_system_prefix() -> None: assert [m.role for m in noted] == ["system", "system", "system", "user"] assert noted[2].content == NO_TOOLS_SYSTEM_NOTE assert convo[0].content == "base prompt" # the input list is not mutated + + +# ── no local runtime at all (#962, ADR-0144) ───────────────────────────────────── +# +# Three states, not two: *absent* (OLLAMA_URL blank — a deliberate hosted-only deployment), +# *unreachable* (one is configured and does not answer) and *ok*. Every test below pins a +# place that used to collapse them — into a 500, into a forever-"warming" readiness, into a +# connection error where a sentence belonged. + + +class _StubOllama: + """An httpx.AsyncClient stand-in that records every request path it is asked for. + + Constructed with ``boom=True`` it raises the transport error a refused connection gives, + which is how "unreachable" is expressed; the recorded paths are how "absent" is proven — + a deployment with no runtime must make **no call at all**, not a call that fails quietly. + """ + + paths: ClassVar[list[str]] = [] + + def __init__(self, *args: Any, boom: bool = False, **kwargs: Any) -> None: + self._boom = boom + + async def __aenter__(self) -> _StubOllama: + return self + + async def __aexit__(self, *args: Any) -> None: + return None + + def _answer(self, path: str) -> Any: + type(self).paths.append(path) + if self._boom: + raise httpx.ConnectError("connection refused") + + class _Resp: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return {"models": [{"name": "llama3.2:latest", "size": 1}]} + + return _Resp() + + async def get(self, path: str) -> Any: + return self._answer(path) + + async def post(self, path: str, **kwargs: Any) -> Any: + return self._answer(path) + + async def request(self, method: str, path: str, **kwargs: Any) -> Any: + return self._answer(path) + + +def _stub_runtime(monkeypatch: pytest.MonkeyPatch, *, boom: bool = False) -> list[str]: + """Point the gateway's httpx client at :class:`_StubOllama`; return the recorded paths.""" + _StubOllama.paths = [] + + def factory(*args: Any, **kwargs: Any) -> _StubOllama: + return _StubOllama(boom=boom) + + monkeypatch.setattr("epicurus_core_app.llm.gateway.httpx.AsyncClient", factory) + return _StubOllama.paths + + +def test_a_blank_url_is_the_absent_state() -> None: + assert _gateway(ollama_url="").local_runtime_enabled is False + assert _gateway(ollama_url=" ").local_runtime_enabled is False + assert _gateway().local_runtime_enabled is True + + +async def test_local_runtime_state_reports_absent_without_touching_the_network( + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths = _stub_runtime(monkeypatch) + assert await _gateway(ollama_url="").local_runtime_state() == "absent" + assert paths == [] # no URL, no call — absence is a configuration fact, not a probe result + + +async def test_local_runtime_state_reports_unreachable(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_runtime(monkeypatch, boom=True) + assert await _gateway().local_runtime_state() == "unreachable" + + +async def test_local_runtime_state_reports_ok(monkeypatch: pytest.MonkeyPatch) -> None: + paths = _stub_runtime(monkeypatch) + assert await _gateway().local_runtime_state() == "ok" + assert paths == ["/api/tags"] + + +async def test_models_is_empty_and_quiet_when_the_runtime_is_absent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Half of the regression test for the 500 the Models page collected every 10 seconds.""" + paths = _stub_runtime(monkeypatch) + assert await _gateway(ollama_url="").models() == [] + assert paths == [] + + +async def test_models_is_empty_when_the_runtime_is_unreachable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The other half: a configured runtime that refuses the connection must not raise.""" + _stub_runtime(monkeypatch, boom=True) + assert await _gateway().models() == [] + + +async def test_show_reports_nothing_without_a_runtime(monkeypatch: pytest.MonkeyPatch) -> None: + paths = _stub_runtime(monkeypatch) + details = await _gateway(ollama_url="").show("llama3.2") + assert details.role == "unknown" and details.capabilities == [] + assert paths == [] + + +async def test_pull_and_delete_refuse_when_there_is_no_runtime() -> None: + gw = _gateway(ollama_url="") + with pytest.raises(LocalRuntimeUnavailableError) as pulled: + await gw.pull("llama3.2") + with pytest.raises(LocalRuntimeUnavailableError) as deleted: + await gw.delete_model("llama3.2") + for excinfo in (pulled, deleted): + assert excinfo.value.state == "absent" + assert "no local LLM runtime" in str(excinfo.value) + + +async def test_pull_stream_refuses_on_the_first_step() -> None: + """The refusal must be reachable before any progress event — the route turns it into 409.""" + stream = _gateway(ollama_url="").pull_stream("llama3.2") + with pytest.raises(LocalRuntimeUnavailableError): + await anext(stream) + + +async def test_unload_is_a_quiet_no_op_without_a_runtime( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`unload` is on the power-pause path, which must keep working on a hosted-only box.""" + paths = _stub_runtime(monkeypatch) + await _gateway(ollama_url="").unload() + await _gateway(ollama_url="").unload("llama3.2") + assert paths == [] + + +async def test_model_readiness_reports_n_a_when_there_is_no_runtime() -> None: + warmth = await _gateway(ollama_url="").model_readiness("llama3.2") + assert warmth == ModelWarmth("llama3.2", None, "absent") + + +async def test_a_hosted_model_still_embeds_while_the_runtime_is_absent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The owner's directive, pinned: with Ollama off, embeddings go straight to the provider.""" + captured: dict[str, Any] = {} + + class _EmbedResp: + def model_dump(self) -> dict[str, Any]: + return {"data": [{"embedding": [0.1, 0.2]}]} + + async def fake_aembedding(**kwargs: Any) -> _EmbedResp: + captured.update(kwargs) + return _EmbedResp() + + monkeypatch.setattr("epicurus_core_app.llm.gateway.litellm.aembedding", fake_aembedding) + _hosted_map(monkeypatch, {"openai/text-embedding-3-small": {"mode": "embedding"}}) + gw = _gateway(ollama_url="", secrets=_FakeSecrets({"llm/openai": {"api_key": "k"}})) + vectors = await gw.embed(["hello"], model="gpt/text-embedding-3-small") + + assert vectors == [[0.1, 0.2]] + assert captured["model"] == "openai/text-embedding-3-small" + assert captured["api_key"] == "k" + assert "api_base" not in captured # no Ollama endpoint anywhere near a hosted embedding + + +async def test_a_local_embedding_model_refuses_with_a_reason_when_absent() -> None: + """Not a connection error to a blank URL — a capability refusal naming the fix.""" + with pytest.raises(ModelCapabilityError) as excinfo: + await _gateway(ollama_url="").embed(["hi"], model="nomic-embed-text") + assert excinfo.value.capability == "embedding" + assert "no local runtime" in excinfo.value.hint.lower() + assert "hosted" in excinfo.value.hint.lower() + + +async def test_the_default_embedding_model_refuses_when_it_is_local_and_absent() -> None: + """The out-of-the-box hosted-only install: a bare env default with nothing to run it.""" + with pytest.raises(ModelCapabilityError): + await _gateway(ollama_url="").embed(["hi"]) + + +async def test_a_hosted_model_still_chats_while_the_runtime_is_absent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, Any] = {} + + async def fake_acompletion(**kwargs: Any) -> _Response: + captured.update(kwargs) + return _Response({"choices": [{"message": {"content": "ok"}}]}) + + monkeypatch.setattr("epicurus_core_app.llm.gateway.litellm.acompletion", fake_acompletion) + _hosted_map(monkeypatch, {"anthropic/c": {"mode": "chat"}}) + gw = _gateway(ollama_url="", secrets=_FakeSecrets({"llm/anthropic": {"api_key": "k"}})) + result = await gw.chat([ChatMessage(role="user", content="hi")], model="claude/c") + + assert result.content == "ok" + assert captured["model"] == "anthropic/c" + + +async def test_a_local_chat_model_refuses_with_the_hint_when_absent() -> None: + with pytest.raises(ModelCapabilityError) as excinfo: + await _gateway(ollama_url="").chat([ChatMessage(role="user", content="hi")]) + assert excinfo.value.capability == "chat" + assert "no local runtime" in excinfo.value.hint.lower() + + +async def test_a_local_fallback_is_skipped_when_there_is_no_runtime( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A hosted primary must never fall back into a runtime that is not there.""" + calls: list[str] = [] + + async def fake_acompletion(**kwargs: Any) -> _Response: + calls.append(str(kwargs["model"])) + if kwargs["model"] == "anthropic/c": + raise RuntimeError("provider down") + return _Response({"choices": [{"message": {"content": "ok"}}]}) + + monkeypatch.setattr("epicurus_core_app.llm.gateway.litellm.acompletion", fake_acompletion) + _hosted_map(monkeypatch, {"anthropic/c": {"mode": "chat"}, "openai/g": {"mode": "chat"}}) + gw = _gateway( + ollama_url="", + fallbacks=["llama3.2", "gpt/g"], + secrets=_FakeSecrets({"llm/anthropic": {"api_key": "k"}, "llm/openai": {"api_key": "k"}}), + ) + result = await gw.chat([ChatMessage(role="user", content="hi")], model="claude/c") + + assert result.content == "ok" + assert calls == ["anthropic/c", "openai/g"] # the local fallback was never tried diff --git a/services/core-app/tests/test_llm_routes.py b/services/core-app/tests/test_llm_routes.py index 6675174c..6a18d2b6 100644 --- a/services/core-app/tests/test_llm_routes.py +++ b/services/core-app/tests/test_llm_routes.py @@ -9,8 +9,9 @@ from sqlalchemy.pool import StaticPool from epicurus_core_app.llm.catalog import CatalogEntry, ModelCatalog +from epicurus_core_app.llm.errors import LocalRuntimeState, LocalRuntimeUnavailableError from epicurus_core_app.llm.model_settings import ModelSettingsStore -from epicurus_core_app.llm.models import ModelDetails +from epicurus_core_app.llm.models import ModelDetails, ModelInfo from epicurus_core_app.llm.ollama_runtime import KvCacheApplyResult, OllamaRuntime from epicurus_core_app.llm.prefs import LlmPrefsStore from epicurus_core_app.llm.routes import create_llm_router @@ -24,12 +25,47 @@ class _StubGateway: ``show`` backs the /models/details route; ``model_role`` backs the role gate on the two default-setting routes (#944), answering ``unknown`` for anything not in ``roles`` — the "catalogue says nothing" case, which the gate lets through; ``unload`` records its calls so - the unload route can be asserted. + the unload route can be asserted. ``local_runtime_enabled`` / ``require_local_runtime`` / + ``local_runtime_state`` back the three-state local-runtime contract (#962): build it with + ``local_runtime=False`` for a hosted-only deployment, where every local-only write route + must refuse with 409 instead of reaching a runtime that is not there. """ - def __init__(self, roles: dict[str, str] | None = None) -> None: + def __init__(self, roles: dict[str, str] | None = None, *, local_runtime: bool = True) -> None: self.unloaded: list[str | None] = [] self.roles = roles or {} + self._local_runtime = local_runtime + self.pulled: list[str] = [] + self.deleted: list[str] = [] + + @property + def local_runtime_enabled(self) -> bool: + return self._local_runtime + + def require_local_runtime(self, action: str) -> None: + if self._local_runtime: + return + raise LocalRuntimeUnavailableError( + state="absent", + message=f"cannot {action}: this deployment runs no local LLM runtime", + ) + + async def local_runtime_state(self) -> LocalRuntimeState: + return "ok" if self._local_runtime else "absent" + + async def models( + self, tenant_id: str | None = None, *, with_capabilities: bool = False + ) -> list[ModelInfo]: + # What the real gateway answers in both non-serving states (#962): an empty list. + return [] if not self._local_runtime else [ModelInfo(name="llama3.2:latest")] + + async def pull(self, model: str) -> None: + self.require_local_runtime("pull a model") + self.pulled.append(model) + + async def delete_model(self, model: str) -> None: + self.require_local_runtime("delete a model") + self.deleted.append(model) async def show(self, model: str, tenant_id: str | None = None) -> ModelDetails: return ModelDetails( @@ -1002,3 +1038,110 @@ async def test_capability_override_rejects_a_bad_role_value() -> None: json={"model": "grok/grok-latest", "role": "reranker"}, ) assert put.status_code == 422 + + +# ── no local runtime at all (#962, ADR-0144) ───────────────────────────────────── +# +# `absent` answers 409 — the request is meaningless on this deployment and retrying will +# never help — and every one of these paths used to answer 500 or (worse) 200. + + +def _hosted_only_app() -> FastAPI: + """An app whose gateway reports a hosted-only deployment (``OLLAMA_URL`` blank).""" + return _app(gateway=_StubGateway(local_runtime=False), prefs=None) + + +async def _client(app: FastAPI) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") + + +def test_the_local_runtime_endpoint_is_declared() -> None: + assert "/platform/v1/llm/local-runtime" in _app().openapi()["paths"] + + +async def test_local_runtime_reports_ok_when_one_answers() -> None: + async with await _client(_app()) as client: + body = (await client.get("/platform/v1/llm/local-runtime")).json() + assert body == {"state": "ok", "url_configured": True} + + +async def test_local_runtime_reports_absent_on_a_hosted_only_deployment() -> None: + async with await _client(_hosted_only_app()) as client: + response = await client.get("/platform/v1/llm/local-runtime") + assert response.status_code == 200 + assert response.json() == {"state": "absent", "url_configured": False} + + +async def test_models_is_a_bare_empty_list_and_200_without_a_runtime() -> None: + """The regression test for the 500 the Models page collected every ten seconds. + + The *shape* matters as much as the status: five web consumers and seven internal callers + read this array, so the state went to its own endpoint rather than into an envelope here. + """ + async with await _client(_hosted_only_app()) as client: + response = await client.get("/platform/v1/llm/models") + assert response.status_code == 200 + assert response.json() == [] + + +async def test_pull_refuses_with_409_without_a_runtime() -> None: + gateway = _StubGateway(local_runtime=False) + async with await _client(_app(gateway=gateway)) as client: + response = await client.post("/platform/v1/llm/pull", json={"model": "llama3.2"}) + assert response.status_code == 409 + assert "no local LLM runtime" in response.json()["detail"] + assert gateway.pulled == [] + + +async def test_pull_stream_refuses_before_the_stream_starts() -> None: + """A 409, not a 200 whose only SSE event is an error — an SSE cannot take back its status.""" + async with await _client(_hosted_only_app()) as client: + response = await client.post("/platform/v1/llm/pull/stream", json={"model": "llama3.2"}) + assert response.status_code == 409 + assert "text/event-stream" not in response.headers.get("content-type", "") + + +async def test_delete_refuses_with_409_without_a_runtime() -> None: + gateway = _StubGateway(local_runtime=False) + async with await _client(_app(gateway=gateway)) as client: + response = await client.delete("/platform/v1/llm/models?name=llama3.2") + assert response.status_code == 409 + assert gateway.deleted == [] + + +async def test_unload_refuses_with_409_without_a_runtime() -> None: + """The gateway's own unload stays silent (the power-pause path needs it); the route says so.""" + gateway = _StubGateway(local_runtime=False) + async with await _client(_app(gateway=gateway)) as client: + response = await client.post("/platform/v1/llm/unload", json={"model": None}) + assert response.status_code == 409 + assert gateway.unloaded == [] + + +async def test_a_runtime_that_answers_with_an_error_is_a_502_not_a_500() -> None: + """`unreachable` is an error — and the core is a gateway in front of it.""" + + class _Unreachable(_StubGateway): + async def pull(self, model: str) -> None: + raise httpx.ConnectError("connection refused") + + async with await _client(_app(gateway=_Unreachable())) as client: + response = await client.post("/platform/v1/llm/pull", json={"model": "llama3.2"}) + assert response.status_code == 502 + assert "unreachable" in response.json()["detail"] + + +async def test_kv_cache_type_refuses_with_409_and_persists_nothing_without_a_runtime() -> None: + """The setting describes how Ollama *starts*; with no Ollama there is nothing to record.""" + prefs = await _fresh_prefs() + runtime = _FakeRuntime() + app = _app( + prefs=prefs, + ollama_runtime=runtime, # type: ignore[arg-type] + gateway=_StubGateway(local_runtime=False), + ) + async with await _client(app) as client: + response = await client.put("/platform/v1/llm/prefs/kv-cache-type", json={"value": "q4_0"}) + assert response.status_code == 409 + assert runtime.applied == [] + assert await prefs.get_kv_cache_type("local") is None diff --git a/services/core-app/tests/test_ollama_runtime.py b/services/core-app/tests/test_ollama_runtime.py index 606b4c76..574d68a1 100644 --- a/services/core-app/tests/test_ollama_runtime.py +++ b/services/core-app/tests/test_ollama_runtime.py @@ -105,3 +105,24 @@ def test_applied_always_implies_staged(tmp_path: Path) -> None: rt, _ = _runtime(tmp_path, docker) result = rt.apply_kv_cache_type("q8_0") assert not result.applied or result.staged + + +def test_no_local_runtime_writes_nothing_and_restarts_nothing(tmp_path: Path) -> None: + """With no Ollama at all (#962, ADR-0144) the apply is the existing "unavailable" shape. + + Not a *new* result shape, and not a failed restart: with nothing to restart, neither arm + of the container seam (ADR-0134) should be asked to look for a workload, on Docker or on + Kubernetes. The route refuses such a call with 409 before it reaches here; this is the + belt-and-braces behind it. + """ + docker = _FakeDocker() + env = tmp_path / "ollama.env" + rt = OllamaRuntime( + docker, # type: ignore[arg-type] + env_path=str(env), + service="ollama", + local_runtime_enabled=False, + ) + assert rt.apply_kv_cache_type("q8_0") == KvCacheApplyResult(applied=False, staged=False) + assert docker.restarted == [] + assert not env.exists() diff --git a/services/core-app/tests/test_readiness.py b/services/core-app/tests/test_readiness.py index a3604c9b..eb883c92 100644 --- a/services/core-app/tests/test_readiness.py +++ b/services/core-app/tests/test_readiness.py @@ -3,7 +3,7 @@ from __future__ import annotations from epicurus_core import ModuleManifest -from epicurus_core_app.llm.models import PowerState +from epicurus_core_app.llm.models import ModelWarmth, PowerState from epicurus_core_app.llm.power import PowerController from epicurus_core_app.modules import ModuleSnapshot, ModuleStatus from epicurus_core_app.readiness import Readiness, ReadinessComponent, ReadinessProbe @@ -12,12 +12,12 @@ class _FakeGateway: """Replays a scripted ``model_readiness`` result (or raises).""" - def __init__(self, result: tuple[str, bool | None] | Exception) -> None: + def __init__(self, result: ModelWarmth | Exception) -> None: self._result = result async def model_readiness( self, model: str | None = None, *, tenant_id: str | None = None - ) -> tuple[str, bool | None]: + ) -> ModelWarmth: if isinstance(self._result, Exception): raise self._result return self._result @@ -64,7 +64,7 @@ def _component(readiness: Readiness, name: str) -> ReadinessComponent: async def test_all_warm_is_ready() -> None: probe = _probe( - gateway=_FakeGateway(("llama3.2", True)), + gateway=_FakeGateway(ModelWarmth("llama3.2", True)), registry=_FakeRegistry([_snap("calendar", True), _snap("notes", True)]), ) readiness = await probe.check() @@ -76,7 +76,7 @@ async def test_all_warm_is_ready() -> None: async def test_cold_local_model_blocks_ready() -> None: probe = _probe( - gateway=_FakeGateway(("llama3.2", False)), + gateway=_FakeGateway(ModelWarmth("llama3.2", False)), registry=_FakeRegistry([_snap("calendar", True)]), ) readiness = await probe.check() @@ -87,7 +87,7 @@ async def test_cold_local_model_blocks_ready() -> None: async def test_hosted_model_is_always_ready() -> None: probe = _probe( - gateway=_FakeGateway(("claude/claude-sonnet-4-6", None)), + gateway=_FakeGateway(ModelWarmth("claude/claude-sonnet-4-6", None, "hosted")), registry=_FakeRegistry([]), ) readiness = await probe.check() @@ -100,7 +100,7 @@ async def test_paused_is_never_ready() -> None: power = PowerController() power.pause() probe = _probe( - gateway=_FakeGateway(("llama3.2", False)), + gateway=_FakeGateway(ModelWarmth("llama3.2", False)), registry=_FakeRegistry([_snap("calendar", True)]), power=power, ) @@ -111,7 +111,7 @@ async def test_paused_is_never_ready() -> None: async def test_no_modules_reports_none_and_does_not_block() -> None: probe = _probe( - gateway=_FakeGateway(("llama3.2", True)), + gateway=_FakeGateway(ModelWarmth("llama3.2", True)), registry=_FakeRegistry([]), ) readiness = await probe.check() @@ -121,7 +121,7 @@ async def test_no_modules_reports_none_and_does_not_block() -> None: async def test_unhealthy_module_is_reported_not_ready() -> None: probe = _probe( - gateway=_FakeGateway(("llama3.2", True)), + gateway=_FakeGateway(ModelWarmth("llama3.2", True)), registry=_FakeRegistry([_snap("calendar", True), _snap("notes", False)]), ) readiness = await probe.check() @@ -132,7 +132,7 @@ async def test_unhealthy_module_is_reported_not_ready() -> None: async def test_registry_failure_degrades_without_blocking() -> None: probe = _probe( - gateway=_FakeGateway(("llama3.2", True)), + gateway=_FakeGateway(ModelWarmth("llama3.2", True)), registry=_FakeRegistry(RuntimeError("registry down")), ) readiness = await probe.check() @@ -153,7 +153,7 @@ async def test_gateway_failure_degrades_without_blocking() -> None: async def test_stream_yields_pending_then_resolved() -> None: probe = _probe( - gateway=_FakeGateway(("llama3.2", True)), + gateway=_FakeGateway(ModelWarmth("llama3.2", True)), registry=_FakeRegistry([_snap("calendar", True)]), ) frames = [snap async for snap in probe.stream()] @@ -164,3 +164,22 @@ async def test_stream_yields_pending_then_resolved() -> None: # The second is the resolved snapshot. assert frames[1].ready is True assert _component(frames[1], "modules").detail == "1/1 healthy" + + +async def test_no_local_runtime_reports_n_a_and_never_blocks() -> None: + """A hosted-only deployment used to report the model "warming" on every turn, forever. + + The probe asked a runtime that was not there, caught the failure, and read it as cold — + so the chat progress bar never completed once in the life of the deployment (#962). The + honest answer is that warm-up does not apply here, and it must not hold `ready` down. + """ + probe = _probe( + gateway=_FakeGateway(ModelWarmth("llama3.2", None, "absent")), + registry=_FakeRegistry([_snap("calendar", True)]), + ) + readiness = await probe.check() + model = _component(readiness, "model") + assert model.ready is True + assert model.detail == "llama3.2 · n/a" + assert "warming" not in model.detail + assert readiness.ready is True diff --git a/services/core-app/tests/test_settings.py b/services/core-app/tests/test_settings.py index bdef615f..2c12e17a 100644 --- a/services/core-app/tests/test_settings.py +++ b/services/core-app/tests/test_settings.py @@ -55,3 +55,35 @@ def test_module_hostnames_skips_hostless_entry(monkeypatch: pytest.MonkeyPatch) monkeypatch.delenv("MODULE_URLS", raising=False) settings = CoreAppSettings(service_name="test", module_urls="http://knowledge:8080, /") assert settings.module_hostnames == ["knowledge"] + + +# ── the local runtime may simply not exist (#962, ADR-0144) ────────────────────── + + +def test_local_runtime_is_enabled_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + """The default stays local-first: an operator opts *out*, never in.""" + monkeypatch.delenv("OLLAMA_URL", raising=False) + settings = CoreAppSettings(service_name="test") + assert settings.ollama_url == "http://localhost:11434" + assert settings.local_runtime_enabled is True + + +def test_a_set_ollama_url_enables_the_local_runtime(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OLLAMA_URL", "http://ollama:11434") + assert CoreAppSettings(service_name="test").local_runtime_enabled is True + + +def test_a_blank_ollama_url_means_there_is_no_local_runtime( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``OLLAMA_URL=`` is how a hosted-only deployment says so — not a misconfiguration.""" + monkeypatch.setenv("OLLAMA_URL", "") + settings = CoreAppSettings(service_name="test") + assert settings.ollama_url == "" + assert settings.local_runtime_enabled is False + + +def test_a_whitespace_ollama_url_is_blank(monkeypatch: pytest.MonkeyPatch) -> None: + """A hand-edited env file leaves spaces behind; they mean the same thing as empty.""" + monkeypatch.setenv("OLLAMA_URL", " ") + assert CoreAppSettings(service_name="test").local_runtime_enabled is False diff --git a/uv.lock b/uv.lock index 996353bc..5b912de2 100644 --- a/uv.lock +++ b/uv.lock @@ -1064,7 +1064,7 @@ provides-extras = ["s3", "db"] [[package]] name = "epicurus-core-app" -version = "0.128.0" +version = "0.129.0" source = { editable = "services/core-app" } dependencies = [ { name = "aioboto3" }, From f3b21aac30bcf187c92525ce6caa7f97a5e5f69b Mon Sep 17 00:00:00 2001 From: NikolaI Baakh Date: Sat, 19 Sep 2026 10:50:18 +0000 Subject: [PATCH 2/4] feat(infra): a deployment can choose to run no local AI, on either runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mode the core now speaks had no way to be selected. The Helm chart refused to render it — `epicurus.ollamaUrl` `required`d an external URL the moment Ollama was disabled, copied from `qdrantUrl`/`openbaoUrl`, components the core genuinely cannot run without — and Compose included the Ollama fragment unconditionally with `core-app` hard-depending on the service. Chart (0.1.2 -> 0.2.0): - `epicurus.ollamaUrl` follows `minioUrl` and renders empty; `LLM_BOOTSTRAP_MODELS` blanks itself when left at `auto` (nothing to pull into) while an explicit list is left as written. - In exchange, the guard actually worth having: a release with no local runtime whose `core.llm.defaultModel` or `core.memoryEmbedModel` is still a bare (local) name fails to render, naming the key and a hosted alias. The old guard refused a legitimate deployment; this one refuses the half-working stack where chat works and every embedding fails at call time. Compose: - `ollama` + `ollama-init` carry `profiles: [local-ai]`, core-app's dependency is `required: false`, and `OLLAMA_URL: ${OLLAMA_URL-http://ollama:11434}` (single dash — `:-` would substitute the default back in for the deliberately empty value that *is* the setting). - Local AI stays on by default: a profile is opt-in by construction, so every path that starts the stack selects it — `.env.example` ships `COMPOSE_PROFILES=local-ai`, every `task *-up` passes `--profile local-ai`, and `infra/cd/reconcile.sh` passes it unless `EPICURUS_LOCAL_AI=0`. `task hosted-only-up` is the opt-out. Nothing about an existing install changes. Gates: - `compose-validate` resolves the hosted-only stack and the local-ai profile. - `chart-validate` renders the hosted-only release *and* proves the guard refuses the half-working one. - `k8s-smoke` upgrades the live release into the mode — a real config change in place of the timestamp that used to force the roll — and asserts `absent`, a 200-and-empty model list, and a 409 from pull. The Ollama stand-in stays: it is the only workload the seam's restart arm and the chart Role's `statefulsets` verb ever have on kind (#919), and retiring it would re-open that hole. chart 0.1.2 -> 0.2.0 (MINOR). Part of #962. --- .env.example | 22 ++ .github/workflows/ci.yml | 57 +++++ CHANGELOG.md | 27 +++ Taskfile.yml | 15 +- docs/infrastructure/index.md | 39 ++++ docs/infrastructure/kubernetes.md | 51 ++++- docs/infrastructure/startup-and-recovery.md | 30 ++- docs/reference/config.md | 2 +- docs/reference/platform-api.md | 45 ++++ docs/services/core-app.md | 58 ++++- infra/cd/reconcile.sh | 14 +- infra/ci/k8s-smoke.sh | 44 +++- infra/ci/ollama-stub.yaml | 10 + infra/ci/smoke.sh | 8 +- infra/ci/values-ci.yaml | 7 + infra/k8s/epicurus/Chart.yaml | 2 +- infra/k8s/epicurus/templates/NOTES.txt | 9 +- infra/k8s/epicurus/templates/_helpers.tpl | 64 +++++- infra/k8s/epicurus/templates/core-app.yaml | 6 +- infra/k8s/epicurus/values.yaml | 24 +- infra/ollama/compose.yaml | 16 ++ services/core-app/compose.yaml | 11 +- tests/test_no_local_runtime.py | 238 ++++++++++++++++++++ 23 files changed, 769 insertions(+), 30 deletions(-) create mode 100644 tests/test_no_local_runtime.py diff --git a/.env.example b/.env.example index 7479938a..91d39dca 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,28 @@ # Environment name: local | staging | production APP_ENV=local +# ── Local AI: on by default, and optional (#962) ───────────────────────────── +# The local LLM runtime (Ollama) carries the `local-ai` compose profile, so a +# deployment can run without one: hosted chat and hosted embeddings, nothing +# local. It is ON by default — this line is what selects it for a bare +# `docker compose up -d`. (`task up` / `task obs-up` / `infra/cd/reconcile.sh` +# pass `--profile local-ai` themselves, so they are correct either way.) +COMPOSE_PROFILES=local-ai + +# For a HOSTED-ONLY stack, all three of these, not just the first: +# 1. COMPOSE_PROFILES= — leave Ollama out of the stack entirely +# 2. OLLAMA_URL= — tell the core there is no local runtime. Blank is a +# deliberate value here, not an omission; without it +# the core keeps probing a host that is not there. +# 3. hosted model defaults — a bare name routes to the local runtime, so: +# LLM_DEFAULT_MODEL=claude/claude-sonnet-4-6 +# MEMORY_EMBED_MODEL=gpt/text-embedding-3-small +# `task hosted-only-up` does 1 and 2 for you. With no local runtime the core +# refuses pull / delete / unload / the KV-cache setting with 409 and a reason, +# the Models page collapses its local half, and chat readiness reports the model +# as n/a rather than "warming" forever. +# OLLAMA_URL=http://ollama:11434 + # ── Image tag pinning (#56) ────────────────────────────────────────────────── # All service compose fragments default to `${EPICURUS_VERSION:-latest}`. # Omitting this var (the default) pulls :latest — convenient for local dev diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87a16f3e..72bd8594 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,6 +158,28 @@ jobs: - name: Validate the assembled compose stack (observability profile) run: docker compose -f compose.yaml --profile observability config -q + # The default stack, local AI on — the shape every operator installs and the one + # `runtime-smoke` boots. Ollama carries `profiles: [local-ai]` since #962 so a + # hosted-only deployment can leave it out; this proves the on path still resolves. + - name: Validate the assembled compose stack (local-ai profile) + run: docker compose -f compose.yaml --profile local-ai config -q + + # And the hosted-only shape (#962, ADR-0144): no `local-ai` profile and a blank + # OLLAMA_URL. The assertion is not just that it parses — it is that core-app's + # dependency on the absent ollama service is `required: false`, which is what makes + # the whole `up` succeed rather than fail on a service that is missing on purpose. + - name: Validate the assembled compose stack (hosted-only — no local runtime) + run: | + OLLAMA_URL= docker compose -f compose.yaml config -q + rendered="$(OLLAMA_URL= docker compose -f compose.yaml config)" + if ! printf '%s' "$rendered" | grep -q 'OLLAMA_URL: ""'; then + echo "a blank OLLAMA_URL did not survive interpolation"; exit 1 + fi + if printf '%s' "$rendered" | grep -qE '^ ollama:'; then + echo "ollama is still selected without the local-ai profile"; exit 1 + fi + echo "hosted-only stack resolves with no local runtime" + # The Helm chart gets the same treatment the compose stack does: rendered, then # checked. `helm lint` catches template errors, `helm template` proves the branches # render at all, and `kubeconform -strict` validates the result against the real @@ -234,6 +256,41 @@ jobs: --set searxng.enabled=false,searxng.external.url=http://searx.example.com:8080 \ | kubeconform -strict -summary -kubernetes-version "$KUBE_VERSION" - + # No local LLM runtime at all (#962, ADR-0144) — the third Ollama state, which the + # chart refused to render until now. Hosted model ids are mandatory here, so this + # renders the escape hatch the guard's message names. + - name: Render and validate — hosted-only (no local runtime) + run: | + rendered="$(helm template epicurus "$CHART" \ + --set ollama.enabled=false \ + --set core.llm.defaultModel=claude/claude-sonnet-4-6 \ + --set core.memoryEmbedModel=gpt/text-embedding-3-small)" + printf '%s' "$rendered" | kubeconform -strict -summary -kubernetes-version "$KUBE_VERSION" - + if ! printf '%s' "$rendered" | grep -A1 'name: OLLAMA_URL' | grep -q 'value: ""'; then + echo "OLLAMA_URL is not empty on a hosted-only release"; exit 1 + fi + if ! printf '%s' "$rendered" | grep -A1 'name: LLM_BOOTSTRAP_MODELS' | grep -q 'value: ""'; then + echo "LLM_BOOTSTRAP_MODELS was not blanked on a hosted-only release"; exit 1 + fi + if printf '%s' "$rendered" | grep -q 'app.kubernetes.io/component: ollama'; then + echo "an Ollama workload was rendered for a hosted-only release"; exit 1 + fi + echo "hosted-only release renders with no local runtime" + + # And the guard that makes the mode safe: a hosted-only release whose models are + # still bare local names must FAIL to render, naming the fix. A guard nothing proves + # is a guard that quietly stops guarding. + - name: Render — hosted-only with a local model name must be refused + run: | + if helm template epicurus "$CHART" --set ollama.enabled=false >/dev/null 2>err.txt; then + echo "the chart rendered a hosted-only release with local model defaults"; exit 1 + fi + grep -q 'no local LLM runtime' err.txt \ + || { echo "the refusal did not name the mode:"; cat err.txt; exit 1; } + grep -q 'core.memoryEmbedModel' err.txt \ + || { echo "the refusal did not name the fix:"; cat err.txt; exit 1; } + echo "the guard refuses a half-working hosted-only release" + # Validates Prometheus and Alertmanager configs without booting the # observability stack. The runtime-smoke gate skips that stack, so a bad rule # (e.g. an undefined template function) would otherwise reach production diff --git a/CHANGELOG.md b/CHANGELOG.md index 3adc1589..14e80854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,33 @@ images to GHCR. ## [Unreleased] +- **A hosted-only deployment runs no local runtime** (#962) — running with hosted chat and + hosted embeddings and *no* Ollama was a documented capability that nothing actually supported. + The Helm chart refused to render it (`ollama.enabled: false` demanded an external URL), Compose + could not express it (an unconditional fragment and a hard `depends_on`), and an operator who + forced it with a placeholder URL met a core that collapsed three different facts into one: + **absent** (no runtime configured — a deliberate choice), **unreachable** (one is configured + and does not answer) and **ok**. `GET /platform/v1/llm/models` **500ed**, on a page that polls + it every ten seconds; pull and delete 500ed the same way; the chat warm-up indicator reported + the model "warming" forever, on every turn, for the life of the deployment; and startup spent + 180 seconds polling an address that was never going to answer. Absence is now a first-class + mode, spelled `OLLAMA_URL=""`: the model list answers **200 and an empty array** in both + non-serving states, the new `GET /platform/v1/llm/local-runtime` says which one applies, the + local-only actions (pull, delete, unload, the KV-cache setting) refuse with **409** and a + sentence naming the mode — **502** when a configured runtime is unreachable, never a bare 500 + — readiness reports the model as **n/a** instead of warming, and the bootstrap logs one line + and returns. A local model id asked to serve is refused before any provider call with the + capability error the rest of the gateway already speaks (ADR-0140), so a hosted embedding model + keeps working while a bare one fails with the fix in the message instead of a connection error. + On **Kubernetes** the chart renders an empty `OLLAMA_URL`, blanks `LLM_BOOTSTRAP_MODELS` on its + own, and gains the guard actually worth having — it refuses to render a runtime-less release + whose chat or embedding default is still a bare local name, naming the hosted alias to set. On + **Compose** Ollama moves behind a `local-ai` profile that every start path (`task up`, + `task obs-up`, `infra/cd/reconcile.sh`, `.env.example`) selects explicitly, so the default + install is unchanged and `task hosted-only-up` is the opt-out. Both gates cover it: + `compose-validate` resolves the hosted-only stack, `chart-validate` renders it *and* proves the + guard refuses the half-working one, and `k8s-smoke` upgrades a live release into the mode and + asserts the core answers. `core-app` 0.128.0→0.129.0 (MINOR), chart 0.1.2→0.2.0 (MINOR). - **The bound on a turn is the operator's; runaway is caught by behaviour** (#925) — the **Agent cycles** setting stopped at 12, and the route enforced it *silently*: type 40 and 12 was stored. A genuinely long task — search → read → read → summarize → write — ran out of rounds and diff --git a/Taskfile.yml b/Taskfile.yml index f77c372d..bfef0762 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -62,9 +62,14 @@ tasks: - uv run python scripts/migrate.py check {{.CLI_ARGS}} up: - desc: Start the full stack (data plane + enabled modules; Docker control is on by default through docker-proxy-core, observability is opt-in — see `obs-up` / `docker-socket-up`) + desc: Start the full stack (data plane + enabled modules + the local AI runtime; Docker control is on by default through docker-proxy-core, observability is opt-in — see `obs-up` / `docker-socket-up` / `hosted-only-up`) cmds: - - docker compose up -d + - docker compose --profile local-ai up -d + + hosted-only-up: + desc: 'Start the stack with NO local LLM runtime (#962) — no Ollama container, and OLLAMA_URL blank so the core says so instead of probing a host that is not there. Set LLM_DEFAULT_MODEL + MEMORY_EMBED_MODEL to hosted ids first, or chat and embedding both fail at call time' + cmds: + - OLLAMA_URL= docker compose up -d down: desc: Stop the full stack (append `-- -v` to also drop volumes) @@ -74,7 +79,7 @@ tasks: obs-up: desc: Start the stack with the opt-in observability profile (Grafana/Prometheus/Loki/Tempo) cmds: - - docker compose --profile observability up -d + - docker compose --profile local-ai --profile observability up -d obs-down: desc: Stop the stack including the observability profile (append `-- -v` to drop volumes) @@ -84,7 +89,7 @@ tasks: docker-socket-up: desc: 'Start the stack with core-app on the RAW Docker socket instead of the default docker-proxy-core (#622, #708) — root-equivalent, no allowlist in front of it; requires DOCKER_GID (see services/core-app/compose.docker-socket.yaml)' cmds: - - docker compose -f compose.yaml -f services/core-app/compose.docker-socket.yaml up -d + - docker compose --profile local-ai -f compose.yaml -f services/core-app/compose.docker-socket.yaml up -d docker-socket-down: desc: Stop the stack including the Docker-socket override (append `-- -v` to drop volumes) @@ -94,7 +99,7 @@ tasks: external-mounts-up: desc: 'Start the stack with operator-declared external file mounts (#731) — additional Files roots bound from the host; requires EXTERNAL_MOUNT_*_HOST_PATH + FILES_EXTERNAL_MOUNTS (see services/core-app/compose.external-mounts.yaml)' cmds: - - docker compose -f compose.yaml -f services/core-app/compose.external-mounts.yaml up -d + - docker compose --profile local-ai -f compose.yaml -f services/core-app/compose.external-mounts.yaml up -d external-mounts-down: desc: Stop the stack including the external-mounts override (append `-- -v` to drop volumes) diff --git a/docs/infrastructure/index.md b/docs/infrastructure/index.md index 573c70ef..4ab950fb 100644 --- a/docs/infrastructure/index.md +++ b/docs/infrastructure/index.md @@ -241,6 +241,45 @@ hit `PermissionError` and the choice would save but never apply (#392). `ollama- only**: the core's write is lazy — it happens when the operator changes the KV-cache type, long after boot — so there is no startup race regardless. +### Hosted-only: no local LLM runtime + +A deployment can run **no local runtime at all** — hosted chat and hosted embeddings, nothing +local (#962, ADR-0144). `ollama` and `ollama-init` carry the **`local-ai` compose profile**, so +they can be left out of the stack entirely; a blank `OLLAMA_URL` is how the core is told that is +deliberate, and `core-app`'s dependency on `ollama` is `required: false` so its absence does not +fail the whole `up`. + +**Local AI is on by default and stays on.** A compose profile is opt-in by construction, so every +path that starts the stack selects it explicitly: `.env.example` ships `COMPOSE_PROFILES=local-ai` +(what a bare `docker compose up -d` reads), `task up` / `task obs-up` / `task docker-socket-up` / +`task external-mounts-up` pass `--profile local-ai`, and `infra/cd/reconcile.sh` passes it unless +`EPICURUS_LOCAL_AI=0`. Nothing about an existing install changes. + +To run hosted-only, three things — the first alone gives you a stack with no runtime *and* a core +still looking for one: + +```bash +# 1. leave Ollama out of the stack 2. tell the core there is no local runtime +# (COMPOSE_PROFILES= in .env, or just) (OLLAMA_URL= in .env, or just) +OLLAMA_URL= docker compose up -d # == task hosted-only-up + +# 3. in .env, point both model defaults at hosted ids — a bare name routes to the +# local runtime, so leaving these is the half-working stack: +# LLM_DEFAULT_MODEL=claude/claude-sonnet-4-6 +# MEMORY_EMBED_MODEL=gpt/text-embedding-3-small +``` + +Add each provider's API key on the **Models** page before the first turn. With no local runtime +the core refuses every local-only action with a reason — `409` from pull, delete, unload and the +KV-cache setting — `GET /platform/v1/llm/models` answers `200` with an empty list rather than +500ing, `GET /platform/v1/llm/local-runtime` reports `absent`, and chat readiness reports the +model as `n/a` instead of "warming" forever. A local model id asked to answer is refused with a +sentence naming the fix, before any provider call. The Kubernetes equivalent is +[`ollama.enabled: false` with a blank `ollama.external.url`](kubernetes.md#hosted-only-no-local-llm-runtime). + +To go back, remove `OLLAMA_URL=` from `.env` and bring the stack up with the profile again; the +`ollama-models` volume is untouched by any of this, so the models are still there. + ## Log retention Every service in every compose fragment sets a bounded `json-file` logging driver diff --git a/docs/infrastructure/kubernetes.md b/docs/infrastructure/kubernetes.md index 0676bc4c..476d1253 100644 --- a/docs/infrastructure/kubernetes.md +++ b/docs/infrastructure/kubernetes.md @@ -398,7 +398,7 @@ cluster points at managed services. | `ollama.resources` | `requests: 500m / 4Gi` (add `limits` — memory especially) | | `ollama.env` | `OLLAMA_KEEP_ALIVE: 5m`, `OLLAMA_FLASH_ATTENTION: "0"`, `OLLAMA_KV_CACHE_TYPE: f16` | | `ollama.gpu.enabled` / `.count` / `.resourceName` / `.runtimeClassName` | `false` / `1` / `nvidia.com/gpu` / `""` | -| `ollama.external.url` | `""` | +| `ollama.external.url` | `""` — blank with `ollama.enabled: false` means **no local runtime at all** (see below) | | `searxng.enabled` | `true` | | `searxng.image.repository` / `.tag` | `searxng/searxng` / `2026.6.10-de03f4eb1` | | `searxng.settings` | `""` (blank = the compose `settings.yml`) | @@ -411,9 +411,52 @@ defaults set `requests` only, and `limits` are yours to add. Each data-plane blo also takes `nodeSelector`, `tolerations` and `affinity`. Every `external.url` is required once its `enabled` is `false` — Helm fails the render with a named message rather than deploying something that cannot connect. -Postgres is the exception in shape: an external server is addressed by -`external.host`/`.port` and the credentials still come from the Secret, so no DSN -with a password in it ever sits in a values file. +**Ollama and MinIO are the two exceptions**, because unlike Postgres, NATS, Qdrant +and OpenBao the core can genuinely run without either: a blank `ollama.external.url` +with `ollama.enabled: false` means *there is no local LLM runtime*, which is a +supported deployment and not an omission (see below). Postgres is the exception in +shape: an external server is addressed by `external.host`/`.port` and the +credentials still come from the Secret, so no DSN with a password in it ever sits in +a values file. + +### Hosted-only: no local LLM runtime + +Three Ollama deployments, not two (#962, ADR-0144): + +| | `ollama.enabled` | `ollama.external.url` | `OLLAMA_URL` in the pod | +| --- | --- | --- | --- | +| the chart runs Ollama (default) | `true` | ignored | `http://ollama:11434` | +| an Ollama you run elsewhere | `false` | your URL | your URL | +| **no local runtime at all** | `false` | `""` | `""` | + +The third is hosted chat and hosted embeddings with nothing local. It is a real mode, not a +degraded one: with an empty `OLLAMA_URL` the core reports `absent` at +`GET /platform/v1/llm/local-runtime`, answers `200` and `[]` from `GET /platform/v1/llm/models` +(rather than the 500 it used to, on a page that polls every ten seconds), refuses pull / delete +/ unload / the KV-cache setting with **409** and a sentence naming the mode, reports the chat +model as `n/a` in readiness instead of "warming" forever, and skips the first-boot model +bootstrap in one log line. The chart also blanks `LLM_BOOTSTRAP_MODELS` for you when it is left +at `auto` — there is nothing to pull into — while leaving an explicit list you set alone. + +**Both model defaults must name hosted models**, and the chart enforces it at render time: + +```bash +helm install epicurus oci://ghcr.io/baakhoff/charts/epicurus \ + --namespace epicurus --create-namespace \ + --set ollama.enabled=false \ + --set core.llm.defaultModel=claude/claude-sonnet-4-6 \ + --set core.memoryEmbedModel=gpt/text-embedding-3-small +``` + +Leave the chart's defaults (`llama3.2`, `nomic-embed-text`) in place and the render **fails**, +naming the key and an example value. That is deliberate: a bare model name routes to the local +runtime, so a runtime-less release with bare defaults is the half-working stack where chat works +through the hosted provider while memory recall and every module index fail at call time. The +old guard — `required` on `ollama.external.url` — refused a *legitimate* deployment; this one +refuses a broken one. Add each provider's API key on the Models page before the first turn. + +The Compose equivalent is the +[`local-ai` profile](index.md#hosted-only-no-local-llm-runtime). **MinIO is on by default**, matching the Compose stack. It backs two different things: the `storage` module's object store (chat uploads, agent-written objects, diff --git a/docs/infrastructure/startup-and-recovery.md b/docs/infrastructure/startup-and-recovery.md index fb2a4723..2fa365cf 100644 --- a/docs/infrastructure/startup-and-recovery.md +++ b/docs/infrastructure/startup-and-recovery.md @@ -94,8 +94,34 @@ unreachable) — pull it from the web UI's **Models** page. Restarting the core retry it: if the other default landed, the runtime is no longer empty and `auto` no-ops (#923). Name the model in `LLM_BOOTSTRAP_MODELS` if you want a restart to keep trying for it. `LLM_BOOTSTRAP_MODELS=` -(blank) disables the bootstrap entirely — intended for hosted-only or air-gapped -deployments, where a local 404 instead means the model was simply never pulled. +(blank) disables the bootstrap entirely — intended for air-gapped deployments, where a +local 404 instead means the model was simply never pulled. + +On a **hosted-only** deployment (`OLLAMA_URL=`, #962) none of this applies: the bootstrap +returns in one log line, and a local model id fails with a **400** naming the mode ("No +local runtime is configured — choose a hosted model") rather than a 404 from a runtime that +is not there. If you *do* see a 404 or a connection error naming Ollama on such a +deployment, the blank `OLLAMA_URL` did not reach the container — check +`docker compose exec core-app printenv OLLAMA_URL` (it must print nothing) and that the +compose interpolation is `${OLLAMA_URL-…}`, not `${OLLAMA_URL:-…}`. + +### The Models page shows nothing, or 500s every few seconds {#models-page-empty} + +`GET /platform/v1/llm/models` answers `200` with an empty list whenever the local runtime +cannot serve — both when there is none (`OLLAMA_URL` blank) and when one is configured but +unreachable (#962, ADR-0144). Which of the two it is comes from +`GET /platform/v1/llm/local-runtime` → `{"state": "absent"|"unreachable"|"ok"}`: + +```bash +docker compose exec core-app python -c \ + "import urllib.request,sys; sys.stdout.write(urllib.request.urlopen('http://127.0.0.1:8080/platform/v1/llm/local-runtime').read().decode())" +``` + +`absent` is the hosted-only mode and nothing is wrong. `unreachable` means the container is +down or the URL is wrong — check that `ollama` is running (`docker compose ps ollama`; it +starts only with the `local-ai` profile, which `task up` and `infra/cd/reconcile.sh` pass for +you) and that `OLLAMA_URL` matches it. A 500 from that endpoint is a bug worth reporting: it +was the symptom this contract exists to remove. ### OpenBao is sealed {#openbao-sealed} diff --git a/docs/reference/config.md b/docs/reference/config.md index 7c04c255..201e7f07 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -74,7 +74,7 @@ in `CoreSettings` plus the LLM-gateway, agent, module, and memory knobs. | Field | Env var | Type | Default | Meaning | | --- | --- | --- | --- | --- | -| `ollama_url` | `OLLAMA_URL` | `str` | `http://localhost:11434` | Local LLM runtime (the stack reaches it at `http://ollama:11434`). | +| `ollama_url` | `OLLAMA_URL` | `str` | `http://localhost:11434` | Local LLM runtime (the stack reaches it at `http://ollama:11434`). **Blank means this deployment has no local runtime at all** (#962, ADR-0144) — a deliberate hosted-only install, not a misconfiguration; whitespace counts as blank. The derived `local_runtime_enabled` is what every call site asks. With no runtime: `GET /llm/models` answers 200 and `[]`, `GET /llm/local-runtime` reports `absent`, pull / delete / unload / the KV-cache setting answer **409**, a local model id is refused with a capability error before any provider call, readiness reports the model `n/a`, and the first-boot bootstrap returns at once. See [Hosted-only deployments](../infrastructure/index.md#hosted-only-no-local-llm-runtime). | | `ollama_runtime_env_path` | `OLLAMA_RUNTIME_ENV_PATH` | `str` | `/etc/epicurus/ollama.env` | Where the core writes Ollama's start-up env file (KV-cache type) for it to source on restart (#307). A shared volume; override only if you remap the mount. | | `ollama_service_name` | `OLLAMA_SERVICE_NAME` | `str` | `ollama` | Compose service the core restarts to apply a KV-cache change (#307). | | `llm_default_model` | `LLM_DEFAULT_MODEL` | `str` | `llama3.2` | Model used when a request names none. | diff --git a/docs/reference/platform-api.md b/docs/reference/platform-api.md index 3748703f..1c14a7ab 100644 --- a/docs/reference/platform-api.md +++ b/docs/reference/platform-api.md @@ -346,6 +346,51 @@ tenant-scoped (both mirror a public registry). --- +## `GET /platform/v1/llm/local-runtime` + +Whether this deployment has a **local** LLM runtime, and whether it is answering (#962, +ADR-0144). Shell-facing; no body, no query params. + +**Response** + +```json +{ "state": "absent", "url_configured": false } +``` + +| `state` | meaning | what a surface should do | +| --- | --- | --- | +| `absent` | `OLLAMA_URL` is blank — a deliberate hosted-only deployment | collapse the local half: no pull card, no catalog, no KV-cache or context-window card, no `Local (Ollama)` group in a model picker | +| `unreachable` | a runtime **is** configured and did not answer | show today's warning — this state *is* an error and should look like one | +| `ok` | it answered | the full local UI | + +`url_configured` is the *why* behind the state, and is `false` exactly when `state` is +`absent`. + +Three facts that used to be one, which is why this endpoint exists rather than an envelope +around the model list. `GET /platform/v1/llm/models` stays a bare `list[ModelInfo]` (twelve +consumers read that array) and **never 500s again**: it answers `200` with `[]` when the +runtime is absent *and* when it is unreachable. Everything that genuinely needs a runtime +answers **409** when it is absent, with a `detail` naming the mode, and **502** when it is +configured but unreachable — never a bare 500: + +- `POST /platform/v1/llm/pull` · `POST /platform/v1/llm/pull/stream` (refused **before** the + stream starts, so the caller sees a real status rather than a 200 whose only event is an + error) · `DELETE /platform/v1/llm/models` · `POST /platform/v1/llm/unload`; +- `PUT /platform/v1/llm/prefs/kv-cache-type` — **409 when absent only**. This path never talks + to Ollama (it writes the start-up env file and asks the container runtime to bounce the + workload), so "unreachable" is not something it can observe, and setting the value while the + server is down is legitimate: `applied`/`staged` already report how far it got (#709). + +Inference itself is refused one layer earlier: a **local** model id on a runtime-less +deployment raises the same `ModelCapabilityError` shape as every other capability refusal +(ADR-0140) → **400** with `{"error": "wrong_model_role", …}` and the hint *"No local runtime is +configured — choose a hosted model."* A hosted model — chat **or** embedding — is untouched, so +memory recall and module indexing keep working with `gpt/text-embedding-3-small` and friends. +`GET /platform/v1/readiness` reports the model component as ` · n/a` and **ready**, not +"warming" forever. + +--- + ## `GET /platform/v1/agent/instructions` · `PUT /platform/v1/agent/instructions` The agent's editable **base system prompt** (#497, ADR-0083) — injected as the **first** message diff --git a/docs/services/core-app.md b/docs/services/core-app.md index 7df8cdaa..5908d8af 100644 --- a/docs/services/core-app.md +++ b/docs/services/core-app.md @@ -527,12 +527,13 @@ own `POST /platform/v1/llm/chat` was **removed in `core-app` 0.2.0** — it dupl | Method · Path | Purpose | | --- | --- | -| `GET /platform/v1/llm/models[?capabilities=true]` · `DELETE /platform/v1/llm/models?name=…` | List / remove local models (the `loaded` flag marks in-memory ones). `?capabilities=true` additionally fills each model's reported `capabilities` (e.g. `tools`, `vision`) and trained `context_length` (#618) from `/api/show` — opt-in (one call per model), so the Models page can badge them and show a context-window chip while the chat picker stays light. `context_length` is `null` when the runtime doesn't report it — never a fake default. | +| `GET /platform/v1/llm/models[?capabilities=true]` · `DELETE /platform/v1/llm/models?name=…` | **The list never fails**: `200` with `[]` when the local runtime is absent or unreachable (#962) — which state applies is `GET /llm/local-runtime`'s job, not an envelope here. `DELETE` answers **409** with no local runtime and **502** when one is unreachable. List / remove local models (the `loaded` flag marks in-memory ones). `?capabilities=true` additionally fills each model's reported `capabilities` (e.g. `tools`, `vision`) and trained `context_length` (#618) from `/api/show` — opt-in (one call per model), so the Models page can badge them and show a context-window chip while the chat picker stays light. `context_length` is `null` when the runtime doesn't report it — never a fake default. | | `GET /platform/v1/llm/models/details?model=…` | Read-only facts about a model: `{quantization, parameter_size, context_length, family, capabilities}` (any field `null`/empty when not reported — never a fake default). Local models read the runtime's `/api/show`; **hosted** models (#633/#618) read LiteLLM's own model-cost/context map instead (no provider call) — `quantization`/`parameter_size`/`family` stay `null` there (Ollama-only concepts), `capabilities` is what the resolution actually decided — `tools` only when the model is resolved tool-capable (no longer hard-coded), `vision` when the map or an override says so, `embedding` for an embedding model. Three resolved fields ride beside it (ADR-0140): `role` (`chat`|`embedding`|`unknown`), `supports_tools` (`true`/`false`, `null` when the local runtime could not be asked at all) and `in_catalogue` (`false` for a hosted id LiteLLM's map has never heard of — `null` for a local model). They exist because `capabilities` cannot express *unknown*: an empty list means both "nothing to badge" and "no idea", and a shell guessing between them shows the wrong hint. Backs the model-settings sheet, the Models page's context-window chip and **unlisted** badge, and the chat "can't use tools" / "can't see images" hints. `model` is a query param (names carry `:`/`/`). | | `GET /platform/v1/llm/catalog` | The browsable model catalog the core parses from upstream on a schedule (#269). Returns `{entries[], source, updated_at, stale}`; each entry's `size_gb` is the **real on-disk size** backfilled from its family's tags page (#571; `null` until the size fill or a variant lookup reaches the family, and always `null` for `cloud` rows). `stale` flags a seed / last-good list served after a failed or skipped refresh. See **Model catalog** below. | | `GET /platform/v1/llm/catalog/variants?model=…` | The quant variants available for a model (#330), looked up on demand from the model's public library **tags page** (the catalog index lists *sizes*, not quants). Returns `{model, variants:[{tag, quant, size_gb}]}` — `size_gb` is the tag row's real on-disk size (#571; `null` when upstream shows none, e.g. a cloud alias). Best-effort — an empty list (offline, or a model not in the public library) makes the UI fall back to a manual tag box. A successful lookup also piggybacks its sizes onto the catalog snapshot. `model` is a query param. See **Model catalog** below. | -| `POST /platform/v1/llm/pull` · `POST /platform/v1/llm/pull/stream` | Pull a model (blocking / SSE progress). | -| `POST /platform/v1/llm/unload` | Drop model(s) from memory now (`keep_alive=0`) **without** changing power state (#331). Body `{model: str\|null}` — `null`/omitted unloads every loaded model, a name unloads just that one. Returns `{status, model}` (`"all"` when none given). The standalone unload the Models page calls; the `loaded` flag refreshes on the next poll. | +| `GET /platform/v1/llm/local-runtime` | Whether this deployment has a local LLM runtime, and whether it answers (#962, ADR-0144): `{state: "absent"|"unreachable"|"ok", url_configured: bool}`. `absent` means `OLLAMA_URL` is blank — a deliberate hosted-only deployment, where a surface should *collapse* its local half rather than draw it broken; `unreachable` is an error and should still look like one. A separate endpoint on purpose: `GET /llm/models` stays a bare array (twelve consumers read it) and now answers `200` with `[]` in both non-serving states instead of the 500 the Models page collected every ten seconds. | +| `POST /platform/v1/llm/pull` · `POST /platform/v1/llm/pull/stream` | Pull a model (blocking / SSE progress). **409** when this deployment runs no local runtime, **502** when one is configured and unreachable (#962) — never a bare 500. The SSE form refuses *before* the response starts, so the caller gets a real status rather than a 200 whose only event is an error. | +| `POST /platform/v1/llm/unload` | Drop model(s) from memory now (`keep_alive=0`) **without** changing power state (#331). Body `{model: str\|null}` — `null`/omitted unloads every loaded model, a name unloads just that one. Returns `{status, model}` (`"all"` when none given). The standalone unload the Models page calls; the `loaded` flag refreshes on the next poll. **409** with no local runtime (#962): the gateway's own `unload` stays silent there because it is also on the power-pause path, which must keep working on a hosted-only box, so the refusal is made at the route — where the caller is an operator who clicked Unload and deserves to know why nothing happened. | | `GET /platform/v1/llm/providers` | Providers and what the secret store knows about each one's key. Each row is `{alias, local, configured, needs_base_url, key_state, key_error}`. `key_state` is `not_required` (the local runtime holds no key) / `present` / `missing` (OpenBao answered and has nothing there) / `unavailable` (OpenBao could not be asked — an expired app token, the service down), with `key_error` naming the reason for the last one. `configured` is unchanged (`true` for `not_required` and `present`) — it was one bit over three facts, and collapsing "we could not ask" into "there is no key" is how #728's expired token read as a fleet of unconfigured providers, sending the operator to re-enter keys that were already set. The core reports the distinction; rendering it is the shell's job (ADR-0018) — the Models page's "Add a hosted model" row (#922) is the first place that reads `key_state`, hinting inline when it is `missing`/`unavailable`. | | `PUT` · `DELETE /platform/v1/llm/providers/{alias}/key` | Store / clear a hosted provider's key (core → OpenBao; never logged or returned). | | `GET /platform/v1/llm/prefs` | Stored preferences: `global_default` (chat), `global_embed_default` (embedding), `global_context_window` (num_ctx), `kv_cache_type` (Ollama KV-cache), `global_agent_max_steps` (agent loop bound), `hidden` (model list). | @@ -573,6 +574,41 @@ with `partition`, never `split("/")` — in the core (`resolve` / `is_hosted`) a OpenRouter *embedding* ids, so `/models/details` and a saved model's `context_length` come back `null` for an OpenRouter embedding id. That is honest, not a bug — never a fake default. +#### No local runtime (#962, ADR-0144) + +`OLLAMA_URL` blank means **this deployment has no local runtime**, deliberately — hosted chat +and hosted embeddings, nothing local. It is a third state, and the point of naming it is that +the gateway used to have only two: *absent*, *unreachable* and *misconfigured* were all "the +call failed", so each call site guessed. Some caught and degraded (`show`, `unload`), some +propagated to a 500 (`models`, `pull`, `delete`), and one polled for three minutes. + +`CoreAppSettings.local_runtime_enabled` (`bool(ollama_url.strip())`) is the fact; everything +else reads it: + +| | with no local runtime | +| --- | --- | +| `LlmGateway.models()` | `[]`, no HTTP call — and `[]` rather than a raise when a *configured* runtime is unreachable, which is what stops `GET /llm/models` 500ing every ten seconds on the Models page | +| `LlmGateway.local_runtime_state()` | `absent` \| `unreachable` \| `ok`, behind `GET /platform/v1/llm/local-runtime` | +| `pull` / `pull_stream` / `delete_model` | raise `LocalRuntimeUnavailableError(state="absent")` → **409**; an unreachable runtime → **502** | +| `unload` | returns quietly (it is on the power-pause path, which must keep working); the **route** answers 409 | +| `_ensure_can_serve` | refuses a **local** model id with `ModelCapabilityError` → **400** and the hint *"No local runtime is configured — choose a hosted model."* — asked **before** the pause rule, because "resume to run inference" is an instruction an operator with no runtime cannot follow | +| the chat fallback chain | skips local candidates (`_is_available`), so a hosted primary never falls back into a runtime that is not there | +| `model_readiness` | `ModelWarmth(model, warm=None, runtime="absent")` → the readiness component reads ` · n/a` and **ready**, instead of "warming" forever | +| `ModelBootstrap` | one log line and return — no 180s poll | +| `OllamaRuntime.apply_kv_cache_type` | the existing "unavailable" result (`applied=False, staged=False`); nothing is written and neither arm of the container seam (ADR-0134) is asked to find a workload | + +The refusal for a local model id is the same `ModelCapabilityError` shape the role gate uses +(ADR-0140) rather than a new one: an operator asking a model that cannot serve is one +situation, and the reason it cannot serve — wrong role, or no runtime to run it — belongs in +the message, not in a second error type for every surface to learn. This is also what closes +the *quiet* failure: `show()` returns empty details when the runtime cannot be asked, the role +reads `unknown`, and `unknown` is waved through by design — so without the absence check a +local model would sail past the gate and die at the provider with a connection error. + +The deployment surfaces that select the mode are documented per runtime: +[Compose](../infrastructure/index.md#hosted-only-no-local-llm-runtime) · +[Kubernetes](../infrastructure/kubernetes.md#hosted-only-no-local-llm-runtime). + #### Embeddings — local and hosted (#865) `LlmGateway.embed` classifies its model id through the same registry as chat, so an embedding @@ -749,9 +785,15 @@ Behaviour is otherwise bounded and defensive, in keeping with what startup may c background while the rest of the core serves. - **Retries with exponential backoff** per model (the pull resumes partial downloads), then gives up with a warning naming the Models page as the manual fallback. -- **Hosted ids are skipped** (`claude/…` cannot be pulled into the local runtime), and an - unreachable runtime (a hosted-only deployment running no Ollama) costs one warning after a - bounded wait, never a crash loop. +- **Hosted ids are skipped** (`claude/…` cannot be pulled into the local runtime), and a + runtime that is configured but still down costs one warning after a bounded wait (180s of + polling), never a crash loop. +- **A deployment with no local runtime at all returns immediately** — one log line, no poll + (#962, ADR-0144). Until then this paragraph claimed the bounded wait covered "a hosted-only + deployment running no Ollama", and it did in the sense that nothing crashed: every process + start simply spent three minutes asking an address that was never going to answer. Absence + is now a fact the core *knows* (`OLLAMA_URL` is blank) rather than one it infers from three + minutes of failures. `LLM_BOOTSTRAP_MODELS` tunes it: `auto` (default) seeds an empty runtime with the effective defaults, then no-ops forever after; blank disables the bootstrap (air-gapped builds — and @@ -1170,7 +1212,7 @@ that previously had no bound at all. | Method · Path | Purpose | | --- | --- | -| `GET /platform/v1/readiness?model=…` | A warming snapshot — `{ready, power, components[]}` — folding the power state, module health (compose health), and whether the turn's model is warm (hosted models are always ready). Best-effort: a slow/failing component reports not-yet-ready rather than erroring. The chat stream emits the **same** snapshot as leading `readiness` events so the UI shows a progress bar before the first token. | +| `GET /platform/v1/readiness?model=…` | A warming snapshot — `{ready, power, components[]}` — folding the power state, module health (compose health), and whether the turn's model is warm. Three ways to be ready without warming up, and the `model` component's `detail` says which: `· hosted` (a provider needs no warm-up), `· n/a` (**this deployment has no local runtime**, #962 — it used to report `warming` forever, on every turn, for the life of the deployment) and `· warm`. None of them holds `ready` down. Best-effort: a slow/failing component reports not-yet-ready rather than erroring. The chat stream emits the **same** snapshot as leading `readiness` events so the UI shows a progress bar before the first token. | ### Module registry (ADR-0004/0007) @@ -1883,7 +1925,7 @@ decision that already landed. Payload shapes and dedup keys are in the | Env var | Default | Meaning | | --- | --- | --- | -| `OLLAMA_URL` | `http://ollama:11434` | Local LLM runtime. | +| `OLLAMA_URL` | `http://ollama:11434` | Local LLM runtime. **Blank means there is none** (#962, ADR-0144) — a deliberate hosted-only deployment; see [no local runtime](#no-local-runtime-962-adr-0144). | | `LLM_DEFAULT_MODEL` | `llama3.2` | Model when a request names none. | | `LLM_FALLBACKS` | — | Comma-separated fallback chain (e.g. `claude/claude-3-5-sonnet-latest`). | | `LLM_KEEP_ALIVE` | `5m` | How long Ollama keeps a model loaded (ADR-0005). | diff --git a/infra/cd/reconcile.sh b/infra/cd/reconcile.sh index f9684a20..83742d82 100644 --- a/infra/cd/reconcile.sh +++ b/infra/cd/reconcile.sh @@ -69,10 +69,22 @@ fi # path re-adds a discovered override explicitly — `docker-compose.override.yml` is gitignored # precisely so an operator can keep one on the box, and dropping it here would be the same # class of silent revert this change exists to fix. +# 3. Local AI is ON by default, here as everywhere else (#962, ADR-0144). Ollama carries the +# `local-ai` profile so a hosted-only deployment can leave it out — but a profile is +# opt-in by construction, and a deploy script that silently stopped selecting it would +# take the local runtime down on the next reconcile of a box that never asked for that. +# So the profile is passed explicitly (`--profile` does not disable override discovery — +# only `-f` does, see 2), and turning it off is a deliberate `EPICURUS_LOCAL_AI=0`, +# paired with `OLLAMA_URL=` in .env or the core keeps probing a host that is gone. set -- +if [ "${EPICURUS_LOCAL_AI:-1}" = "0" ]; then + log "EPICURUS_LOCAL_AI=0 — reconciling WITHOUT the local AI runtime (hosted models only)." +else + set -- --profile local-ai +fi if [ -n "${DOCKER_GID}" ]; then log "DOCKER_GID is set — including the Docker-socket opt-in overlay." - set -- -f compose.yaml + set -- "$@" -f compose.yaml for override in \ compose.override.yaml compose.override.yml \ docker-compose.override.yaml docker-compose.override.yml; do diff --git a/infra/ci/k8s-smoke.sh b/infra/ci/k8s-smoke.sh index 4f909e6d..4d7a14f6 100644 --- a/infra/ci/k8s-smoke.sh +++ b/infra/ci/k8s-smoke.sh @@ -252,11 +252,23 @@ smoke_assert # this job already runs ~10 of its 20 minutes with 11 uncached builds inside. A changed # `core.extraEnv` value forces the same thing the tag would — a real pod-template change, # so core-app genuinely rolls — for the cost of one API call. -log "Upgrading the release in place (helm upgrade over a running install)" +# +# The upgrade also carries the **hosted-only** switch (#962, ADR-0144): `ollama.external.url` +# goes blank and both model defaults become hosted ids, which is exactly what an operator +# with no local runtime installs. Folded into this step rather than given a second boot +# because a second boot would not fit the 20-minute budget — and because it makes the +# upgrade a *real* config change instead of a timestamp, which is a better upgrade test +# than the one it replaces. Everything asserted before this point ran with a runtime +# configured (the KV-cache restart through the seam needs the workload to exist); what is +# asserted after it is the mode where there is none. +log "Upgrading the release in place, into the hosted-only mode (no local runtime)" helm upgrade "$RELEASE" "$CHART" \ --namespace "$NS" \ --values "$VALUES" \ --set "image.tag=$IMAGE_TAG" \ + --set "ollama.external.url=" \ + --set "core.llm.defaultModel=claude/claude-sonnet-4-6" \ + --set "core.memoryEmbedModel=gpt/text-embedding-3-small" \ --set-string "core.extraEnv.EPICURUS_SMOKE_UPGRADE=$(date -u +%s)" \ --wait --timeout 8m kc rollout status deployment/core-app --timeout=300s >/dev/null @@ -278,6 +290,36 @@ printf '%s' "$prov" | grep -oE '"alias":"claude"[^}]*' | grep -q '"configured":t || die "the provider key did not survive a helm upgrade (vault re-initialised, or a new app token?)" ok "state survived the upgrade: every module still registered, the stored secret still readable" +# ── a deployment with no local runtime at all (#962, ADR-0144) ──────────────── +# The mode the chart refused to render until now, and the one the core used to 500 in. +# `infra/ci/ollama-stub.yaml` is still applied above — it is what gives the seam's restart +# arm a real StatefulSet to patch (#919) — but the core no longer has a URL for it, which +# is precisely the deployment being asserted here. +log "Asserting the hosted-only mode (OLLAMA_URL is blank)" + +lr="$(http "http://core-app:8080/platform/v1/llm/local-runtime" || true)" +printf '%s' "$lr" | grep -q '"state":"absent"' \ + || die "the core does not report an absent local runtime on a hosted-only release: $lr" +printf '%s' "$lr" | grep -q '"url_configured":false' \ + || die "the core reports a configured URL on a hosted-only release: $lr" +ok "GET /platform/v1/llm/local-runtime reports absent" + +# The regression this issue exists for: the Models page polls this every 10 seconds, and +# it used to answer 500 every time. `-f` fails the curl on any non-2xx, so a 500 dies here. +models="$(http -f "http://core-app:8080/platform/v1/llm/models")" \ + || die "GET /platform/v1/llm/models did not return 2xx with no local runtime (#962)" +[ "$(printf '%s' "$models" | tr -d ' \n')" = "[]" ] \ + || die "the local model list is not empty with no local runtime: $models" +ok "GET /platform/v1/llm/models is 200 and empty, not a 500 every ten seconds" + +# And the local-only actions refuse with a reason rather than a bare 500. 409, because +# nothing is broken — the request is meaningless on this deployment. +pull_code="$(http -o /dev/null -w '%{http_code}' -X POST \ + "http://core-app:8080/platform/v1/llm/pull" \ + -H 'Content-Type: application/json' -d '{"model":"llama3.2"}' || true)" +[ "$pull_code" = "409" ] || die "a pull with no local runtime answered $pull_code (expected 409)" +ok "POST /platform/v1/llm/pull refuses with 409" + # ── the Kubernetes arm of the container-runtime seam (#891, ADR-0134) ────────── # Everything above is true of any deployment. This is the part that was mock-only # until now: in a pod there is no Docker daemon, so `CONTAINER_RUNTIME=auto` must diff --git a/infra/ci/ollama-stub.yaml b/infra/ci/ollama-stub.yaml index 67ce86bf..04edc663 100644 --- a/infra/ci/ollama-stub.yaml +++ b/infra/ci/ollama-stub.yaml @@ -1,5 +1,15 @@ # A stand-in for the Ollama workload, for the `k8s-smoke` gate only (#919). # +# #962 note. That issue named this stub as the proof we had papered over a missing +# deployment mode — "we wrote a stub instead of a mode" — and proposed retiring it once the +# mode existed. The mode now exists (ADR-0144) and the gate asserts it, in the upgrade phase +# (`ollama.external.url` goes blank and the core must answer `absent`). The stub stays +# because it is load-bearing for a *different* assertion: it is the only Ollama workload a +# kind run ever has, and without it `KubernetesController.restart_service` and the chart +# Role's `statefulsets` verb go straight back to mock-only — the #919 hole. The gate now +# proves both halves: with a runtime configured, a KV-cache change restarts this workload +# through the seam; with none, the core refuses every local action with a reason. +# # Why this exists. `infra/ci/values-ci.yaml` turns the chart's Ollama off — the image # is multi-gigabyte and its pod asks for 4Gi before it has done anything, and nothing # the smoke asserts needs a model. The cost of that was silent: with no Ollama diff --git a/infra/ci/smoke.sh b/infra/ci/smoke.sh index c2ab1996..f55d69f9 100644 --- a/infra/ci/smoke.sh +++ b/infra/ci/smoke.sh @@ -53,7 +53,11 @@ mkdir -p "$SMOKE_MOUNT_DIR" chmod 0777 "$SMOKE_MOUNT_DIR" printf 'seed\n' > "$SMOKE_MOUNT_DIR/seed.txt" BOOT_LOG="$(mktemp)" -DC="docker compose -f compose.yaml -f infra/ci/compose.ci.yaml --env-file $ENV_FILE" +# `--profile local-ai`: the Ollama services carry that profile since #962 (it is how a +# hosted-only deployment leaves them out). This gate boots the *default* stack — local AI on +# — and asserts things that need the workload to exist: the ollama-init one-shot's exit code, +# and the KV-cache change restarting it through the container seam. +DC="docker compose --profile local-ai -f compose.yaml -f infra/ci/compose.ci.yaml --env-file $ENV_FILE" CURL_IMG="curlimages/curl:8.11.1" DATA_PLANE="openbao postgres valkey nats qdrant minio minio-init" @@ -155,7 +159,7 @@ docker pull -q "$CURL_IMG" >/dev/null # Pre-flight: two fragments publishing the same host port is the #68 collision class. # The smoke itself clears ports (for isolation), so check the real compose instead. log "Pre-flight: checking for duplicate published host ports" -dupes="$(docker compose -f compose.yaml config 2>/dev/null | +dupes="$(docker compose --profile local-ai -f compose.yaml config 2>/dev/null | grep -oE 'published: "?[0-9]+' | grep -oE '[0-9]+' | sort | uniq -d | tr '\n' ' ')" [ -z "$dupes" ] || die "two services publish the same host port(s): $dupes — pick a unique one" ok "no duplicate published host ports" diff --git a/infra/ci/values-ci.yaml b/infra/ci/values-ci.yaml index 92868014..13f1fac0 100644 --- a/infra/ci/values-ci.yaml +++ b/infra/ci/values-ci.yaml @@ -21,6 +21,13 @@ # Compose gate, which boots Ollama but never pulls weights). Pointing the endpoint # at loopback inside the core pod means an accidental inference call is refused in # microseconds instead of hanging the gate on a DNS or connect timeout. +# +# A URL is set here *on purpose*, even though nothing serves it: the install phase has to run +# with a local runtime **configured**, because that is what the shared KV-cache assertion +# needs — it restarts the Ollama workload (the stand-in, infra/ci/ollama-stub.yaml) through +# the container seam. The third state — no local runtime at all (#962, ADR-0144) — is +# asserted in the gate's **upgrade** phase, which blanks this value and re-points both model +# defaults at hosted ids. One boot covers both, and neither ends up proven only in a unit test. ollama: enabled: false external: diff --git a/infra/k8s/epicurus/Chart.yaml b/infra/k8s/epicurus/Chart.yaml index 692b796d..05b9c918 100644 --- a/infra/k8s/epicurus/Chart.yaml +++ b/infra/k8s/epicurus/Chart.yaml @@ -7,7 +7,7 @@ type: application # Chart version — the packaging's own SemVer, bumped when the templates change # (ADR-0017 treats the chart as its own component). -version: 0.1.2 +version: 0.2.0 # Placeholder. `appVersion` is the image tag every service defaults to, and the # release pipeline overrides it at package time (`helm package --app-version `, diff --git a/infra/k8s/epicurus/templates/NOTES.txt b/infra/k8s/epicurus/templates/NOTES.txt index c79af003..fdf82ad1 100644 --- a/infra/k8s/epicurus/templates/NOTES.txt +++ b/infra/k8s/epicurus/templates/NOTES.txt @@ -60,8 +60,15 @@ Images: tag {{ default .Chart.AppVersion .Values.image.tag }} from {{ .Values.im Ollama starts with an EMPTY model store — models are never baked into the image. The core pulls the default chat and embedding models in the background - on first boot (LLM_BOOTSTRAP_MODELS={{ .Values.core.llm.bootstrapModels }}); on CPU that + on first boot (LLM_BOOTSTRAP_MODELS={{ include "epicurus.bootstrapModels" $ }}); on CPU that takes a while and chat 404s until it finishes. +{{- else if not (include "epicurus.localRuntimeEnabled" $) }} + + This release runs NO LOCAL LLM RUNTIME (#962): chat and embeddings both go to + hosted providers. Add each provider's API key on the Models page before the + first turn — nothing local will answer. The local half of that page is shown + collapsed rather than broken, and every local-runtime action (pull, delete, + unload, the KV-cache setting) refuses with 409 instead of hanging. {{- end }} {{- if and (not .Values.minio.enabled) (not .Values.minio.external.url) (and .Values.modules.storage .Values.modules.storage.enabled) }} diff --git a/infra/k8s/epicurus/templates/_helpers.tpl b/infra/k8s/epicurus/templates/_helpers.tpl index 025eec70..1ca20d3c 100644 --- a/infra/k8s/epicurus/templates/_helpers.tpl +++ b/infra/k8s/epicurus/templates/_helpers.tpl @@ -110,11 +110,73 @@ http://openbao:8200 {{- end -}} {{- end -}} +{{/* +The local LLM runtime's URL — **and the empty string is a valid answer** (#962, ADR-0144). + +Three deployments, not two: the chart's own Ollama, an external one, and *none at all* — +hosted chat and hosted embeddings, no local runtime anywhere. The third was unreachable +because this helper was written by copying `epicurus.qdrantUrl` / `epicurus.openbaoUrl`, +components the core genuinely cannot run without, where `required` is right. Ollama is not +one of them, and `epicurus.minioUrl` three definitions down already shows the other shape. +An empty `OLLAMA_URL` is how the core is told there is no local runtime, and it then refuses +every local-runtime action with a reason instead of timing out against a placeholder. +*/}} {{- define "epicurus.ollamaUrl" -}} {{- if .Values.ollama.enabled -}} http://ollama:11434 {{- else -}} -{{- required "ollama.external.url is required when ollama.enabled is false" .Values.ollama.external.url -}} +{{- default "" .Values.ollama.external.url -}} +{{- end -}} +{{- end -}} + +{{/* +Whether this deployment has a local LLM runtime at all — "true" or "" (Helm's falsy string). +The one place the question is answered, so the env, the bootstrap default and the guard below +cannot disagree about what "hosted-only" means. +*/}} +{{- define "epicurus.localRuntimeEnabled" -}} +{{- if include "epicurus.ollamaUrl" . -}}true{{- end -}} +{{- end -}} + +{{/* +`LLM_BOOTSTRAP_MODELS` for this deployment. + +With no local runtime there is nothing to pull into, so the chart's default (`auto`) becomes +blank on its own rather than making every hosted-only operator discover the setting. An +operator who set the value themselves is left alone — an explicit list is a stated pin, and +the core answers a pull with a clean refusal now, not a three-minute poll. +*/}} +{{- define "epicurus.bootstrapModels" -}} +{{- if and (not (include "epicurus.localRuntimeEnabled" .)) (eq .Values.core.llm.bootstrapModels "auto") -}} +{{- else -}} +{{- .Values.core.llm.bootstrapModels -}} +{{- end -}} +{{- end -}} + +{{/* +Refuse to render the half-working stack (#962, ADR-0144). + +With no local runtime, a **bare** model name — `llama3.2`, `nomic-embed-text`, the chart's own +defaults — routes to a runtime that does not exist. Chat would work through whatever hosted +provider the operator configured while memory recall and every module index failed at call +time. That is exactly the deployment we refuse to ship elsewhere, so it fails at render time +with the fix in the message. The old guard (`required` on `ollama.external.url`) refused a +*legitimate* deployment; this one refuses a broken one. + +A hosted id is `/`, mirroring `providers.is_hosted` in the core — +`local/…` is deliberately not on the list, and neither is an unknown prefix, because both +route to the local runtime there too. +*/}} +{{- define "epicurus.assertHostedOnlyModels" -}} +{{- if not (include "epicurus.localRuntimeEnabled" .) -}} +{{- $hosted := list "claude" "gpt" "grok" "deepseek" "gemini" "openrouter" "custom" -}} +{{- $checks := dict "core.llm.defaultModel" .Values.core.llm.defaultModel "core.memoryEmbedModel" .Values.core.memoryEmbedModel -}} +{{- range $key, $model := $checks -}} +{{- $alias := (splitList "/" ($model | toString)) | first -}} +{{- if not (and (contains "/" ($model | toString)) (has $alias $hosted)) -}} +{{- fail (printf "this release has no local LLM runtime (ollama.enabled is false and ollama.external.url is blank), but %s is %q — a local model name, which nothing here can run. Set %s to a hosted alias, e.g. core.memoryEmbedModel=gpt/text-embedding-3-small and core.llm.defaultModel=claude/claude-sonnet-4-6, or enable Ollama." $key ($model | toString) $key) -}} +{{- end -}} +{{- end -}} {{- end -}} {{- end -}} diff --git a/infra/k8s/epicurus/templates/core-app.yaml b/infra/k8s/epicurus/templates/core-app.yaml index 06c3a82a..69beeee9 100644 --- a/infra/k8s/epicurus/templates/core-app.yaml +++ b/infra/k8s/epicurus/templates/core-app.yaml @@ -1,3 +1,7 @@ +{{- /* A hosted-only release must not render with a local model name nothing here can run + (#962, ADR-0144) — see `epicurus.assertHostedOnlyModels`. Evaluated first, so the + failure names the fix before anything downstream can fail for a vaguer reason. */}} +{{- include "epicurus.assertHostedOnlyModels" . -}} {{- $files := .Values.core.persistence -}} {{- $localFiles := eq .Values.core.filesBackend "local" -}} {{- $baoTokenSecret := include "epicurus.openbaoTokenSecret" . -}} @@ -196,7 +200,7 @@ spec: - name: LLM_NUM_CTX value: {{ .Values.core.llm.numCtx | quote }} - name: LLM_BOOTSTRAP_MODELS - value: {{ .Values.core.llm.bootstrapModels | quote }} + value: {{ include "epicurus.bootstrapModels" $ | quote }} - name: OAUTH_REDIRECT_BASE_URL value: {{ include "epicurus.oauthRedirectBaseUrl" $ | quote }} - name: OAUTH_STATE_SECRET diff --git a/infra/k8s/epicurus/values.yaml b/infra/k8s/epicurus/values.yaml index 99984b69..b88afc12 100644 --- a/infra/k8s/epicurus/values.yaml +++ b/infra/k8s/epicurus/values.yaml @@ -129,6 +129,14 @@ core: # Use a ServiceAccount you manage instead of the chart's. serviceAccountName: "" # LLM gateway (#114). Blank = the code default; every knob is env-only. + # + # On a **hosted-only** release — `ollama.enabled: false` with a blank + # `ollama.external.url` (#962, ADR-0144) — `defaultModel` and `memoryEmbedModel` must both + # name a *hosted* model, `/`: e.g. `claude/claude-sonnet-4-6` and + # `gpt/text-embedding-3-small`. A bare name routes to the local runtime, so leaving the + # defaults below in place there gives a stack where chat works through the hosted provider + # and every embedding (memory recall, each module's index) fails at call time. The chart + # refuses to render that, and the failure names the fix. llm: defaultModel: llama3.2 keepAlive: 5m @@ -139,7 +147,9 @@ core: topP: "" numCtx: "" # auto = pull the effective chat + embedding defaults on first boot; - # "" = never pull; "a,b" = pull exactly these. + # "" = never pull; "a,b" = pull exactly these. On a hosted-only release `auto` becomes + # blank on its own — there is nothing to pull into — while an explicit list is left as + # the operator wrote it. bootstrapModels: auto memoryEmbedModel: nomic-embed-text oauth: @@ -485,6 +495,16 @@ minio: # The local LLM runtime. A StatefulSet (component `ollama`) so the # container-runtime seam can rollout-restart it by that label (#891). +# +# Three deployments, not two (#962, ADR-0144): +# enabled: true — the chart runs Ollama (the default); +# enabled: false + external.url: http://… — an Ollama you run elsewhere; +# enabled: false + external.url: "" — **no local runtime at all**: hosted chat and +# hosted embeddings only. `OLLAMA_URL` renders +# empty, the core refuses every local-runtime +# action with a reason instead of hanging, and +# `core.llm.defaultModel` / `core.memoryEmbedModel` +# must name hosted models (the chart checks). ollama: enabled: true image: @@ -515,6 +535,8 @@ ollama: resourceName: nvidia.com/gpu runtimeClassName: "" external: + # An Ollama you run elsewhere, when `enabled` is false. **Blank is meaningful**: it says + # this deployment has no local runtime at all, rather than being a missing setting (#962). url: "" nodeSelector: {} tolerations: [] diff --git a/infra/ollama/compose.yaml b/infra/ollama/compose.yaml index aac3f4ba..4b4927a3 100644 --- a/infra/ollama/compose.yaml +++ b/infra/ollama/compose.yaml @@ -8,6 +8,16 @@ # # Internal-only: the core reaches it at http://ollama:11434 on the epicurus network; # it is not published to the host (private-by-default — the core is the front door). +# +# ── Leaving it out (#962, ADR-0144) ───────────────────────────────────────────── +# Both services carry the `local-ai` profile, so a deployment can run *no* local runtime — +# hosted chat and hosted embeddings, nothing local to run a model on. The profile is ON +# everywhere the stack is actually started, so the default install is unchanged: +# `.env.example` ships `COMPOSE_PROFILES=local-ai`, every `task *-up` passes +# `--profile local-ai`, and `infra/cd/reconcile.sh` passes it unless `EPICURUS_LOCAL_AI=0`. +# A hosted-only stack is `COMPOSE_PROFILES=` **and** `OLLAMA_URL=`: the blank URL is what +# tells the core there is no local runtime, and without it the core would keep probing a +# hostname that no longer resolves instead of refusing with a reason. # Bounded json-file logging (#462) — see infra/compose/docker-compose.yml for the rationale. x-logging: &default-logging @@ -26,6 +36,9 @@ services: # the root — no subdirs needed. Mirrors qdrant-init, the core image entrypoint, and the OpenBao chown. ollama-init: image: alpine:3.21 + # Same profile as the server it prepares: with no local runtime there is no volume to + # chown and no reason to run a one-shot for it (#962). + profiles: [local-ai] restart: "no" user: root volumes: @@ -44,6 +57,9 @@ services: ollama: image: ollama/ollama:0.30.7 + # On by default (see the header): the local runtime is the local-first default, and the + # profile exists so an operator can *opt out* of it, never so they must opt in (#962). + profiles: [local-ai] restart: unless-stopped # Wait for the chown one-shot so the volume the core later writes to is owned by uid 10001 # (#392). This is ordering only — nothing here races the core's write, which is lazy: it diff --git a/services/core-app/compose.yaml b/services/core-app/compose.yaml index bb088bc3..1a20707f 100644 --- a/services/core-app/compose.yaml +++ b/services/core-app/compose.yaml @@ -20,7 +20,12 @@ services: NATS_URL: nats://nats:4222 NATS_USER: core NATS_PASSWORD: ${NATS_CORE_PASSWORD:-epicurus-dev} - OLLAMA_URL: http://ollama:11434 + # The local LLM runtime. **Blank means there is none** (#962, ADR-0144): a hosted-only + # stack sets `OLLAMA_URL=` and `COMPOSE_PROFILES=` (which leaves the `local-ai` profile, + # and so the ollama service itself, out of the stack). Single-dash `-`, not `:-`: `:-` + # would substitute the default back in for a deliberately *empty* value, which is + # precisely the setting an operator uses to say "there is no local runtime here". + OLLAMA_URL: ${OLLAMA_URL-http://ollama:11434} # ── LLM gateway tuning (all optional; blank/unset = provider default) ── # Set any of these and restart — no code edit needed (issue #114). LLM_DEFAULT_MODEL: ${LLM_DEFAULT_MODEL:-llama3.2} @@ -82,6 +87,10 @@ services: condition: service_started ollama: condition: service_started + # Not required (#962, ADR-0144): a hosted-only stack leaves the `local-ai` profile + # out, and without this the whole `up` would fail on a dependency that is absent on + # purpose. With local AI on (the default) the ordering is unchanged. + required: false openbao: condition: service_healthy postgres: diff --git a/tests/test_no_local_runtime.py b/tests/test_no_local_runtime.py new file mode 100644 index 00000000..bf8049fb --- /dev/null +++ b/tests/test_no_local_runtime.py @@ -0,0 +1,238 @@ +"""A deployment may run no local LLM runtime at all — on either runtime (#962, ADR-0144). + +Three states, not two: the stack's own Ollama, an external one, and *none*. The third was +unreachable on both runtimes — the Helm chart `required`d an external URL the moment Ollama +was disabled, and Compose included the fragment unconditionally with `core-app` hard-depending +on the service. These are the static and render-time halves of the fix: + +* **Compose** — parsed from the fragments directly, in the style of ``test_compose_ports.py``: + the profile that lets Ollama be left out, the `required: false` that stops its absence + failing the whole `up`, and the env interpolation that carries a *deliberately blank* + ``OLLAMA_URL`` through to the core. Plus the thing that is easy to get wrong and impossible + to see: that turning the runtime into a profile did not turn local AI off for everyone who + was not asking for that. +* **Chart** — real ``helm template`` renders (no cluster), because a guard nobody renders is a + guard that has quietly stopped guarding. Skipped, not silently passed, without ``helm``. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path +from typing import Any + +import pytest +import yaml + +REPO = Path(__file__).resolve().parents[1] +CHART = REPO / "infra" / "k8s" / "epicurus" +OLLAMA_FRAGMENT = REPO / "infra" / "ollama" / "compose.yaml" +CORE_FRAGMENT = REPO / "services" / "core-app" / "compose.yaml" + +LOCAL_AI_PROFILE = "local-ai" + +# Every command that starts the stack must select the profile, or the operator who typed it +# silently loses local AI. The Taskfile tasks are listed by name so a *new* start task that +# forgets it is a conversation at review, not a surprise on someone's box. +_START_TASKS = ("up", "obs-up", "docker-socket-up", "external-mounts-up") + + +def _fragment(path: Path) -> dict[str, Any]: + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + assert isinstance(loaded, dict) + return loaded + + +# ── Compose ────────────────────────────────────────────────────────────────────── + + +def test_the_local_runtime_can_be_left_out_of_the_stack() -> None: + services = _fragment(OLLAMA_FRAGMENT)["services"] + for name in ("ollama", "ollama-init"): + assert services[name].get("profiles") == [LOCAL_AI_PROFILE], ( + f"{name} must carry the {LOCAL_AI_PROFILE!r} profile — it is how a hosted-only " + "deployment leaves the local runtime out (#962)" + ) + + +def test_the_core_does_not_hard_depend_on_the_local_runtime() -> None: + """Without this the whole `up` fails on a service that is absent on purpose.""" + depends = _fragment(CORE_FRAGMENT)["services"]["core-app"]["depends_on"] + assert depends["ollama"] == {"condition": "service_started", "required": False} + + +def test_a_blank_ollama_url_survives_interpolation() -> None: + """``${OLLAMA_URL-...}``, not ``${OLLAMA_URL:-...}``. + + The difference is the whole feature: ``:-`` substitutes the default back in for an + *empty* value, and empty is exactly what an operator sets to say "there is no local + runtime here". A one-character slip turns the mode off with no other symptom. + """ + env = _fragment(CORE_FRAGMENT)["services"]["core-app"]["environment"] + assert env["OLLAMA_URL"] == "${OLLAMA_URL-http://ollama:11434}" + + +def test_every_start_path_still_turns_local_ai_on() -> None: + """A profile is opt-in by construction; the default install must not have changed. + + This is the regression guard for the way this feature could hurt people who did not ask + for it: put a profile on Ollama and every existing `task up`, every deploy reconcile and + every documented `.env` stops starting the local runtime. So each of those paths selects + the profile explicitly, and this test is what keeps it that way. + """ + taskfile = yaml.safe_load((REPO / "Taskfile.yml").read_text(encoding="utf-8")) + for name in _START_TASKS: + cmds = " ".join(str(c) for c in taskfile["tasks"][name]["cmds"]) + assert f"--profile {LOCAL_AI_PROFILE}" in cmds, ( + f"task {name} no longer starts the local AI runtime — a profile is opt-in, so " + "every start path has to select it or the default install silently changes" + ) + + reconcile = (REPO / "infra" / "cd" / "reconcile.sh").read_text(encoding="utf-8") + assert f"--profile {LOCAL_AI_PROFILE}" in reconcile, ( + "infra/cd/reconcile.sh would take the local runtime down on the next deploy of a " + "box that never asked for a hosted-only stack" + ) + assert "EPICURUS_LOCAL_AI" in reconcile, "the deploy path has no way to opt out" + + env_example = (REPO / ".env.example").read_text(encoding="utf-8") + assert f"COMPOSE_PROFILES={LOCAL_AI_PROFILE}" in env_example, ( + ".env.example must ship the profile: it is what selects local AI for a bare " + "`docker compose up -d`" + ) + + +def test_the_compose_smoke_gate_boots_the_local_runtime() -> None: + """`runtime-smoke` asserts ollama-init's exit code and a KV-cache restart — both need it.""" + smoke = (REPO / "infra" / "ci" / "smoke.sh").read_text(encoding="utf-8") + assert f"--profile {LOCAL_AI_PROFILE}" in smoke + + +def test_the_kubernetes_gate_asserts_the_hosted_only_mode() -> None: + """The mode is hosted-only-shaped; only a boot can say the core answers instead of 500ing.""" + gate = (REPO / "infra" / "ci" / "k8s-smoke.sh").read_text(encoding="utf-8") + assert "/platform/v1/llm/local-runtime" in gate, ( + "k8s-smoke no longer asserts the local-runtime state endpoint (#962)" + ) + assert '"state":"absent"' in gate + assert "ollama.external.url=" in gate, ( + "k8s-smoke never reaches the hosted-only mode — the upgrade phase is what blanks the " + "URL, and without it the assertions above are made against a configured runtime" + ) + + +# ── Chart ──────────────────────────────────────────────────────────────────────── + +pytestmark_helm = pytest.mark.skipif( + shutil.which("helm") is None, reason="these assertions render the chart with helm" +) + + +def _render(*sets: str) -> subprocess.CompletedProcess[str]: + args = ["helm", "template", "epicurus", str(CHART)] + for pair in sets: + args += ["--set", pair] + return subprocess.run(args, capture_output=True, text=True, timeout=120) + + +def _env_of(rendered: str, container: str = "core-app") -> dict[str, str]: + """The core-app container's plain-value env, keyed by name.""" + for doc in yaml.safe_load_all(rendered): + if not isinstance(doc, dict) or doc.get("kind") != "Deployment": + continue + if doc["metadata"]["name"] != container: + continue + spec = doc["spec"]["template"]["spec"]["containers"][0] + return {e["name"]: e.get("value", "") for e in spec["env"]} + raise AssertionError(f"no {container} Deployment in the render") + + +@pytestmark_helm +def test_the_chart_renders_a_deployment_with_no_local_runtime() -> None: + """It did not, before: `ollama.enabled: false` `required`d an external URL (#962).""" + result = _render( + "ollama.enabled=false", + "core.llm.defaultModel=claude/claude-sonnet-4-6", + "core.memoryEmbedModel=gpt/text-embedding-3-small", + ) + assert result.returncode == 0, result.stderr + env = _env_of(result.stdout) + assert env["OLLAMA_URL"] == "" + # Nothing to pull into, so the chart's own default stops asking the core to try. + assert env["LLM_BOOTSTRAP_MODELS"] == "" + assert "app.kubernetes.io/component: ollama" not in result.stdout + + +@pytestmark_helm +def test_an_explicit_bootstrap_list_is_left_alone() -> None: + """Only the chart's own `auto` is rewritten — an operator's list is a stated pin.""" + result = _render( + "ollama.enabled=false", + "core.llm.defaultModel=claude/claude-sonnet-4-6", + "core.memoryEmbedModel=gpt/text-embedding-3-small", + "core.llm.bootstrapModels=llama3.2", + ) + assert result.returncode == 0, result.stderr + assert _env_of(result.stdout)["LLM_BOOTSTRAP_MODELS"] == "llama3.2" + + +@pytestmark_helm +def test_an_external_runtime_still_renders_its_url() -> None: + result = _render("ollama.enabled=false", "ollama.external.url=http://gpu-box:11434") + assert result.returncode == 0, result.stderr + assert _env_of(result.stdout)["OLLAMA_URL"] == "http://gpu-box:11434" + + +@pytestmark_helm +def test_the_default_release_is_unchanged() -> None: + result = _render() + assert result.returncode == 0, result.stderr + env = _env_of(result.stdout) + assert env["OLLAMA_URL"] == "http://ollama:11434" + assert env["LLM_BOOTSTRAP_MODELS"] == "auto" + + +@pytestmark_helm +@pytest.mark.parametrize("key", ["core.memoryEmbedModel", "core.llm.defaultModel"]) +def test_a_hosted_only_release_with_a_local_model_name_is_refused(key: str) -> None: + """The guard that replaces the old one — and the reason the old one was wrong. + + `required: ollama.external.url` refused a *legitimate* deployment. This refuses a broken + one: with no local runtime, a bare model name routes to a runtime that does not exist, so + chat works through the hosted provider while memory recall and every module index fail at + call time. That is the half-working stack we refuse to ship elsewhere. + """ + hosted = { + "core.memoryEmbedModel": "gpt/text-embedding-3-small", + "core.llm.defaultModel": "claude/claude-sonnet-4-6", + } + sets = ["ollama.enabled=false"] + sets += [f"{k}={v}" for k, v in hosted.items() if k != key] + sets.append(f"{key}=a-local-model-name") + + result = _render(*sets) + assert result.returncode != 0, "the chart rendered a half-working hosted-only release" + assert "no local LLM runtime" in result.stderr + assert key in result.stderr, "the refusal must name the key the operator has to change" + assert "gpt/text-embedding-3-small" in result.stderr, "and an example of what to set it to" + + +@pytestmark_helm +def test_an_explicitly_local_prefixed_model_is_refused_too() -> None: + """`local/llama3.2` is as unrunnable here as `llama3.2` — mirrors `providers.is_hosted`.""" + result = _render( + "ollama.enabled=false", + "core.llm.defaultModel=local/llama3.2", + "core.memoryEmbedModel=gpt/text-embedding-3-small", + ) + assert result.returncode != 0 + assert "no local LLM runtime" in result.stderr + + +@pytestmark_helm +def test_the_guard_does_not_fire_when_a_runtime_exists() -> None: + """Bare names are the *right* answer with a runtime — the chart's own defaults are bare.""" + assert _render().returncode == 0 + external = _render("ollama.enabled=false", "ollama.external.url=http://gpu-box:11434") + assert external.returncode == 0 From 1615979d683bd8da5e24799dc490f23cffea492a Mon Sep 17 00:00:00 2001 From: NikolaI Baakh Date: Sat, 19 Sep 2026 11:11:01 +0000 Subject: [PATCH 3/4] fix(infra): hosted-only is an opt-out overlay, not a profile on Ollama MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first cut expressed "no local AI runtime" as `profiles: [local-ai]` on the `ollama` services, kept on by making every start path select the profile. That regressed the one install path the repo actually publishes: README.md and docs/user/installation.md both document `git clone` -> `cd epicurus` -> `docker compose up -d`, and neither mentions an `.env` at that point. A profile is opt-in by construction, so a fresh clone of a public repo would have come up with no local runtime *and* the shipped `llama3.2` / `nomic-embed-text` defaults still pointing at it — the exact half-working stack the new chart guard exists to refuse, delivered by default to the person least able to diagnose it. An existing operator who pulled and re-upped without an `.env` lost Ollama too. `.env.example` carrying the profile does not cover that: nothing copies it into place and no doc tells anyone to. So the mode moves to `infra/ollama/compose.hosted-only.yaml`, layered over the stack — the idiom the repo already uses for `compose.docker-socket.yaml` and `compose.external-mounts.yaml`. It does both halves in one step (removes the two services, blanks `OLLAMA_URL`), because removing the container alone leaves the core probing a host that is gone, which is `unreachable`, not `absent`. `task hosted-only-up` / `hosted-only-down` drive it, and `EPICURUS_HOSTED_ONLY=1` applies it from `infra/cd/reconcile.sh` — whose default path passes no `-f` at all, exactly as before. `infra/ollama/compose.yaml` now differs from `main` by a comment block only, and `infra/ci/smoke.sh` is byte-identical again. `compose.yaml`'s header says what is true — the default `up` brings up everything, opt-ins are overlays — and names the three. Pinned, both directions: `tests/test_no_local_runtime.py` asserts no profile gates the Ollama services and no start task needs a flag, plus that the overlay removes them and blanks the URL; `compose-validate` proves the same against a real `docker compose config` on both shapes. Also folded in: `GET /platform/v1/llm/local-runtime` answering 404 (an older core) or anything unusable **reads as `ok`** — a client keeps the pre-#962 behaviour rather than collapsing every local control against a core that is serving one; the state steers what a surface draws, never what the core does. Recorded in the platform-API reference and the core-app page for the web lane. Part of #962. --- .env.example | 42 +++---- .github/workflows/ci.yml | 46 ++++--- CHANGELOG.md | 15 ++- Taskfile.yml | 19 +-- compose.yaml | 11 +- docs/infrastructure/index.md | 53 ++++---- docs/infrastructure/kubernetes.md | 2 +- docs/infrastructure/startup-and-recovery.md | 9 +- docs/reference/platform-api.md | 8 ++ docs/services/core-app.md | 2 +- infra/cd/reconcile.sh | 34 +++--- infra/ci/smoke.sh | 8 +- infra/ollama/compose.hosted-only.yaml | 60 +++++++++ infra/ollama/compose.yaml | 23 ++-- services/core-app/compose.yaml | 17 +-- tests/test_no_local_runtime.py | 129 +++++++++++++------- 16 files changed, 311 insertions(+), 167 deletions(-) create mode 100644 infra/ollama/compose.hosted-only.yaml diff --git a/.env.example b/.env.example index 91d39dca..af91c640 100644 --- a/.env.example +++ b/.env.example @@ -7,27 +7,27 @@ # Environment name: local | staging | production APP_ENV=local -# ── Local AI: on by default, and optional (#962) ───────────────────────────── -# The local LLM runtime (Ollama) carries the `local-ai` compose profile, so a -# deployment can run without one: hosted chat and hosted embeddings, nothing -# local. It is ON by default — this line is what selects it for a bare -# `docker compose up -d`. (`task up` / `task obs-up` / `infra/cd/reconcile.sh` -# pass `--profile local-ai` themselves, so they are correct either way.) -COMPOSE_PROFILES=local-ai - -# For a HOSTED-ONLY stack, all three of these, not just the first: -# 1. COMPOSE_PROFILES= — leave Ollama out of the stack entirely -# 2. OLLAMA_URL= — tell the core there is no local runtime. Blank is a -# deliberate value here, not an omission; without it -# the core keeps probing a host that is not there. -# 3. hosted model defaults — a bare name routes to the local runtime, so: -# LLM_DEFAULT_MODEL=claude/claude-sonnet-4-6 -# MEMORY_EMBED_MODEL=gpt/text-embedding-3-small -# `task hosted-only-up` does 1 and 2 for you. With no local runtime the core -# refuses pull / delete / unload / the KV-cache setting with 409 and a reason, -# the Models page collapses its local half, and chat readiness reports the model -# as n/a rather than "warming" forever. -# OLLAMA_URL=http://ollama:11434 +# ── Running without a local AI runtime (#962) ──────────────────────────────── +# The local LLM runtime (Ollama) is ON by default and nothing here changes that: +# `docker compose up -d` from a fresh clone starts it, with or without this file. +# +# To run HOSTED-ONLY — hosted chat and hosted embeddings, no Ollama at all — use +# the opt-out overlay, which removes the container *and* blanks OLLAMA_URL (the +# blank is what tells the core there is no local runtime rather than leaving it +# probing a host that is gone): +# +# task hosted-only-up +# # == docker compose -f compose.yaml -f infra/ollama/compose.hosted-only.yaml up -d +# +# Then set both model defaults to hosted ids below — a bare name routes to the +# local runtime, so leaving the shipped defaults gives working chat and a failing +# embed on every memory recall and module index — and add each provider's API key +# on the Models page. With no local runtime the core refuses pull / delete / +# unload / the KV-cache setting with 409 and a reason, the Models page collapses +# its local half, and chat readiness reports the model as n/a rather than +# "warming" forever. +# LLM_DEFAULT_MODEL=claude/claude-sonnet-4-6 +# MEMORY_EMBED_MODEL=gpt/text-embedding-3-small # ── Image tag pinning (#56) ────────────────────────────────────────────────── # All service compose fragments default to `${EPICURUS_VERSION:-latest}`. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72bd8594..c633708d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,27 +158,41 @@ jobs: - name: Validate the assembled compose stack (observability profile) run: docker compose -f compose.yaml --profile observability config -q - # The default stack, local AI on — the shape every operator installs and the one - # `runtime-smoke` boots. Ollama carries `profiles: [local-ai]` since #962 so a - # hosted-only deployment can leave it out; this proves the on path still resolves. - - name: Validate the assembled compose stack (local-ai profile) - run: docker compose -f compose.yaml --profile local-ai config -q - - # And the hosted-only shape (#962, ADR-0144): no `local-ai` profile and a blank - # OLLAMA_URL. The assertion is not just that it parses — it is that core-app's - # dependency on the absent ollama service is `required: false`, which is what makes - # the whole `up` succeed rather than fail on a service that is missing on purpose. - - name: Validate the assembled compose stack (hosted-only — no local runtime) + # The documented install — `git clone` && `docker compose up -d`, no `.env`, no flags + # (README.md, docs/user/installation.md) — must keep starting the local AI runtime. + # #962 made "no local runtime" a supported mode, and the first cut expressed it as a + # compose profile on `ollama`: profiles are opt-in, so that silently removed Ollama + # from every fresh clone while the shipped `llama3.2` / `nomic-embed-text` defaults + # still pointed at it. This step is the guard against that being reintroduced. + - name: Validate the assembled compose stack (default install starts the local runtime) run: | - OLLAMA_URL= docker compose -f compose.yaml config -q - rendered="$(OLLAMA_URL= docker compose -f compose.yaml config)" + rendered="$(docker compose -f compose.yaml config)" + for svc in ollama ollama-init; do + if ! printf '%s' "$rendered" | grep -qE "^ ${svc}:"; then + echo "the default stack no longer starts '${svc}' — see #962"; exit 1 + fi + done + if ! printf '%s' "$rendered" | grep -q 'OLLAMA_URL: http://ollama:11434'; then + echo "the default stack does not point the core at the local runtime"; exit 1 + fi + echo "the documented install still brings up the local AI runtime" + + # And the opt-out overlay (#962, ADR-0144): no Ollama, blank OLLAMA_URL. The assertion + # is not just that it parses — it is that core-app's dependency on the now-absent + # ollama service is `required: false`, which is what lets the whole `up` succeed on a + # service that is missing on purpose. + - name: Validate the assembled compose stack (hosted-only overlay — no local runtime) + run: | + set -- -f compose.yaml -f infra/ollama/compose.hosted-only.yaml + docker compose "$@" config -q + rendered="$(docker compose "$@" config)" if ! printf '%s' "$rendered" | grep -q 'OLLAMA_URL: ""'; then - echo "a blank OLLAMA_URL did not survive interpolation"; exit 1 + echo "the overlay did not blank OLLAMA_URL"; exit 1 fi if printf '%s' "$rendered" | grep -qE '^ ollama:'; then - echo "ollama is still selected without the local-ai profile"; exit 1 + echo "the overlay did not remove the ollama service"; exit 1 fi - echo "hosted-only stack resolves with no local runtime" + echo "the hosted-only overlay resolves with no local runtime" # The Helm chart gets the same treatment the compose stack does: rendered, then # checked. `helm lint` catches template errors, `helm template` proves the branches diff --git a/CHANGELOG.md b/CHANGELOG.md index 14e80854..4c54648b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,12 +33,15 @@ images to GHCR. On **Kubernetes** the chart renders an empty `OLLAMA_URL`, blanks `LLM_BOOTSTRAP_MODELS` on its own, and gains the guard actually worth having — it refuses to render a runtime-less release whose chat or embedding default is still a bare local name, naming the hosted alias to set. On - **Compose** Ollama moves behind a `local-ai` profile that every start path (`task up`, - `task obs-up`, `infra/cd/reconcile.sh`, `.env.example`) selects explicitly, so the default - install is unchanged and `task hosted-only-up` is the opt-out. Both gates cover it: - `compose-validate` resolves the hosted-only stack, `chart-validate` renders it *and* proves the - guard refuses the half-working one, and `k8s-smoke` upgrades a live release into the mode and - asserts the core answers. `core-app` 0.128.0→0.129.0 (MINOR), chart 0.1.2→0.2.0 (MINOR). + **Compose** it is an opt-out overlay — `task hosted-only-up` — which removes the Ollama + services *and* blanks `OLLAMA_URL` in one step, the same idiom as the Docker-socket and + external-mount opt-ins; the documented install (`git clone`, `docker compose up -d`, no + `.env`) is byte-identical and still starts the local runtime, which a compose profile on + those services would have quietly stopped doing. Three gates cover it: `compose-validate` + proves both the default stack and the overlay resolve, `chart-validate` renders the + hosted-only release *and* proves the guard refuses the half-working one, and `k8s-smoke` + upgrades a live release into the mode and asserts the core answers. `core-app` + 0.128.0→0.129.0 (MINOR), chart 0.1.2→0.2.0 (MINOR). - **The bound on a turn is the operator's; runaway is caught by behaviour** (#925) — the **Agent cycles** setting stopped at 12, and the route enforced it *silently*: type 40 and 12 was stored. A genuinely long task — search → read → read → summarize → write — ran out of rounds and diff --git a/Taskfile.yml b/Taskfile.yml index bfef0762..6d76d9e2 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -62,14 +62,19 @@ tasks: - uv run python scripts/migrate.py check {{.CLI_ARGS}} up: - desc: Start the full stack (data plane + enabled modules + the local AI runtime; Docker control is on by default through docker-proxy-core, observability is opt-in — see `obs-up` / `docker-socket-up` / `hosted-only-up`) + desc: Start the full stack (data plane + enabled modules; Docker control is on by default through docker-proxy-core, observability is opt-in — see `obs-up` / `docker-socket-up`, and `hosted-only-up` to run without the local AI runtime) cmds: - - docker compose --profile local-ai up -d + - docker compose up -d hosted-only-up: - desc: 'Start the stack with NO local LLM runtime (#962) — no Ollama container, and OLLAMA_URL blank so the core says so instead of probing a host that is not there. Set LLM_DEFAULT_MODEL + MEMORY_EMBED_MODEL to hosted ids first, or chat and embedding both fail at call time' + desc: 'Start the stack with NO local LLM runtime (#962) — no Ollama container, and OLLAMA_URL blank so the core says so rather than probing a host that is not there. Set LLM_DEFAULT_MODEL + MEMORY_EMBED_MODEL to hosted ids in .env first, or chat and every embedding fail at call time' cmds: - - OLLAMA_URL= docker compose up -d + - docker compose -f compose.yaml -f infra/ollama/compose.hosted-only.yaml up -d + + hosted-only-down: + desc: Stop the stack including the hosted-only override (append `-- -v` to drop volumes) + cmds: + - docker compose -f compose.yaml -f infra/ollama/compose.hosted-only.yaml down {{.CLI_ARGS}} down: desc: Stop the full stack (append `-- -v` to also drop volumes) @@ -79,7 +84,7 @@ tasks: obs-up: desc: Start the stack with the opt-in observability profile (Grafana/Prometheus/Loki/Tempo) cmds: - - docker compose --profile local-ai --profile observability up -d + - docker compose --profile observability up -d obs-down: desc: Stop the stack including the observability profile (append `-- -v` to drop volumes) @@ -89,7 +94,7 @@ tasks: docker-socket-up: desc: 'Start the stack with core-app on the RAW Docker socket instead of the default docker-proxy-core (#622, #708) — root-equivalent, no allowlist in front of it; requires DOCKER_GID (see services/core-app/compose.docker-socket.yaml)' cmds: - - docker compose --profile local-ai -f compose.yaml -f services/core-app/compose.docker-socket.yaml up -d + - docker compose -f compose.yaml -f services/core-app/compose.docker-socket.yaml up -d docker-socket-down: desc: Stop the stack including the Docker-socket override (append `-- -v` to drop volumes) @@ -99,7 +104,7 @@ tasks: external-mounts-up: desc: 'Start the stack with operator-declared external file mounts (#731) — additional Files roots bound from the host; requires EXTERNAL_MOUNT_*_HOST_PATH + FILES_EXTERNAL_MOUNTS (see services/core-app/compose.external-mounts.yaml)' cmds: - - docker compose --profile local-ai -f compose.yaml -f services/core-app/compose.external-mounts.yaml up -d + - docker compose -f compose.yaml -f services/core-app/compose.external-mounts.yaml up -d external-mounts-down: desc: Stop the stack including the external-mounts override (append `-- -v` to drop volumes) diff --git a/compose.yaml b/compose.yaml index 925c5bed..c97ce972 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,14 @@ # Top-level stack (ADR-0006): the data plane plus enabled modules, assembled from -# fragments. `docker compose up -d` (or `task up`) brings up everything. +# fragments. `docker compose up -d` (or `task up`) brings up everything — the local AI +# runtime included, with no `.env`, no profile and no flags. That is the install the README +# and docs/user/installation.md publish, and nothing added here may quietly ask more of it: +# anything optional is layered *over* this file as an overlay the operator opts into. +# +# services/core-app/compose.docker-socket.yaml the raw Docker socket (#708) +# services/core-app/compose.external-mounts.yaml extra Files roots from the host (#731) +# infra/ollama/compose.hosted-only.yaml no local AI runtime at all (#962) +# +# (`--profile observability` is the one profile the stack uses, and it *adds* services.) # # Each module contributes its own services//compose.yaml fragment; add a # module by including its fragment here (the installer does this dynamically diff --git a/docs/infrastructure/index.md b/docs/infrastructure/index.md index 4ab950fb..56110084 100644 --- a/docs/infrastructure/index.md +++ b/docs/infrastructure/index.md @@ -244,29 +244,34 @@ after boot — so there is no startup race regardless. ### Hosted-only: no local LLM runtime A deployment can run **no local runtime at all** — hosted chat and hosted embeddings, nothing -local (#962, ADR-0144). `ollama` and `ollama-init` carry the **`local-ai` compose profile**, so -they can be left out of the stack entirely; a blank `OLLAMA_URL` is how the core is told that is -deliberate, and `core-app`'s dependency on `ollama` is `required: false` so its absence does not -fail the whole `up`. - -**Local AI is on by default and stays on.** A compose profile is opt-in by construction, so every -path that starts the stack selects it explicitly: `.env.example` ships `COMPOSE_PROFILES=local-ai` -(what a bare `docker compose up -d` reads), `task up` / `task obs-up` / `task docker-socket-up` / -`task external-mounts-up` pass `--profile local-ai`, and `infra/cd/reconcile.sh` passes it unless -`EPICURUS_LOCAL_AI=0`. Nothing about an existing install changes. - -To run hosted-only, three things — the first alone gives you a stack with no runtime *and* a core -still looking for one: +local (#962, ADR-0144). It is an **opt-out overlay**, +[`infra/ollama/compose.hosted-only.yaml`](../../infra/ollama/compose.hosted-only.yaml), the same +idiom as the Docker-socket and external-mount opt-ins: ```bash -# 1. leave Ollama out of the stack 2. tell the core there is no local runtime -# (COMPOSE_PROFILES= in .env, or just) (OLLAMA_URL= in .env, or just) -OLLAMA_URL= docker compose up -d # == task hosted-only-up - -# 3. in .env, point both model defaults at hosted ids — a bare name routes to the -# local runtime, so leaving these is the half-working stack: -# LLM_DEFAULT_MODEL=claude/claude-sonnet-4-6 -# MEMORY_EMBED_MODEL=gpt/text-embedding-3-small +task hosted-only-up +# == docker compose -f compose.yaml -f infra/ollama/compose.hosted-only.yaml up -d +``` + +The overlay does **both** halves, because either alone is wrong: it removes `ollama` and +`ollama-init` from the stack (which `core-app`'s `required: false` dependency on `ollama` is what +makes survivable), *and* it blanks `OLLAMA_URL`, which is how the core is told the absence is +deliberate. Remove only the container and the core is left probing a host that is gone — that is +the `unreachable` state, not `absent`. + +**Nothing about the default install changes.** `docker compose up -d` from a fresh clone, with no +`.env` and no flags, still starts the local runtime exactly as before — which is why this is an +overlay and not a compose profile on the `ollama` services. A profile is opt-in by construction, +so it would have taken the runtime out of every fresh clone while the shipped `llama3.2` / +`nomic-embed-text` defaults still pointed at it, which is the half-working stack the Kubernetes +chart now refuses to render. + +Set both model defaults to hosted ids in `.env` as well — a bare name routes to the local +runtime, so leaving them is that same half-working stack: + +```dotenv +LLM_DEFAULT_MODEL=claude/claude-sonnet-4-6 +MEMORY_EMBED_MODEL=gpt/text-embedding-3-small ``` Add each provider's API key on the **Models** page before the first turn. With no local runtime @@ -277,8 +282,10 @@ model as `n/a` instead of "warming" forever. A local model id asked to answer is sentence naming the fix, before any provider call. The Kubernetes equivalent is [`ollama.enabled: false` with a blank `ollama.external.url`](kubernetes.md#hosted-only-no-local-llm-runtime). -To go back, remove `OLLAMA_URL=` from `.env` and bring the stack up with the profile again; the -`ollama-models` volume is untouched by any of this, so the models are still there. +On the deploy box, `EPICURUS_HOSTED_ONLY=1` makes `infra/cd/reconcile.sh` apply the same overlay; +unset (the default) it keeps the local runtime, so a box that never asked for a hosted-only stack +cannot lose Ollama to a reconcile. To go back, drop the overlay (`task up`): the `ollama-models` +volume is untouched by any of this, so the models are still there. ## Log retention diff --git a/docs/infrastructure/kubernetes.md b/docs/infrastructure/kubernetes.md index 476d1253..348086c1 100644 --- a/docs/infrastructure/kubernetes.md +++ b/docs/infrastructure/kubernetes.md @@ -456,7 +456,7 @@ old guard — `required` on `ollama.external.url` — refused a *legitimate* dep refuses a broken one. Add each provider's API key on the Models page before the first turn. The Compose equivalent is the -[`local-ai` profile](index.md#hosted-only-no-local-llm-runtime). +[hosted-only overlay](index.md#hosted-only-no-local-llm-runtime) (`task hosted-only-up`). **MinIO is on by default**, matching the Compose stack. It backs two different things: the `storage` module's object store (chat uploads, agent-written objects, diff --git a/docs/infrastructure/startup-and-recovery.md b/docs/infrastructure/startup-and-recovery.md index 2fa365cf..3eb4f5a8 100644 --- a/docs/infrastructure/startup-and-recovery.md +++ b/docs/infrastructure/startup-and-recovery.md @@ -117,11 +117,10 @@ docker compose exec core-app python -c \ "import urllib.request,sys; sys.stdout.write(urllib.request.urlopen('http://127.0.0.1:8080/platform/v1/llm/local-runtime').read().decode())" ``` -`absent` is the hosted-only mode and nothing is wrong. `unreachable` means the container is -down or the URL is wrong — check that `ollama` is running (`docker compose ps ollama`; it -starts only with the `local-ai` profile, which `task up` and `infra/cd/reconcile.sh` pass for -you) and that `OLLAMA_URL` matches it. A 500 from that endpoint is a bug worth reporting: it -was the symptom this contract exists to remove. +`absent` is the hosted-only mode (`task hosted-only-up`, or `EPICURUS_HOSTED_ONLY=1` on a +deploy box) and nothing is wrong. `unreachable` means the container is down or the URL is +wrong — check `docker compose ps ollama` and that `OLLAMA_URL` matches it. A 500 from that +endpoint is a bug worth reporting: it was the symptom this contract exists to remove. ### OpenBao is sealed {#openbao-sealed} diff --git a/docs/reference/platform-api.md b/docs/reference/platform-api.md index 1c14a7ab..c96289fd 100644 --- a/docs/reference/platform-api.md +++ b/docs/reference/platform-api.md @@ -366,6 +366,14 @@ ADR-0144). Shell-facing; no body, no query params. `url_configured` is the *why* behind the state, and is `false` exactly when `state` is `absent`. +**A client that cannot get an answer reads it as `ok`.** A 404 (an older core, which has no +such route), a non-2xx, or an unparseable body all mean *keep doing what you did before this +endpoint existed* — render the full local UI. The alternative default, `absent`, would have a +shell collapse every local-runtime control against a core that is serving one perfectly well, +which is a worse failure than showing a control that then refuses. The state is an +optimisation on what the surface shows, never the authority on what the core will do: the +core refuses for itself, with a status and a sentence, whatever the shell believed. + Three facts that used to be one, which is why this endpoint exists rather than an envelope around the model list. `GET /platform/v1/llm/models` stays a bare `list[ModelInfo]` (twelve consumers read that array) and **never 500s again**: it answers `200` with `[]` when the diff --git a/docs/services/core-app.md b/docs/services/core-app.md index 5908d8af..c4eb2f8b 100644 --- a/docs/services/core-app.md +++ b/docs/services/core-app.md @@ -531,7 +531,7 @@ own `POST /platform/v1/llm/chat` was **removed in `core-app` 0.2.0** — it dupl | `GET /platform/v1/llm/models/details?model=…` | Read-only facts about a model: `{quantization, parameter_size, context_length, family, capabilities}` (any field `null`/empty when not reported — never a fake default). Local models read the runtime's `/api/show`; **hosted** models (#633/#618) read LiteLLM's own model-cost/context map instead (no provider call) — `quantization`/`parameter_size`/`family` stay `null` there (Ollama-only concepts), `capabilities` is what the resolution actually decided — `tools` only when the model is resolved tool-capable (no longer hard-coded), `vision` when the map or an override says so, `embedding` for an embedding model. Three resolved fields ride beside it (ADR-0140): `role` (`chat`|`embedding`|`unknown`), `supports_tools` (`true`/`false`, `null` when the local runtime could not be asked at all) and `in_catalogue` (`false` for a hosted id LiteLLM's map has never heard of — `null` for a local model). They exist because `capabilities` cannot express *unknown*: an empty list means both "nothing to badge" and "no idea", and a shell guessing between them shows the wrong hint. Backs the model-settings sheet, the Models page's context-window chip and **unlisted** badge, and the chat "can't use tools" / "can't see images" hints. `model` is a query param (names carry `:`/`/`). | | `GET /platform/v1/llm/catalog` | The browsable model catalog the core parses from upstream on a schedule (#269). Returns `{entries[], source, updated_at, stale}`; each entry's `size_gb` is the **real on-disk size** backfilled from its family's tags page (#571; `null` until the size fill or a variant lookup reaches the family, and always `null` for `cloud` rows). `stale` flags a seed / last-good list served after a failed or skipped refresh. See **Model catalog** below. | | `GET /platform/v1/llm/catalog/variants?model=…` | The quant variants available for a model (#330), looked up on demand from the model's public library **tags page** (the catalog index lists *sizes*, not quants). Returns `{model, variants:[{tag, quant, size_gb}]}` — `size_gb` is the tag row's real on-disk size (#571; `null` when upstream shows none, e.g. a cloud alias). Best-effort — an empty list (offline, or a model not in the public library) makes the UI fall back to a manual tag box. A successful lookup also piggybacks its sizes onto the catalog snapshot. `model` is a query param. See **Model catalog** below. | -| `GET /platform/v1/llm/local-runtime` | Whether this deployment has a local LLM runtime, and whether it answers (#962, ADR-0144): `{state: "absent"|"unreachable"|"ok", url_configured: bool}`. `absent` means `OLLAMA_URL` is blank — a deliberate hosted-only deployment, where a surface should *collapse* its local half rather than draw it broken; `unreachable` is an error and should still look like one. A separate endpoint on purpose: `GET /llm/models` stays a bare array (twelve consumers read it) and now answers `200` with `[]` in both non-serving states instead of the 500 the Models page collected every ten seconds. | +| `GET /platform/v1/llm/local-runtime` | Whether this deployment has a local LLM runtime, and whether it answers (#962, ADR-0144): `{state: "absent"|"unreachable"|"ok", url_configured: bool}`. `absent` means `OLLAMA_URL` is blank — a deliberate hosted-only deployment, where a surface should *collapse* its local half rather than draw it broken; `unreachable` is an error and should still look like one. A client that gets a 404 (an older core) or any unusable answer reads it as `ok` — keep the pre-#962 behaviour; the state only steers what a surface *draws*, never what the core will do. A separate endpoint on purpose: `GET /llm/models` stays a bare array (twelve consumers read it) and now answers `200` with `[]` in both non-serving states instead of the 500 the Models page collected every ten seconds. | | `POST /platform/v1/llm/pull` · `POST /platform/v1/llm/pull/stream` | Pull a model (blocking / SSE progress). **409** when this deployment runs no local runtime, **502** when one is configured and unreachable (#962) — never a bare 500. The SSE form refuses *before* the response starts, so the caller gets a real status rather than a 200 whose only event is an error. | | `POST /platform/v1/llm/unload` | Drop model(s) from memory now (`keep_alive=0`) **without** changing power state (#331). Body `{model: str\|null}` — `null`/omitted unloads every loaded model, a name unloads just that one. Returns `{status, model}` (`"all"` when none given). The standalone unload the Models page calls; the `loaded` flag refreshes on the next poll. **409** with no local runtime (#962): the gateway's own `unload` stays silent there because it is also on the power-pause path, which must keep working on a hosted-only box, so the refusal is made at the route — where the caller is an operator who clicked Unload and deserves to know why nothing happened. | | `GET /platform/v1/llm/providers` | Providers and what the secret store knows about each one's key. Each row is `{alias, local, configured, needs_base_url, key_state, key_error}`. `key_state` is `not_required` (the local runtime holds no key) / `present` / `missing` (OpenBao answered and has nothing there) / `unavailable` (OpenBao could not be asked — an expired app token, the service down), with `key_error` naming the reason for the last one. `configured` is unchanged (`true` for `not_required` and `present`) — it was one bit over three facts, and collapsing "we could not ask" into "there is no key" is how #728's expired token read as a fleet of unconfigured providers, sending the operator to re-enter keys that were already set. The core reports the distinction; rendering it is the shell's job (ADR-0018) — the Models page's "Add a hosted model" row (#922) is the first place that reads `key_state`, hinting inline when it is `missing`/`unavailable`. | diff --git a/infra/cd/reconcile.sh b/infra/cd/reconcile.sh index 83742d82..3c18cad8 100644 --- a/infra/cd/reconcile.sh +++ b/infra/cd/reconcile.sh @@ -69,22 +69,17 @@ fi # path re-adds a discovered override explicitly — `docker-compose.override.yml` is gitignored # precisely so an operator can keep one on the box, and dropping it here would be the same # class of silent revert this change exists to fix. -# 3. Local AI is ON by default, here as everywhere else (#962, ADR-0144). Ollama carries the -# `local-ai` profile so a hosted-only deployment can leave it out — but a profile is -# opt-in by construction, and a deploy script that silently stopped selecting it would -# take the local runtime down on the next reconcile of a box that never asked for that. -# So the profile is passed explicitly (`--profile` does not disable override discovery — -# only `-f` does, see 2), and turning it off is a deliberate `EPICURUS_LOCAL_AI=0`, -# paired with `OLLAMA_URL=` in .env or the core keeps probing a host that is gone. +# 3. Local AI stays on unless the box asks otherwise (#962, ADR-0144). A deployment can run +# with no local runtime, but that is an **overlay** the operator opts into +# (`EPICURUS_HOSTED_ONLY=1`), never something this script stops selecting: a deploy path +# that silently dropped Ollama would take the runtime down on the next reconcile of a box +# that never asked for it. The default path below still passes no `-f` at all. set -- -if [ "${EPICURUS_LOCAL_AI:-1}" = "0" ]; then - log "EPICURUS_LOCAL_AI=0 — reconciling WITHOUT the local AI runtime (hosted models only)." -else - set -- --profile local-ai -fi -if [ -n "${DOCKER_GID}" ]; then - log "DOCKER_GID is set — including the Docker-socket opt-in overlay." - set -- "$@" -f compose.yaml +# Any opt-in overlay means passing `-f`, which disables override auto-discovery (see 2) — so +# the base file and any discovered local override are re-added explicitly, once, ahead of +# whichever overlays are requested. +if [ -n "${DOCKER_GID}" ] || [ "${EPICURUS_HOSTED_ONLY:-0}" = "1" ]; then + set -- -f compose.yaml for override in \ compose.override.yaml compose.override.yml \ docker-compose.override.yaml docker-compose.override.yml; do @@ -94,7 +89,14 @@ if [ -n "${DOCKER_GID}" ]; then break fi done - set -- "$@" -f services/core-app/compose.docker-socket.yaml + if [ -n "${DOCKER_GID}" ]; then + log "DOCKER_GID is set — including the Docker-socket opt-in overlay." + set -- "$@" -f services/core-app/compose.docker-socket.yaml + fi + if [ "${EPICURUS_HOSTED_ONLY:-0}" = "1" ]; then + log "EPICURUS_HOSTED_ONLY=1 — reconciling with NO local AI runtime (hosted models only)." + set -- "$@" -f infra/ollama/compose.hosted-only.yaml + fi fi log "Pulling images (EPICURUS_VERSION=${VERSION:-latest})..." diff --git a/infra/ci/smoke.sh b/infra/ci/smoke.sh index f55d69f9..c2ab1996 100644 --- a/infra/ci/smoke.sh +++ b/infra/ci/smoke.sh @@ -53,11 +53,7 @@ mkdir -p "$SMOKE_MOUNT_DIR" chmod 0777 "$SMOKE_MOUNT_DIR" printf 'seed\n' > "$SMOKE_MOUNT_DIR/seed.txt" BOOT_LOG="$(mktemp)" -# `--profile local-ai`: the Ollama services carry that profile since #962 (it is how a -# hosted-only deployment leaves them out). This gate boots the *default* stack — local AI on -# — and asserts things that need the workload to exist: the ollama-init one-shot's exit code, -# and the KV-cache change restarting it through the container seam. -DC="docker compose --profile local-ai -f compose.yaml -f infra/ci/compose.ci.yaml --env-file $ENV_FILE" +DC="docker compose -f compose.yaml -f infra/ci/compose.ci.yaml --env-file $ENV_FILE" CURL_IMG="curlimages/curl:8.11.1" DATA_PLANE="openbao postgres valkey nats qdrant minio minio-init" @@ -159,7 +155,7 @@ docker pull -q "$CURL_IMG" >/dev/null # Pre-flight: two fragments publishing the same host port is the #68 collision class. # The smoke itself clears ports (for isolation), so check the real compose instead. log "Pre-flight: checking for duplicate published host ports" -dupes="$(docker compose --profile local-ai -f compose.yaml config 2>/dev/null | +dupes="$(docker compose -f compose.yaml config 2>/dev/null | grep -oE 'published: "?[0-9]+' | grep -oE '[0-9]+' | sort | uniq -d | tr '\n' ' ')" [ -z "$dupes" ] || die "two services publish the same host port(s): $dupes — pick a unique one" ok "no duplicate published host ports" diff --git a/infra/ollama/compose.hosted-only.yaml b/infra/ollama/compose.hosted-only.yaml new file mode 100644 index 00000000..ffbb5078 --- /dev/null +++ b/infra/ollama/compose.hosted-only.yaml @@ -0,0 +1,60 @@ +# Opt-out overlay: run the stack with NO local LLM runtime (#962, ADR-0144). +# +# Hosted chat and hosted embeddings, no Ollama container at all. Layered *over* the assembled +# stack, never included by it: +# +# docker compose -f compose.yaml -f infra/ollama/compose.hosted-only.yaml up -d +# # or simply: task hosted-only-up +# +# Why an overlay and not a profile on the `ollama` services. A compose profile is opt-in by +# construction: the moment those services carry one, they stop starting unless something +# selects it — and the repo's documented install is `git clone` → `cd epicurus` → +# `docker compose up -d` (README, docs/user/installation.md), which mentions no `.env` at all. +# A profile would therefore have taken the local runtime out of every fresh clone while the +# shipped defaults (`llama3.2`, `nomic-embed-text`) still pointed at it — precisely the +# half-working stack the chart's render-time guard exists to refuse, delivered by default to +# the person least able to diagnose it. This is the idiom the repo already uses for exactly +# this shape (`services/core-app/compose.docker-socket.yaml`, +# `services/core-app/compose.external-mounts.yaml`): the default path stays byte-identical and +# the opt-out is a visible command. `tests/test_no_local_runtime.py` pins both halves. +# +# Two changes, and both are needed — the first alone leaves a core still looking for a runtime +# that is no longer there, which is *unreachable*, not *absent*: +# +# 1. the `ollama` / `ollama-init` services are assigned a profile nothing ever enables, so +# they are excluded from the stack (`core-app`'s `depends_on` is `required: false`, +# which is what lets the rest of the stack come up without them); +# 2. `OLLAMA_URL` is blanked, which is how the core is told there is no local runtime: it +# then reports `absent` at `GET /platform/v1/llm/local-runtime`, answers `200` + `[]` +# from the model list instead of 500ing, refuses pull / delete / unload / the KV-cache +# setting with `409` and a reason, and reports chat readiness as `n/a` rather than +# "warming" forever. +# +# **Set hosted model defaults too**, in `.env` — a bare name routes to the local runtime, so +# leaving the shipped defaults in place gives you working chat and a failing embed on every +# memory recall and module index: +# +# LLM_DEFAULT_MODEL=claude/claude-sonnet-4-6 +# MEMORY_EMBED_MODEL=gpt/text-embedding-3-small +# +# then add each provider's API key on the Models page before the first turn. +# +# Nothing here is destructive: the `ollama-models` volume is untouched, so dropping the +# overlay and bringing the stack up normally restores the runtime with its models intact. + +services: + # `local-ai-disabled` is never enabled by anything in this repo — assigning it is how a + # compose overlay *removes* a service, since an override file can add to the model but + # cannot delete from it. Do not enable this profile; that is not a supported combination + # (you would get the container back while the core is still told it has no runtime). + ollama: + profiles: [local-ai-disabled] + + ollama-init: + profiles: [local-ai-disabled] + + core-app: + environment: + # Blank is a deliberate value, not an omission (see the header). It overrides the + # fragment's `${OLLAMA_URL-http://ollama:11434}`, so this works with no `.env` at all. + OLLAMA_URL: "" diff --git a/infra/ollama/compose.yaml b/infra/ollama/compose.yaml index 4b4927a3..b44b2503 100644 --- a/infra/ollama/compose.yaml +++ b/infra/ollama/compose.yaml @@ -10,14 +10,15 @@ # it is not published to the host (private-by-default — the core is the front door). # # ── Leaving it out (#962, ADR-0144) ───────────────────────────────────────────── -# Both services carry the `local-ai` profile, so a deployment can run *no* local runtime — -# hosted chat and hosted embeddings, nothing local to run a model on. The profile is ON -# everywhere the stack is actually started, so the default install is unchanged: -# `.env.example` ships `COMPOSE_PROFILES=local-ai`, every `task *-up` passes -# `--profile local-ai`, and `infra/cd/reconcile.sh` passes it unless `EPICURUS_LOCAL_AI=0`. -# A hosted-only stack is `COMPOSE_PROFILES=` **and** `OLLAMA_URL=`: the blank URL is what -# tells the core there is no local runtime, and without it the core would keep probing a -# hostname that no longer resolves instead of refusing with a reason. +# A deployment can run *no* local runtime — hosted chat and hosted embeddings, nothing +# local to run a model on. That is an **opt-out overlay**, `infra/ollama/compose.hosted-only.yaml` +# (`task hosted-only-up`), the same idiom as the Docker-socket and external-mount opt-ins — +# never a profile on the services below. A profile is opt-in by construction, and these +# services sit on the repo's documented install path (`git clone` → `docker compose up -d`, +# README and docs/user/installation.md, neither of which mentions an `.env`), so putting one +# here would have silently removed the local runtime from every fresh clone and left the +# shipped `llama3.2` / `nomic-embed-text` defaults pointing at nothing. The overlay keeps the +# default path byte-identical and makes the hosted-only choice a visible command. # Bounded json-file logging (#462) — see infra/compose/docker-compose.yml for the rationale. x-logging: &default-logging @@ -36,9 +37,6 @@ services: # the root — no subdirs needed. Mirrors qdrant-init, the core image entrypoint, and the OpenBao chown. ollama-init: image: alpine:3.21 - # Same profile as the server it prepares: with no local runtime there is no volume to - # chown and no reason to run a one-shot for it (#962). - profiles: [local-ai] restart: "no" user: root volumes: @@ -57,9 +55,6 @@ services: ollama: image: ollama/ollama:0.30.7 - # On by default (see the header): the local runtime is the local-first default, and the - # profile exists so an operator can *opt out* of it, never so they must opt in (#962). - profiles: [local-ai] restart: unless-stopped # Wait for the chown one-shot so the volume the core later writes to is owned by uid 10001 # (#392). This is ordering only — nothing here races the core's write, which is lazy: it diff --git a/services/core-app/compose.yaml b/services/core-app/compose.yaml index 1a20707f..3b5d7a9d 100644 --- a/services/core-app/compose.yaml +++ b/services/core-app/compose.yaml @@ -20,11 +20,12 @@ services: NATS_URL: nats://nats:4222 NATS_USER: core NATS_PASSWORD: ${NATS_CORE_PASSWORD:-epicurus-dev} - # The local LLM runtime. **Blank means there is none** (#962, ADR-0144): a hosted-only - # stack sets `OLLAMA_URL=` and `COMPOSE_PROFILES=` (which leaves the `local-ai` profile, - # and so the ollama service itself, out of the stack). Single-dash `-`, not `:-`: `:-` - # would substitute the default back in for a deliberately *empty* value, which is - # precisely the setting an operator uses to say "there is no local runtime here". + # The local LLM runtime. **Blank means there is none** (#962, ADR-0144) — the + # hosted-only overlay (infra/ollama/compose.hosted-only.yaml, `task hosted-only-up`) + # sets it blank and drops the ollama service in one step. Single-dash `-`, not `:-`: + # `:-` would substitute the default back in for a deliberately *empty* value, which is + # precisely what an operator sets to say "there is no local runtime here" — so the + # env-var route (`OLLAMA_URL=` in .env) works too. OLLAMA_URL: ${OLLAMA_URL-http://ollama:11434} # ── LLM gateway tuning (all optional; blank/unset = provider default) ── # Set any of these and restart — no code edit needed (issue #114). @@ -87,9 +88,9 @@ services: condition: service_started ollama: condition: service_started - # Not required (#962, ADR-0144): a hosted-only stack leaves the `local-ai` profile - # out, and without this the whole `up` would fail on a dependency that is absent on - # purpose. With local AI on (the default) the ordering is unchanged. + # Not required (#962, ADR-0144): the hosted-only overlay removes the ollama service + # altogether, and without this the whole `up` would fail on a dependency that is + # absent on purpose. With local AI on (the default) the ordering is unchanged. required: false openbao: condition: service_healthy diff --git a/tests/test_no_local_runtime.py b/tests/test_no_local_runtime.py index bf8049fb..58571d14 100644 --- a/tests/test_no_local_runtime.py +++ b/tests/test_no_local_runtime.py @@ -6,11 +6,11 @@ on the service. These are the static and render-time halves of the fix: * **Compose** — parsed from the fragments directly, in the style of ``test_compose_ports.py``: - the profile that lets Ollama be left out, the `required: false` that stops its absence + the opt-out overlay that removes Ollama, the `required: false` that stops its absence failing the whole `up`, and the env interpolation that carries a *deliberately blank* - ``OLLAMA_URL`` through to the core. Plus the thing that is easy to get wrong and impossible - to see: that turning the runtime into a profile did not turn local AI off for everyone who - was not asking for that. + ``OLLAMA_URL`` through to the core. And, first, the thing this feature could most easily + break for people who never asked for it: the **documented install** — `git clone` && + `docker compose up -d`, no `.env`, no flags — must still bring up the local runtime. * **Chart** — real ``helm template`` renders (no cluster), because a guard nobody renders is a guard that has quietly stopped guarding. Skipped, not silently passed, without ``helm``. """ @@ -30,11 +30,14 @@ OLLAMA_FRAGMENT = REPO / "infra" / "ollama" / "compose.yaml" CORE_FRAGMENT = REPO / "services" / "core-app" / "compose.yaml" -LOCAL_AI_PROFILE = "local-ai" +HOSTED_ONLY_OVERLAY = REPO / "infra" / "ollama" / "compose.hosted-only.yaml" -# Every command that starts the stack must select the profile, or the operator who typed it -# silently loses local AI. The Taskfile tasks are listed by name so a *new* start task that -# forgets it is a conversation at review, not a surprise on someone's box. +# The services that *are* the local runtime. Named here so a rename has to come past these +# assertions rather than quietly making them vacuous. +_LOCAL_AI_SERVICES = ("ollama", "ollama-init") + +# Every Taskfile command that starts the stack for ordinary use. None of them may need a flag +# to get local AI: the flagless path is the one the docs publish. _START_TASKS = ("up", "obs-up", "docker-socket-up", "external-mounts-up") @@ -47,14 +50,73 @@ def _fragment(path: Path) -> dict[str, Any]: # ── Compose ────────────────────────────────────────────────────────────────────── -def test_the_local_runtime_can_be_left_out_of_the_stack() -> None: +def test_the_default_install_still_brings_up_the_local_runtime() -> None: + """The regression guard on the *documented* install path. + + `README.md` and `docs/user/installation.md` both publish `git clone` → `cd epicurus` → + `docker compose up -d`, and neither mentions an `.env` before that step. So whatever + expresses "no local runtime" must be something the operator **adds**, never something the + default path has to remember to select. The first cut of #962 used a compose profile on + these services; a profile is opt-in, so it removed Ollama from every fresh clone while the + shipped `llama3.2` / `nomic-embed-text` defaults still pointed at it — the half-working + stack the chart's render-time guard refuses, delivered by default. + + Asserted on the fragments (no Docker): nothing may gate these services behind a profile, + and no start command may need a flag to get them. `compose-validate` proves the same thing + against a real `docker compose config`. + """ services = _fragment(OLLAMA_FRAGMENT)["services"] - for name in ("ollama", "ollama-init"): - assert services[name].get("profiles") == [LOCAL_AI_PROFILE], ( - f"{name} must carry the {LOCAL_AI_PROFILE!r} profile — it is how a hosted-only " - "deployment leaves the local runtime out (#962)" + for name in _LOCAL_AI_SERVICES: + assert "profiles" not in services[name], ( + f"{name} is behind a compose profile, so `docker compose up -d` from a fresh " + "clone no longer starts the local runtime — use the opt-out overlay " + "(infra/ollama/compose.hosted-only.yaml) instead (#962)" + ) + + taskfile = yaml.safe_load((REPO / "Taskfile.yml").read_text(encoding="utf-8")) + for name in _START_TASKS: + cmds = " ".join(str(c) for c in taskfile["tasks"][name]["cmds"]) + assert "hosted-only" not in cmds and "--profile local-ai" not in cmds, ( + f"task {name} should start the local runtime with no extra selection" ) + reconcile = (REPO / "infra" / "cd" / "reconcile.sh").read_text(encoding="utf-8") + assert "EPICURUS_HOSTED_ONLY" in reconcile, ( + "the deploy path has no way to opt out of the local runtime" + ) + assert "EPICURUS_HOSTED_ONLY:-0" in reconcile, ( + "the deploy path must default to *keeping* the local runtime — a box that never " + "asked for a hosted-only stack must not lose Ollama on its next reconcile" + ) + + +def test_the_local_runtime_can_be_left_out_through_the_overlay() -> None: + """The opt-out, and both halves of it. + + Removing the container alone leaves a core still pointed at `http://ollama:11434` — that + is *unreachable*, not *absent*, and the whole point of #962 is that those are different + facts. The overlay does both, so `task hosted-only-up` is one command and cannot be + half-applied. + """ + overlay = _fragment(HOSTED_ONLY_OVERLAY)["services"] + + for name in _LOCAL_AI_SERVICES: + profiles = overlay[name].get("profiles") + assert profiles, f"the overlay does not remove {name}" + # An override file can add to the compose model but not delete from it, so assigning a + # profile nothing enables is how a service is taken out. Any profile name works; what + # must stay true is that it is not one this repo ever activates. + for profile in profiles: + assert profile != "observability", ( + "the overlay parks a service on a profile the repo actually enables, so the " + "removal would undo itself" + ) + + assert overlay["core-app"]["environment"]["OLLAMA_URL"] == "", ( + "the overlay removes the container but leaves the core looking for it — that is the " + "`unreachable` state, not `absent` (#962)" + ) + def test_the_core_does_not_hard_depend_on_the_local_runtime() -> None: """Without this the whole `up` fails on a service that is absent on purpose.""" @@ -73,40 +135,23 @@ def test_a_blank_ollama_url_survives_interpolation() -> None: assert env["OLLAMA_URL"] == "${OLLAMA_URL-http://ollama:11434}" -def test_every_start_path_still_turns_local_ai_on() -> None: - """A profile is opt-in by construction; the default install must not have changed. - - This is the regression guard for the way this feature could hurt people who did not ask - for it: put a profile on Ollama and every existing `task up`, every deploy reconcile and - every documented `.env` stops starting the local runtime. So each of those paths selects - the profile explicitly, and this test is what keeps it that way. - """ +def test_the_opt_out_is_reachable_as_one_command() -> None: + """The overlay only helps if an operator can find it; `task hosted-only-up` is how.""" taskfile = yaml.safe_load((REPO / "Taskfile.yml").read_text(encoding="utf-8")) - for name in _START_TASKS: - cmds = " ".join(str(c) for c in taskfile["tasks"][name]["cmds"]) - assert f"--profile {LOCAL_AI_PROFILE}" in cmds, ( - f"task {name} no longer starts the local AI runtime — a profile is opt-in, so " - "every start path has to select it or the default install silently changes" - ) + cmds = " ".join(str(c) for c in taskfile["tasks"]["hosted-only-up"]["cmds"]) + assert "infra/ollama/compose.hosted-only.yaml" in cmds + assert "-f compose.yaml" in cmds, "the overlay must be layered over the assembled stack" - reconcile = (REPO / "infra" / "cd" / "reconcile.sh").read_text(encoding="utf-8") - assert f"--profile {LOCAL_AI_PROFILE}" in reconcile, ( - "infra/cd/reconcile.sh would take the local runtime down on the next deploy of a " - "box that never asked for a hosted-only stack" - ) - assert "EPICURUS_LOCAL_AI" in reconcile, "the deploy path has no way to opt out" - - env_example = (REPO / ".env.example").read_text(encoding="utf-8") - assert f"COMPOSE_PROFILES={LOCAL_AI_PROFILE}" in env_example, ( - ".env.example must ship the profile: it is what selects local AI for a bare " - "`docker compose up -d`" - ) +def test_the_compose_gate_keeps_booting_the_default_stack() -> None: + """`runtime-smoke` asserts ollama-init's exit code and a KV-cache restart — both need it. -def test_the_compose_smoke_gate_boots_the_local_runtime() -> None: - """`runtime-smoke` asserts ollama-init's exit code and a KV-cache restart — both need it.""" + It boots the plain stack with no overlay, which is the point: the gate and the documented + install are the same shape, so one cannot drift from the other unnoticed. + """ smoke = (REPO / "infra" / "ci" / "smoke.sh").read_text(encoding="utf-8") - assert f"--profile {LOCAL_AI_PROFILE}" in smoke + assert "compose.hosted-only.yaml" not in smoke + assert "ollama-init" in smoke, "the gate no longer asserts the local runtime's one-shot" def test_the_kubernetes_gate_asserts_the_hosted_only_mode() -> None: From f62be8b0330e60567532d71ebc3aed65113763ae Mon Sep 17 00:00:00 2001 From: baakhoff Date: Sat, 19 Sep 2026 21:15:11 +0000 Subject: [PATCH 4/4] docs(core-app): say what the refusals actually do, not what the contract asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the merge set against the running endpoint. Three paths do not do what the new docs claim, and the claim is the part that is wrong: * `POST /llm/unload` answers **200** when a configured runtime is unreachable, not 502. The route's absence check is 409-only by design, and the gateway's `unload` never raises because it is also on the power-pause path. The platform-API reference listed it in the bullet promising the full 409/502 pair; `docs/services/core-app.md` already described it correctly, so only the reference and the changelog were wrong. * `POST /llm/pull/stream` likewise refuses 409 before the response starts but can only report an unreachable runtime as the `event: error` frame it always used — an SSE cannot revise its status once it has begun. Both the reference and the core-app page promised a 502 there. * `config.md`'s `LLM_BOOTSTRAP_MODELS` row still said a blank value is for "hosted-only builds". That was the over-promise #962 was filed against: a hosted-only deployment needs no setting there now, because a blank `OLLAMA_URL` short-circuits the bootstrap and the chart blanks the value itself when it is left at `auto`. The twin sentence in `startup-and-recovery.md` was corrected; this one was missed. Also one log line: the KV-cache apply logged "choice recorded but nothing to apply" with no runtime, but the route refuses with 409 *before* persisting the preference, so nothing is recorded. No behaviour change beyond the log string. `core-app` stays 0.129.0 (PATCH-level wording inside an unreleased MINOR). Part of #962. --- CHANGELOG.md | 5 +++-- docs/reference/config.md | 2 +- docs/reference/platform-api.md | 19 +++++++++++++------ docs/services/core-app.md | 4 ++-- .../epicurus_core_app/llm/ollama_runtime.py | 2 +- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c54648b..d84cbe61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,9 @@ images to GHCR. mode, spelled `OLLAMA_URL=""`: the model list answers **200 and an empty array** in both non-serving states, the new `GET /platform/v1/llm/local-runtime` says which one applies, the local-only actions (pull, delete, unload, the KV-cache setting) refuse with **409** and a - sentence naming the mode — **502** when a configured runtime is unreachable, never a bare 500 - — readiness reports the model as **n/a** instead of warming, and the bootstrap logs one line + sentence naming the mode — and pull and delete answer **502** when a configured runtime is + unreachable, never a bare 500 — readiness reports the model as **n/a** instead of warming, + and the bootstrap logs one line and returns. A local model id asked to serve is refused before any provider call with the capability error the rest of the gateway already speaks (ADR-0140), so a hosted embedding model keeps working while a bare one fails with the fix in the message instead of a connection error. diff --git a/docs/reference/config.md b/docs/reference/config.md index 201e7f07..cc38216c 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -85,7 +85,7 @@ in `CoreSettings` plus the LLM-gateway, agent, module, and memory knobs. | `llm_temperature` | `LLM_TEMPERATURE` | `float \| None` | `None` | Sampling temperature passed to each chat completion (local + hosted). A blank env value means unset. | | `llm_top_p` | `LLM_TOP_P` | `float \| None` | `None` | Nucleus-sampling `top_p` passed to each chat completion (local + hosted). | | `llm_num_ctx` | `LLM_NUM_CTX` | `int \| None` | `None` | Ollama context-window size (`num_ctx`); applied to local models only. | -| `llm_bootstrap_models` | `LLM_BOOTSTRAP_MODELS` | `str` | `auto` | First-boot model bootstrap (#773, ADR-0118, amended #923), pulled in the background (never blocking readiness). `auto` = seed an *empty* runtime with the effective chat + embedding defaults, then no-op on every later start once anything is installed (a deleted default stays deleted); blank = disabled (air-gapped / hosted-only builds; the CI smoke gate sets this); an explicit comma-separated list is a standing pin — ensured on *every* start regardless of what else is installed. Hosted-prefixed ids are skipped. | +| `llm_bootstrap_models` | `LLM_BOOTSTRAP_MODELS` | `str` | `auto` | First-boot model bootstrap (#773, ADR-0118, amended #923), pulled in the background (never blocking readiness). `auto` = seed an *empty* runtime with the effective chat + embedding defaults, then no-op on every later start once anything is installed (a deleted default stays deleted); blank = disabled (air-gapped builds; the CI smoke gate sets this). A **hosted-only** deployment needs no setting here since #962: a blank `OLLAMA_URL` makes the bootstrap return in one log line whatever this says, and the Helm chart blanks it for you when it is left at `auto`; an explicit comma-separated list is a standing pin — ensured on *every* start regardless of what else is installed. Hosted-prefixed ids are skipped. | | `llm_catalog_url` | `LLM_CATALOG_URL` | `str` | `https://ollama.com/library` | Source the core parses the browsable model catalog from (#269). Point at a mirror for an air-gapped deployment. | | `llm_catalog_refresh_seconds` | `LLM_CATALOG_REFRESH_SECONDS` | `int` | `21600` (6h) | How often the background loop re-parses the catalog source. Floored to 60s. | | `llm_catalog_max_models` | `LLM_CATALOG_MAX_MODELS` | `int` | `0` | Cap on model families kept (the most-popular survive); `0` = unlimited. | diff --git a/docs/reference/platform-api.md b/docs/reference/platform-api.md index c96289fd..058256e7 100644 --- a/docs/reference/platform-api.md +++ b/docs/reference/platform-api.md @@ -378,12 +378,19 @@ Three facts that used to be one, which is why this endpoint exists rather than a around the model list. `GET /platform/v1/llm/models` stays a bare `list[ModelInfo]` (twelve consumers read that array) and **never 500s again**: it answers `200` with `[]` when the runtime is absent *and* when it is unreachable. Everything that genuinely needs a runtime -answers **409** when it is absent, with a `detail` naming the mode, and **502** when it is -configured but unreachable — never a bare 500: - -- `POST /platform/v1/llm/pull` · `POST /platform/v1/llm/pull/stream` (refused **before** the - stream starts, so the caller sees a real status rather than a 200 whose only event is an - error) · `DELETE /platform/v1/llm/models` · `POST /platform/v1/llm/unload`; +refuses with a reason and never a bare 500: **409** when it is absent, with a `detail` naming +the mode, and **502** when it is configured but unreachable — except on the three paths whose +own shape rules the second half out, which are called out below: + +- `POST /platform/v1/llm/pull` · `DELETE /platform/v1/llm/models` — the full pair, 409 and 502; +- `POST /platform/v1/llm/pull/stream` — **409 when absent only**, refused *before* the stream + starts so the caller sees a real status rather than a 200 whose only event is an error. Once + the stream has begun it cannot take its status back, so an *unreachable* runtime is still + reported the way it always was: a `200` whose last frame is `event: error`; +- `POST /platform/v1/llm/unload` — **409 when absent only**. The gateway's `unload` never + raises (it is also on the power-pause path, which must keep working on a hosted-only + deployment), so an unreachable runtime answers `200` with nothing unloaded, exactly as + before this change; - `PUT /platform/v1/llm/prefs/kv-cache-type` — **409 when absent only**. This path never talks to Ollama (it writes the start-up env file and asks the container runtime to bounce the workload), so "unreachable" is not something it can observe, and setting the value while the diff --git a/docs/services/core-app.md b/docs/services/core-app.md index c4eb2f8b..76d3fd9f 100644 --- a/docs/services/core-app.md +++ b/docs/services/core-app.md @@ -532,7 +532,7 @@ own `POST /platform/v1/llm/chat` was **removed in `core-app` 0.2.0** — it dupl | `GET /platform/v1/llm/catalog` | The browsable model catalog the core parses from upstream on a schedule (#269). Returns `{entries[], source, updated_at, stale}`; each entry's `size_gb` is the **real on-disk size** backfilled from its family's tags page (#571; `null` until the size fill or a variant lookup reaches the family, and always `null` for `cloud` rows). `stale` flags a seed / last-good list served after a failed or skipped refresh. See **Model catalog** below. | | `GET /platform/v1/llm/catalog/variants?model=…` | The quant variants available for a model (#330), looked up on demand from the model's public library **tags page** (the catalog index lists *sizes*, not quants). Returns `{model, variants:[{tag, quant, size_gb}]}` — `size_gb` is the tag row's real on-disk size (#571; `null` when upstream shows none, e.g. a cloud alias). Best-effort — an empty list (offline, or a model not in the public library) makes the UI fall back to a manual tag box. A successful lookup also piggybacks its sizes onto the catalog snapshot. `model` is a query param. See **Model catalog** below. | | `GET /platform/v1/llm/local-runtime` | Whether this deployment has a local LLM runtime, and whether it answers (#962, ADR-0144): `{state: "absent"|"unreachable"|"ok", url_configured: bool}`. `absent` means `OLLAMA_URL` is blank — a deliberate hosted-only deployment, where a surface should *collapse* its local half rather than draw it broken; `unreachable` is an error and should still look like one. A client that gets a 404 (an older core) or any unusable answer reads it as `ok` — keep the pre-#962 behaviour; the state only steers what a surface *draws*, never what the core will do. A separate endpoint on purpose: `GET /llm/models` stays a bare array (twelve consumers read it) and now answers `200` with `[]` in both non-serving states instead of the 500 the Models page collected every ten seconds. | -| `POST /platform/v1/llm/pull` · `POST /platform/v1/llm/pull/stream` | Pull a model (blocking / SSE progress). **409** when this deployment runs no local runtime, **502** when one is configured and unreachable (#962) — never a bare 500. The SSE form refuses *before* the response starts, so the caller gets a real status rather than a 200 whose only event is an error. | +| `POST /platform/v1/llm/pull` · `POST /platform/v1/llm/pull/stream` | Pull a model (blocking / SSE progress). **409** when this deployment runs no local runtime (#962) — never a bare 500. The blocking form also answers **502** when a configured runtime is unreachable; the SSE form refuses *before* the response starts for the 409, but once the stream has begun it cannot take its status back, so an unreachable runtime still ends the stream with an `event: error` frame as it always did. | | `POST /platform/v1/llm/unload` | Drop model(s) from memory now (`keep_alive=0`) **without** changing power state (#331). Body `{model: str\|null}` — `null`/omitted unloads every loaded model, a name unloads just that one. Returns `{status, model}` (`"all"` when none given). The standalone unload the Models page calls; the `loaded` flag refreshes on the next poll. **409** with no local runtime (#962): the gateway's own `unload` stays silent there because it is also on the power-pause path, which must keep working on a hosted-only box, so the refusal is made at the route — where the caller is an operator who clicked Unload and deserves to know why nothing happened. | | `GET /platform/v1/llm/providers` | Providers and what the secret store knows about each one's key. Each row is `{alias, local, configured, needs_base_url, key_state, key_error}`. `key_state` is `not_required` (the local runtime holds no key) / `present` / `missing` (OpenBao answered and has nothing there) / `unavailable` (OpenBao could not be asked — an expired app token, the service down), with `key_error` naming the reason for the last one. `configured` is unchanged (`true` for `not_required` and `present`) — it was one bit over three facts, and collapsing "we could not ask" into "there is no key" is how #728's expired token read as a fleet of unconfigured providers, sending the operator to re-enter keys that were already set. The core reports the distinction; rendering it is the shell's job (ADR-0018) — the Models page's "Add a hosted model" row (#922) is the first place that reads `key_state`, hinting inline when it is `missing`/`unavailable`. | | `PUT` · `DELETE /platform/v1/llm/providers/{alias}/key` | Store / clear a hosted provider's key (core → OpenBao; never logged or returned). | @@ -589,7 +589,7 @@ else reads it: | --- | --- | | `LlmGateway.models()` | `[]`, no HTTP call — and `[]` rather than a raise when a *configured* runtime is unreachable, which is what stops `GET /llm/models` 500ing every ten seconds on the Models page | | `LlmGateway.local_runtime_state()` | `absent` \| `unreachable` \| `ok`, behind `GET /platform/v1/llm/local-runtime` | -| `pull` / `pull_stream` / `delete_model` | raise `LocalRuntimeUnavailableError(state="absent")` → **409**; an unreachable runtime → **502** | +| `pull` / `pull_stream` / `delete_model` | raise `LocalRuntimeUnavailableError(state="absent")` → **409**; an unreachable runtime → **502** from `pull` and `delete_model`. `pull_stream` is the exception: its 409 is raised before the response starts, but an unreachable runtime surfaces mid-stream as the `event: error` frame it always did — an SSE response cannot revise its status | | `unload` | returns quietly (it is on the power-pause path, which must keep working); the **route** answers 409 | | `_ensure_can_serve` | refuses a **local** model id with `ModelCapabilityError` → **400** and the hint *"No local runtime is configured — choose a hosted model."* — asked **before** the pause rule, because "resume to run inference" is an instruction an operator with no runtime cannot follow | | the chat fallback chain | skips local candidates (`_is_available`), so a hosted primary never falls back into a runtime that is not there | diff --git a/services/core-app/src/epicurus_core_app/llm/ollama_runtime.py b/services/core-app/src/epicurus_core_app/llm/ollama_runtime.py index a6fc36c8..8b9c7060 100644 --- a/services/core-app/src/epicurus_core_app/llm/ollama_runtime.py +++ b/services/core-app/src/epicurus_core_app/llm/ollama_runtime.py @@ -84,7 +84,7 @@ def apply_kv_cache_type(self, kv_cache_type: str | None) -> KvCacheApplyResult: to exist. """ if not self._local_runtime_enabled: - log.info("no local LLM runtime; KV-cache choice recorded but nothing to apply") + log.info("no local LLM runtime; nothing to write and nothing to restart") return KvCacheApplyResult(applied=False, staged=False) try: self._write_env_file(kv_cache_type)