From 62e6f6d970347680f93bce04589e6214eaced870 Mon Sep 17 00:00:00 2001 From: Krstan Vjestica Date: Sat, 1 Aug 2026 17:48:00 +0200 Subject: [PATCH 1/3] feat(assistant): split Ollama into its own Deployment, with optional GPU Follows the same split made in forail-assistant: the all-in-one image could never load a model, and Ollama now runs as a separate service. - forail-assistant-ollama Deployment + Service + its own PVC. Model blobs are far larger than the vector index and outlive a rebuild of the API, so keeping the claims apart means the index PVC is sized by the corpus, not the model. - assistant.ollama.gpu.enabled requests nvidia.com/gpu on that pod alone. Only the model server has to land on a GPU node; the API stays schedulable anywhere. Off by default -- it pins the pod to a node advertising the device, so it must stay off until the cluster has one and the device plugin is installed. - images.assistantOllama pinned to ollama/ollama:0.30.10, for the same reason the application image stopped tracking `latest`. - assistant.storage drops 20Gi -> 5Gi now that models live elsewhere, and the API's resource budget no longer has to cover inference. PVCs cannot shrink, so an existing install needs a fresh claim; noted in values.yaml. --- templates/forail-assistant.yaml | 128 ++++++++++++++++++++++++++++++-- values.yaml | 47 ++++++++++-- 2 files changed, 163 insertions(+), 12 deletions(-) diff --git a/templates/forail-assistant.yaml b/templates/forail-assistant.yaml index ec4e0a6..f9aa35e 100644 --- a/templates/forail-assistant.yaml +++ b/templates/forail-assistant.yaml @@ -15,6 +15,25 @@ spec: storageClassName: {{ . | quote }} {{- end }} --- +# Model blobs get their own claim: they are far larger than the vector index, +# they survive a rebuild of the assistant, and keeping them apart means the +# index PVC does not have to be sized for whatever model is configured. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: forail-assistant-ollama-models + namespace: {{ include "forail.namespace" . }} + labels: + {{- include "forail.componentLabels" (dict "root" . "component" "assistant-ollama") | nindent 4 }} +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: {{ .Values.assistant.ollama.storage.size }} + {{- with .Values.assistant.ollama.storage.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} +--- apiVersion: v1 kind: Service metadata: @@ -31,6 +50,102 @@ spec: selector: {{- include "forail.selectorLabels" (dict "root" . "component" "assistant") | nindent 4 }} --- +apiVersion: v1 +kind: Service +metadata: + name: forail-assistant-ollama + namespace: {{ include "forail.namespace" . }} + labels: + {{- include "forail.componentLabels" (dict "root" . "component" "assistant-ollama") | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: 11434 + targetPort: 11434 + selector: + {{- include "forail.selectorLabels" (dict "root" . "component" "assistant-ollama") | nindent 4 }} +--- +# Model server, separate from the API on purpose. Only this pod needs a GPU, +# so only this pod carries the GPU resource request and node placement; the +# API stays schedulable anywhere. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: forail-assistant-ollama + namespace: {{ include "forail.namespace" . }} + labels: + {{- include "forail.componentLabels" (dict "root" . "component" "assistant-ollama") | nindent 4 }} +spec: + # One replica: replicas do not share the model cache, and each would pull + # the model again onto its own volume. + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "forail.selectorLabels" (dict "root" . "component" "assistant-ollama") | nindent 6 }} + template: + metadata: + labels: + {{- include "forail.componentLabels" (dict "root" . "component" "assistant-ollama") | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podSecurityContext }} + securityContext: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.assistant.ollama.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.assistant.ollama.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ollama + image: "{{ .Values.images.assistantOllama.repository }}:{{ .Values.images.assistantOllama.tag }}" + imagePullPolicy: {{ .Values.images.assistantOllama.pullPolicy }} + {{- with .Values.securityContext.assistantOllama }} + securityContext: {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - { name: http, containerPort: 11434 } + env: + - name: OLLAMA_HOST + value: "0.0.0.0:11434" + # No HTTP probe path is guaranteed beyond `/`, which answers as soon + # as the server is listening — that is exactly what the API waits for. + startupProbe: + httpGet: { path: /, port: 11434 } + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 24 + readinessProbe: + httpGet: { path: /, port: 11434 } + periodSeconds: 30 + livenessProbe: + httpGet: { path: /, port: 11434 } + periodSeconds: 60 + failureThreshold: 3 + volumeMounts: + - name: models + mountPath: /root/.ollama + resources: + requests: + {{- toYaml .Values.assistant.ollama.resources.requests | nindent 14 }} + limits: + {{- toYaml .Values.assistant.ollama.resources.limits | nindent 14 }} + {{- if .Values.assistant.ollama.gpu.enabled }} + # Kubernetes mirrors extended-resource limits into requests, so + # this alone pins the pod to a GPU node advertising the device. + nvidia.com/gpu: {{ .Values.assistant.ollama.gpu.count }} + {{- end }} + volumes: + - name: models + persistentVolumeClaim: + claimName: forail-assistant-ollama-models +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -39,9 +154,9 @@ metadata: labels: {{- include "forail.componentLabels" (dict "root" . "component" "assistant") | nindent 4 }} spec: - # All-in-one image bundles Ollama + ChromaDB; scaling beyond 1 replica - # would duplicate the model cache and split the vector index, so keep - # this at 1 unless the data volume is moved to an external ChromaDB. + # The image bundles ChromaDB as an embedded index on the PVC; scaling beyond + # 1 replica would split the vector index, so keep this at 1 unless the data + # volume is moved to an external ChromaDB. replicas: 1 strategy: type: Recreate @@ -69,12 +184,15 @@ spec: ports: - { name: http, containerPort: 8100 } env: + - name: FORAIL_ASSISTANT_OLLAMA_BASE_URL + value: "http://forail-assistant-ollama.{{ include "forail.namespace" . }}.svc.cluster.local:11434" - name: FORAIL_ASSISTANT_OLLAMA_MODEL value: {{ .Values.assistant.model | quote }} - name: FORAIL_ASSISTANT_LOG_LEVEL value: {{ .Values.assistant.logLevel | quote }} - # First boot pulls the model (gemma3:1b ≈ 800MB) and indexes - # docs; tolerate up to ~5 min before declaring the pod unhealthy. + # First boot waits for Ollama, pulls the model (gemma3:1b ≈ 800MB) + # over its API, and indexes docs; tolerate up to ~5 min before + # declaring the pod unhealthy. startupProbe: httpGet: { path: /api/v1/health, port: 8100 } initialDelaySeconds: 30 diff --git a/values.yaml b/values.yaml index ba588dc..8f5384a 100644 --- a/values.yaml +++ b/values.yaml @@ -28,6 +28,13 @@ images: # it. Bump this in lockstep the next time the assistant is released. tag: "2026.06.0" pullPolicy: IfNotPresent + assistantOllama: + # Model server for the assistant. Pinned on purpose: the assistant image + # used to carry a copy of the Ollama binary taken from `latest`, and an + # upstream layout change broke inference silently. Bump deliberately. + repository: ollama/ollama + tag: "0.30.10" + pullPolicy: IfNotPresent postgres: repository: postgres tag: 15-alpine @@ -199,15 +206,40 @@ assistant: model: "gemma3:1b" logLevel: INFO storage: - # Sized for gemma3:1b + nomic-embed-text + a few thousand chunks of - # docs. Bump to 30Gi+ if switching to llama3.1:8b or larger corpora. - size: 20Gi + # ChromaDB index only — models live on the ollama claim below. Sized for + # a few thousand chunks of docs; the corpus, not the model, drives this. + # NOTE: this dropped from 20Gi when Ollama moved to its own Deployment. + # PVCs cannot shrink, so an existing install needs a fresh claim. + size: 5Gi storageClass: "" resources: - # Ollama + Chroma + uvicorn share this budget. gemma3:1b inference - # peaks around 1.5GB; chunk indexing during first boot briefly spikes. - requests: { memory: 1Gi, cpu: 250m } - limits: { memory: 4Gi, cpu: 2000m } + # Chroma + uvicorn only; inference is no longer in this pod. Indexing + # during first boot is the peak here. + requests: { memory: 512Mi, cpu: 250m } + limits: { memory: 2Gi, cpu: 1000m } + + # ── Model server ──────────────────────────────────────── + # Runs as its own Deployment so that only it needs a GPU and a large volume. + ollama: + gpu: + # Requests nvidia.com/gpu, which pins the pod to a node advertising the + # device — so it stays off until the cluster actually has one and the + # NVIDIA device plugin is installed. Without it Ollama runs on CPU. + enabled: false + count: 1 + nodeSelector: {} + tolerations: [] + storage: + # gemma3:1b (≈800MB) + nomic-embed-text with room to spare. Bump to + # 30Gi+ before switching to an 8B-class model. + size: 20Gi + storageClass: "" + resources: + # CPU inference is memory-hungry and slow; on GPU most of the model + # sits in VRAM instead. Sized for gemma3:1b — raise both alongside + # `assistant.model`. + requests: { memory: 2Gi, cpu: 500m } + limits: { memory: 6Gi, cpu: 4000m } # ── Network policy (needtofix M11) ──────────────────────── # Off by default: k3s' default flannel CNI does not enforce NetworkPolicy, and @@ -230,6 +262,7 @@ securityContext: web: {} frontend: {} assistant: {} + assistantOllama: {} # ── Ingress ─────────────────────────────────────────────── ingress: From c42a338188f820114ce970f442f69abb6e06f5ba Mon Sep 17 00:00:00 2001 From: Krstan Vjestica Date: Sat, 15 Aug 2026 10:25:00 +0200 Subject: [PATCH 2/3] docs: document the assistant's two Deployments and the PVC that cannot shrink The chart README never mentioned the assistant at all, and the split added values nobody could discover from it (images.assistantOllama, assistant.ollama.gpu.*). The storage drop also needs saying out loud: it is the one part of this change that breaks an existing install rather than a fresh one. --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index 1dc4f2b..dd64c7d 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,11 @@ This chart ships **no working secret defaults**: - `podSecurityContext` and per-workload `securityContext.{web,frontend,assistant}` are available for pod hardening (empty by default; validate per image — the frontend binds `:80` and needs `NET_BIND_SERVICE` or a non-root port). +- **`assistant.storage.size` dropped 20Gi → 5Gi** when the model server moved to + its own claim. PVCs cannot shrink, so an existing install with the assistant + enabled keeps its 20Gi claim and the upgrade fails on the immutable field — + delete `forail-assistant-data` (the vector index rebuilds itself) or pin + `--set assistant.storage.size=20Gi`. Fresh installs are unaffected. ## Running jobs in the cluster @@ -80,6 +85,37 @@ in place, all shipped by the chart: The podman-in-pod execution path additionally needs `--set task.privileged=true --set task.hostCgroup=true` (see the secure defaults above). +## AI assistant (optional) + +Off by default (`assistant.enabled=false`). When enabled it renders **two** +Deployments, not one: + +| Workload | Contains | Claim | +|----------|----------|-------| +| `forail-assistant` | FastAPI + embedded ChromaDB | `forail-assistant-data`, 5Gi | +| `forail-assistant-ollama` | the model server, `images.assistantOllama` | `forail-assistant-ollama-models`, 20Gi | + +They are split so that only the model server needs a GPU and a large volume; +the API stays schedulable on any node. Ollama has no authentication, so its +Service is ClusterIP and only the API talks to it. + +```sh +# CPU (default) +helm upgrade forail . -n forail --set assistant.enabled=true + +# GPU — requires a node advertising nvidia.com/gpu and the NVIDIA device plugin +helm upgrade forail . -n forail \ + --set assistant.enabled=true \ + --set assistant.ollama.gpu.enabled=true +``` + +`assistant.ollama.gpu.enabled` requests `nvidia.com/gpu`, which pins that pod +to a node advertising the device — leave it off until the cluster has one, or +the pod stays `Pending`. `images.assistantOllama.tag` is pinned deliberately: +the assistant image used to carry a binary copied out of `ollama/ollama:latest`, +and an upstream layout change broke inference without a line of our code +changing. Bump it on purpose. + ## Layout ``` From a76a61b3d3beb96ddd3cc5a147eca22e6e8a8325 Mon Sep 17 00:00:00 2001 From: Krstan Vjestica Date: Mon, 17 Aug 2026 13:50:00 +0200 Subject: [PATCH 3/3] changelog: record the Ollama split, the pin, and the PVC that has to be recreated The storage change is the only part of this that can fail an upgrade, so it is called out under Breaking with both ways out rather than left as a values.yaml comment. --- CHANGELOG.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 812f9db..e7b9b4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,37 @@ and the chart uses SemVer (`version`) plus the upstream Forail CalVer ## [Unreleased] +### Added +- **`forail-assistant-ollama` Deployment, Service and PVC.** The model server is + no longer part of the assistant image; it runs beside it from + `images.assistantOllama` (`ollama/ollama`, pinned). The Service is ClusterIP — + Ollama has no authentication, so only the API is allowed to reach it. +- `assistant.ollama.gpu.enabled` requests `nvidia.com/gpu` **on that pod alone**, + so only the model server has to land on a GPU node while the API stays + schedulable anywhere. Off by default: the request pins the pod to a node + advertising the device, so without one the pod stays `Pending`. +- `assistant.ollama.{nodeSelector,tolerations,storage,resources}` for placing and + sizing the model server independently of the API. + +### Changed +- `images.assistantOllama` is pinned to `ollama/ollama:0.30.10` rather than + tracking `latest`. The assistant image used to carry a binary copied out of + `latest`, and an upstream layout change broke inference silently — the server + answered `/api/tags` so the health check passed, while every generation + returned 500. Bump this tag deliberately. +- `assistant.resources` drops to `512Mi/250m` requests and `2Gi/1000m` limits: + inference left this pod, so the budget only has to cover Chroma, uvicorn and + the first-boot indexing spike. + +### Breaking +- **`assistant.storage.size` 20Gi → 5Gi.** Model blobs moved to + `forail-assistant-ollama-models` (20Gi), so the index claim is now sized by + the corpus. PVCs cannot shrink: an existing install with `assistant.enabled=true` + will fail the upgrade on the immutable field. Either delete the + `forail-assistant-data` claim — the vector index rebuilds itself from + `docs_to_index/`, nothing irreplaceable is stored there — or keep the old size + with `--set assistant.storage.size=20Gi`. Fresh installs need no action. + ## [2026.7.1] - 2026-07-26 ### Fixed