From ca58e1518c53c72c2037c22d07644b2bf9ebf8f6 Mon Sep 17 00:00:00 2001
From: Brandon Geraci <brandon.geraci@gmail.com>
Date: Wed, 5 Aug 2026 13:48:12 -0500
Subject: [PATCH 1/9] feat: harden Ray cluster isolation (ServiceAccount,
 securityContext, NetworkPolicy, quotas)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Ray has no native auth on the client (10001), GCS (6379), or dashboard/job
API (8265) ports — any pod that can reach them can execute arbitrary code.
This makes NetworkPolicy the in-cluster authorization boundary and asserts
pod-level hardening:

- Dedicated ServiceAccount with automountServiceAccountToken: false (Ray
  never talks to the K8s API).
- Default pod/container securityContext matching the stock rayproject/ray
  runtime user (1000:100), runAsNonRoot, seccomp RuntimeDefault, no
  privilege escalation, all capabilities dropped — restricted-PSS
  compatible; override with {} for root-running custom images. The
  orgCABundle initContainer gets an explicit non-root securityContext in
  the same change so runAsNonRoot doesn't reject it.
- Opt-in NetworkPolicy (networkPolicy.enabled): intra-Ray allow-all +
  kuberay-operator control-plane ports on all Ray pods; gateway
  namespaces and explicitly allow-listed cross-namespace clients on the
  head only. Selectors use ray.io/is-ray-node / ray.io/node-type (never
  ray.io/cluster — RayService generates random-suffixed names, and
  zero-downtime upgrades run two clusters at once). Ingress-only: Ray
  egress is legitimately broad.
- Opt-in ResourceQuota + LimitRange for one-RayService-per-namespace
  isolation; quota must be sized for 2x a cluster (upgrade overlap).
- New head/worker nodeSelector, affinity, priorityClassName values for
  dedicated node-pool pinning.
---
 chart/templates/_helpers.tpl        |  25 +++++++
 chart/templates/limitrange.yaml     |  11 +++
 chart/templates/networkpolicy.yaml  |  97 ++++++++++++++++++++++++
 chart/templates/rayservice.yaml     |  42 +++++++++++
 chart/templates/resourcequota.yaml  |  14 ++++
 chart/templates/serviceaccount.yaml |  10 +++
 chart/values.yaml                   | 111 ++++++++++++++++++++++++++++
 7 files changed, 310 insertions(+)
 create mode 100644 chart/templates/limitrange.yaml
 create mode 100644 chart/templates/networkpolicy.yaml
 create mode 100644 chart/templates/resourcequota.yaml
 create mode 100644 chart/templates/serviceaccount.yaml

diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl
index a918c40..d3b5967 100644
--- a/chart/templates/_helpers.tpl
+++ b/chart/templates/_helpers.tpl
@@ -62,6 +62,17 @@ Ray serve service name - RayService creates a service named <rayservice-name>-se
 {{- printf "%s-serve-svc" (include "nebari-rayserve.fullname" .) }}
 {{- end }}
 
+{{/*
+Name of the ServiceAccount used by the Ray head and worker pods.
+*/}}
+{{- define "nebari-rayserve.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "nebari-rayserve.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
 {{/*
 Whether organization CA bundle injection is enabled.
 */}}
@@ -80,6 +91,20 @@ specs so the SSL_CERT_FILE bundle exists before the main container starts.
 {{- if include "nebari-rayserve.orgCABundle.enabled" . -}}
 - name: build-ca-bundle
   image: {{ .Values.orgCABundle.initImage | quote }}
+  # Explicit non-root securityContext: the default podSecurityContext sets
+  # runAsNonRoot: true, which the kubelet enforces on initContainers too —
+  # alpine would otherwise start as root and be rejected. Reading the
+  # ConfigMap and the system trust store needs no privileges, and the pod's
+  # fsGroup makes the shared emptyDir group-writable. Deliberately NOT
+  # driven by .Values.containerSecurityContext: this container's needs are
+  # fixed and known.
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    runAsGroup: 100
+    allowPrivilegeEscalation: false
+    capabilities:
+      drop: ["ALL"]
   command:
     - sh
     - -c
diff --git a/chart/templates/limitrange.yaml b/chart/templates/limitrange.yaml
new file mode 100644
index 0000000..446ca83
--- /dev/null
+++ b/chart/templates/limitrange.yaml
@@ -0,0 +1,11 @@
+{{- if .Values.limitRange.enabled }}
+apiVersion: v1
+kind: LimitRange
+metadata:
+  name: {{ include "nebari-rayserve.fullname" . }}
+  labels:
+    {{- include "nebari-rayserve.labels" . | nindent 4 }}
+spec:
+  limits:
+    {{- toYaml .Values.limitRange.limits | nindent 4 }}
+{{- end }}
diff --git a/chart/templates/networkpolicy.yaml b/chart/templates/networkpolicy.yaml
new file mode 100644
index 0000000..08a366d
--- /dev/null
+++ b/chart/templates/networkpolicy.yaml
@@ -0,0 +1,97 @@
+{{- if .Values.networkPolicy.enabled }}
+# Ray has no native authentication on the client port (10001), GCS (6379),
+# or dashboard/job API (8265): anything that can reach those ports can run
+# arbitrary code on the cluster. These policies make NetworkPolicy the
+# authorization boundary for in-cluster access.
+#
+# Selectors deliberately use the KubeRay-injected labels ray.io/is-ray-node /
+# ray.io/node-type and NOT ray.io/cluster — RayService generates RayCluster
+# names with a random suffix, and during zero-downtime upgrades the old and
+# new clusters coexist; these selectors cover both.
+#
+# Both policies are Ingress-only. Egress is deliberately unrestricted: Ray
+# workloads legitimately fetch models, datasets, and packages (that is why
+# orgCABundle exists). There is also no namespace-wide default-deny — the
+# policies select only Ray pods, so the colocated kuberay-operator is
+# untouched.
+#
+# Policy 1 — all Ray pods: intra-cluster traffic + operator control plane.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ include "nebari-rayserve.fullname" . }}-ray-cluster
+  labels:
+    {{- include "nebari-rayserve.labels" . | nindent 4 }}
+spec:
+  podSelector:
+    matchLabels:
+      ray.io/is-ray-node: "yes"
+  policyTypes:
+    - Ingress
+  ingress:
+    # Ray inter-node traffic (object manager, node manager, GCS, metrics)
+    # uses many ephemeral ports — allow-all between Ray pods rather than
+    # port-enumerated.
+    - from:
+        - podSelector:
+            matchLabels:
+              ray.io/is-ray-node: "yes"
+    # kuberay-operator (same namespace): dashboard API 8265 (serve config
+    # submission + RayService health checks), dashboard agent 52365,
+    # serve 8000. Without this the RayService never reconciles to Ready.
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: kuberay-operator
+      ports:
+        - port: 8265
+        - port: 52365
+        - port: 8000
+---
+# Policy 2 — head pod only: gateway + allow-listed cross-namespace clients.
+# Unions with policy 1 (the head is also is-ray-node), so the head accepts
+# intra-cluster + operator + gateway + clients; workers accept only
+# intra-cluster + operator.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ include "nebari-rayserve.fullname" . }}-ray-head
+  labels:
+    {{- include "nebari-rayserve.labels" . | nindent 4 }}
+spec:
+  podSelector:
+    matchLabels:
+      ray.io/node-type: head
+  policyTypes:
+    - Ingress
+  ingress:
+    # Envoy Gateway → dashboard (8265) and serve (8000) via the stable
+    # Services. Namespaces are matched by the kubernetes.io/metadata.name
+    # auto-label; verify the actual gateway namespace per cluster.
+    - from:
+        {{- range .Values.networkPolicy.gatewayNamespaces }}
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ . }}
+        {{- end }}
+      ports:
+        - port: 8265
+        - port: 8000
+    {{- range .Values.networkPolicy.allowedClients }}
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ required "networkPolicy.allowedClients entries require a namespace" .namespace }}
+          {{- with .podSelector }}
+          podSelector:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      ports:
+        {{- range (.ports | default (list 10001 8000 8265)) }}
+        - port: {{ . }}
+        {{- end }}
+    {{- end }}
+    {{- with .Values.networkPolicy.extraIngress }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+{{- end }}
diff --git a/chart/templates/rayservice.yaml b/chart/templates/rayservice.yaml
index b2475ee..91961d8 100644
--- a/chart/templates/rayservice.yaml
+++ b/chart/templates/rayservice.yaml
@@ -24,9 +24,26 @@ spec:
         dashboard-host: "0.0.0.0"
       template:
         spec:
+          serviceAccountName: {{ include "nebari-rayserve.serviceAccountName" . }}
+          automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+          {{- with .Values.podSecurityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           {{- with .Values.head.runtimeClassName }}
           runtimeClassName: {{ . }}
           {{- end }}
+          {{- with .Values.head.nodeSelector }}
+          nodeSelector:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.head.affinity }}
+          affinity:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.head.priorityClassName }}
+          priorityClassName: {{ . }}
+          {{- end }}
           {{- with (include "nebari-rayserve.orgCABundle.initContainers" .) }}
           initContainers:
             {{- . | nindent 12 }}
@@ -42,6 +59,10 @@ spec:
           containers:
             - name: ray-head
               image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+              {{- with .Values.containerSecurityContext }}
+              securityContext:
+                {{- toYaml . | nindent 16 }}
+              {{- end }}
               {{- $headEnv := concat (.Values.head.containerEnv | default list) (fromYamlArray (include "nebari-rayserve.orgCABundle.env" .)) }}
               {{- with $headEnv }}
               env:
@@ -78,9 +99,26 @@ spec:
         rayStartParams: {}
         template:
           spec:
+            serviceAccountName: {{ include "nebari-rayserve.serviceAccountName" . }}
+            automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+            {{- with .Values.podSecurityContext }}
+            securityContext:
+              {{- toYaml . | nindent 14 }}
+            {{- end }}
             {{- with .Values.worker.runtimeClassName }}
             runtimeClassName: {{ . }}
             {{- end }}
+            {{- with .Values.worker.nodeSelector }}
+            nodeSelector:
+              {{- toYaml . | nindent 14 }}
+            {{- end }}
+            {{- with .Values.worker.affinity }}
+            affinity:
+              {{- toYaml . | nindent 14 }}
+            {{- end }}
+            {{- with .Values.worker.priorityClassName }}
+            priorityClassName: {{ . }}
+            {{- end }}
             {{- with (include "nebari-rayserve.orgCABundle.initContainers" .) }}
             initContainers:
               {{- . | nindent 14 }}
@@ -96,6 +134,10 @@ spec:
             containers:
               - name: ray-worker
                 image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+                {{- with .Values.containerSecurityContext }}
+                securityContext:
+                  {{- toYaml . | nindent 18 }}
+                {{- end }}
                 {{- $workerEnv := concat (.Values.worker.containerEnv | default list) (fromYamlArray (include "nebari-rayserve.orgCABundle.env" .)) }}
                 {{- with $workerEnv }}
                 env:
diff --git a/chart/templates/resourcequota.yaml b/chart/templates/resourcequota.yaml
new file mode 100644
index 0000000..1a22f6c
--- /dev/null
+++ b/chart/templates/resourcequota.yaml
@@ -0,0 +1,14 @@
+{{- if .Values.resourceQuota.enabled }}
+# Namespace resource cap for the Ray cluster. Size the hard limits for TWO
+# full clusters: RayService zero-downtime upgrades run the old and new
+# RayCluster simultaneously, and a tight quota wedges the upgrade.
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: {{ include "nebari-rayserve.fullname" . }}
+  labels:
+    {{- include "nebari-rayserve.labels" . | nindent 4 }}
+spec:
+  hard:
+    {{- toYaml .Values.resourceQuota.hard | nindent 4 }}
+{{- end }}
diff --git a/chart/templates/serviceaccount.yaml b/chart/templates/serviceaccount.yaml
new file mode 100644
index 0000000..06d0083
--- /dev/null
+++ b/chart/templates/serviceaccount.yaml
@@ -0,0 +1,10 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "nebari-rayserve.serviceAccountName" . }}
+  labels:
+    {{- include "nebari-rayserve.labels" . | nindent 4 }}
+# Ray never talks to the Kubernetes API; don't hand its pods a token.
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}
diff --git a/chart/values.yaml b/chart/values.yaml
index b25c4d7..dccd1e6 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -122,6 +122,109 @@ orgCABundle:
   # Only needs `sh` and `cat` — defaults to a tiny Alpine image.
   initImage: "alpine:3.20"
 
+# =============================================================================
+# Security
+# =============================================================================
+# Dedicated ServiceAccount for the Ray head and worker pods. Ray itself never
+# talks to the Kubernetes API, so the token is not mounted (automount: false)
+# — a compromised Ray job gets no K8s credentials. Set automount: true only
+# if your serve applications genuinely need the API.
+serviceAccount:
+  create: true
+  # Defaults to the chart fullname.
+  name: ""
+  automount: false
+
+# Pod-level securityContext applied to BOTH head and worker pod specs.
+# The defaults assert what the stock rayproject/ray image already does at
+# runtime (user `ray`, uid 1000, gid 100), so default installs behave
+# identically — but the kubelet now enforces it, and the pods satisfy the
+# "restricted" Pod Security Standard. fsGroup keeps the orgCABundle shared
+# emptyDir writable by the non-root initContainer.
+# Running a custom image that needs root? Override with podSecurityContext: {}.
+podSecurityContext:
+  runAsNonRoot: true
+  runAsUser: 1000
+  runAsGroup: 100
+  fsGroup: 100
+  seccompProfile:
+    type: RuntimeDefault
+
+# Container-level securityContext for the ray-head and ray-worker containers.
+# readOnlyRootFilesystem is deliberately NOT set: Ray writes logs, object
+# spilling, and runtime envs under /tmp/ray.
+containerSecurityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop: ["ALL"]
+
+# NetworkPolicy allow-listing for the Ray pods. Ray has NO native
+# authentication on the client port (10001), GCS (6379), or dashboard/job
+# API (8265) — any pod that can reach them can execute arbitrary code on the
+# cluster. With enabled: true, ingress to Ray pods is restricted to:
+#   - other Ray pods in this namespace (inter-node traffic uses many
+#     ephemeral ports, so this is allow-all between Ray pods)
+#   - the kuberay-operator (dashboard API 8265, dashboard agent 52365,
+#     serve 8000 — required or RayService never reconciles)
+#   - the Envoy Gateway namespace(s) below → head 8265 + 8000 (the NebariApp
+#     path for dashboard/serve hostnames)
+#   - explicitly allow-listed cross-namespace clients (see allowedClients)
+# Egress is deliberately NOT restricted: Ray workloads legitimately fetch
+# models, datasets, and packages (that's why orgCABundle exists); the threat
+# model here is inbound access to unauthenticated ports.
+# NOTE: your CNI must enforce NetworkPolicy (Calico, Cilium, ...). kind's
+# default kindnet does NOT.
+networkPolicy:
+  enabled: false
+  # Namespace(s) hosting the Envoy Gateway data plane, matched via the
+  # kubernetes.io/metadata.name auto-label (k8s >= 1.21). MUST be verified
+  # per cluster:  kubectl get pods -A -o wide | grep -i envoy
+  gatewayNamespaces:
+    - envoy-gateway-system
+  # Cross-namespace clients allowed to reach the Ray head. Each entry:
+  #   namespace: <name>          (required — matched via metadata.name label)
+  #   podSelector: {...}         (optional — restricts to matching pods)
+  #   ports: [10001, ...]        (optional — defaults to [10001, 8000, 8265])
+  # Examples:
+  #   - namespace: checkmaite        # checkmaite api: Ray client only
+  #     podSelector:
+  #       matchLabels:
+  #         app.kubernetes.io/component: api
+  #     ports: [10001]
+  #   - namespace: jupyter           # data-science-pack notebooks
+  #     ports: [10001, 8000, 8265]
+  allowedClients: []
+  # Extra raw ingress rules appended verbatim to the head policy (e.g.
+  # Prometheus scraping metrics from a monitoring namespace).
+  extraIngress: []
+
+# Opt-in namespace ResourceQuota. Deploy one RayService per team namespace
+# and cap the blast radius here.
+# WARNING: size for TWO full clusters — RayService zero-downtime upgrades run
+# the old and new RayCluster simultaneously; a tight quota wedges upgrades.
+resourceQuota:
+  enabled: false
+  hard:
+    requests.cpu: "16"
+    requests.memory: 64Gi
+    limits.cpu: "32"
+    limits.memory: 128Gi
+    # requests.nvidia.com/gpu: "4"
+
+# Opt-in namespace LimitRange — default requests/limits for containers that
+# don't set their own (head/worker and the kuberay-operator all set explicit
+# resources, so this only catches ad-hoc pods in the namespace).
+limitRange:
+  enabled: false
+  limits:
+    - type: Container
+      defaultRequest:
+        cpu: 250m
+        memory: 512Mi
+      default:
+        cpu: "1"
+        memory: 2Gi
+
 # =============================================================================
 # Ray Image
 # =============================================================================
@@ -136,6 +239,10 @@ image:
 # =============================================================================
 head:
   # runtimeClassName: nvidia  # Set for GPU-enabled head nodes
+  # Pin the head to a dedicated node pool for scheduling isolation.
+  nodeSelector: {}
+  affinity: {}
+  priorityClassName: ""
   containerEnv: []
   resources:
     limits:
@@ -166,6 +273,10 @@ worker:
   minReplicas: 1
   maxReplicas: 1
   # runtimeClassName: nvidia  # Set for GPU-enabled worker nodes
+  # Pin workers to a dedicated node pool for scheduling isolation.
+  nodeSelector: {}
+  affinity: {}
+  priorityClassName: ""
   containerEnv: []
   resources:
     limits:

From db78bc73a8bd1f5c2f599563af0134735c9dda16 Mon Sep 17 00:00:00 2001
From: Brandon Geraci <brandon.geraci@gmail.com>
Date: Wed, 5 Aug 2026 13:48:28 -0500
Subject: [PATCH 2/9] docs: Security section, ArgoCD ignoreDifferences warning
 for hardening fields, NOTES

- README Security section: threat model (unauthenticated Ray ports;
  Keycloak/NebariApp only covers external hostnames), NetworkPolicy
  design table, allowedClients recipes for checkmaite (10001 only) and
  Jupyter, pod-hardening defaults and how to disable them, quota 2x
  sizing rule, restricted-PSS note.
- Extend the ArgoCD footgun: /spec/rayClusterConfig ignoreDifferences +
  RespectIgnoreDifferences silently drops all new pod-spec hardening
  fields (securityContext, serviceAccountName, automount, nodeSelector,
  affinity, priorityClassName), same as orgCABundle; verify against the
  running pod, not sync status.
- NOTES.txt: print the effective netpol allow-list when enabled; warn
  that Ray ports are unauthenticated when disabled.
---
 README.md                 | 79 ++++++++++++++++++++++++++++++++++++++-
 chart/templates/NOTES.txt | 29 ++++++++++++++
 2 files changed, 107 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a74e09c..8643fb8 100644
--- a/README.md
+++ b/README.md
@@ -123,7 +123,7 @@ spec:
 - `managedNamespaceMetadata` with `nebari.dev/managed: "true"` is required for the nebari-operator to manage NebariApp resources
 - `redirectURI` must be `/oauth2/callback` (Envoy Gateway rejects `/`)
 - Set `serve.enabled: false` to keep the serve endpoint internal-only (recommended — notebooks access it via cluster DNS)
-- The `/spec/rayClusterConfig` ignore rule combined with `RespectIgnoreDifferences=true` makes ArgoCD stop managing everything under `rayClusterConfig`. If you enable [`orgCABundle`](#organization-ca-bundle-injection), the CA injection lives under that path and will **silently not apply** — see the ArgoCD footgun warning in that section before turning it on.
+- The `/spec/rayClusterConfig` ignore rule combined with `RespectIgnoreDifferences=true` makes ArgoCD stop managing everything under `rayClusterConfig`. If you enable [`orgCABundle`](#organization-ca-bundle-injection), the CA injection lives under that path and will **silently not apply** — see the ArgoCD footgun warning in that section before turning it on. The same applies to every pod-spec hardening field this chart injects (`securityContext`, `serviceAccountName`, `automountServiceAccountToken`, `nodeSelector`, `affinity`, `priorityClassName`) — see [Security](#security).
 
 ## Connecting from Jupyter
 
@@ -187,6 +187,66 @@ serveApplications:
 
 The RayService controller handles deployment, health monitoring, and zero-downtime upgrades automatically.
 
+## Security
+
+Ray has **no native authentication** on the client port (10001), GCS (6379), or dashboard/job API (8265): any pod that can reach those ports can execute arbitrary code on the cluster with the cluster's privileges. Keycloak OIDC (via NebariApp) only protects the *external* hostnames — in-cluster traffic bypasses the gateway entirely. This chart therefore treats **NetworkPolicy as the in-cluster authorization boundary**, plus pod hardening and namespace resource isolation.
+
+### NetworkPolicy (opt-in)
+
+```yaml
+networkPolicy:
+  enabled: true
+  # MUST match your cluster's Envoy Gateway data-plane namespace(s) —
+  # verify with: kubectl get pods -A -o wide | grep -i envoy
+  gatewayNamespaces:
+    - envoy-gateway-system
+  allowedClients:
+    # checkmaite api — its ray job backend uses ray.init(ray://...:10001) only
+    - namespace: checkmaite
+      podSelector:
+        matchLabels:
+          app.kubernetes.io/component: api
+      ports: [10001]
+    # Jupyter notebooks (data-science-pack): interactive ray client, serve
+    # endpoint, and the job-submission SDK
+    - namespace: jupyter
+      ports: [10001, 8000, 8265]
+```
+
+What gets rendered (two Ingress-only policies, selecting only Ray pods):
+
+| Traffic | Allowed by |
+|---------|-----------|
+| Ray pod ↔ Ray pod (all ports — inter-node traffic uses ephemeral ports) | `-ray-cluster` policy |
+| kuberay-operator → 8265/52365/8000 (required for RayService reconciliation) | `-ray-cluster` policy |
+| Gateway namespaces → head 8265 (dashboard) + 8000 (serve) | `-ray-head` policy |
+| `allowedClients` entries → head, their listed ports (default `[10001, 8000, 8265]`) | `-ray-head` policy |
+| Everything else → any Ray pod | **denied** |
+
+Egress is deliberately **not** restricted — Ray workloads legitimately fetch models, datasets, and packages (the reason [`orgCABundle`](#organization-ca-bundle-injection) exists). Notes:
+
+- Your CNI must enforce NetworkPolicy (Calico, Cilium, …). kind's default kindnet silently enforces nothing.
+- `podSelector` inside an `allowedClients` entry matches labels on pods in *that* namespace — with one release per namespace, `app.kubernetes.io/component: api` is unambiguous for checkmaite.
+- Extra ingress (e.g. Prometheus scraping) goes in `networkPolicy.extraIngress` as raw rules.
+- NetworkPolicy restricts *who can connect*; traffic is still plaintext and any allowed client is fully trusted by Ray. In-transit mTLS (`RAY_USE_TLS`) is a tracked follow-up.
+
+### Pod hardening (on by default)
+
+`podSecurityContext` / `containerSecurityContext` default to non-root uid 1000 / gid 100, seccomp `RuntimeDefault`, no privilege escalation, all capabilities dropped — exactly what the stock `rayproject/ray` image already does at runtime, so default installs behave identically, but the kubelet now enforces it and the pods satisfy the **restricted** [Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/) (labeling the namespace, e.g. `pod-security.kubernetes.io/enforce: restricted`, is up to your GitOps layer; Ray's `/dev/shm` Memory-medium emptyDir is restricted-compatible). Running a custom image that needs root? Set `podSecurityContext: {}` and `containerSecurityContext: {}`.
+
+Ray pods also get a dedicated ServiceAccount with `automountServiceAccountToken: false` — Ray never talks to the Kubernetes API, so a compromised job finds no K8s credentials.
+
+### Resource isolation (opt-in)
+
+Deploy one RayService per team namespace and cap it: `resourceQuota.enabled=true` + `limitRange.enabled=true`. **Size the quota for two full clusters** — RayService zero-downtime upgrades run old + new RayClusters simultaneously; a tight quota wedges upgrades. Pin to dedicated node pools with the new `head.nodeSelector` / `worker.nodeSelector` / `affinity` / `priorityClassName` values (GPU tolerations are still auto-injected).
+
+> **⚠️ ArgoCD footgun (again):** every pod-spec field above (`securityContext`, `serviceAccountName`, `automountServiceAccountToken`, `nodeSelector`, `affinity`, `priorityClassName`) lives under `/spec/rayClusterConfig`, the exact path the example ArgoCD `Application` tells ArgoCD to stop managing. On such clusters these fields **silently never apply** while ArgoCD reports Synced/Healthy. The top-level NetworkPolicy / ServiceAccount / ResourceQuota / LimitRange resources are unaffected. Narrow the ignore rule as described in the [orgCABundle warning](#organization-ca-bundle-injection), then verify against the running pod, not the sync status:
+>
+> ```bash
+> kubectl get pod -n rayserve -l ray.io/node-type=head \
+>   -o jsonpath='{.items[0].spec.securityContext}'
+> ```
+
 ## Chart Configuration
 
 Key values in `chart/values.yaml`:
@@ -219,6 +279,23 @@ Key values in `chart/values.yaml`:
 | `worker.resources.requests.cpu` | `1` | Worker CPU request |
 | `worker.resources.requests.memory` | `2Gi` | Worker memory request |
 | `worker.runtimeClassName` | - | Runtime class for worker pods (e.g., `nvidia` for GPU) |
+| `head.nodeSelector` / `worker.nodeSelector` | `{}` | Pin pods to a dedicated node pool |
+| `head.affinity` / `worker.affinity` | `{}` | Pod affinity/anti-affinity |
+| `head.priorityClassName` / `worker.priorityClassName` | `""` | PriorityClass for the pods |
+
+### Security
+
+| Value | Default | Description |
+|-------|---------|-------------|
+| `serviceAccount.create` | `true` | Dedicated ServiceAccount for Ray pods |
+| `serviceAccount.automount` | `false` | Mount the SA token (Ray doesn't use the K8s API) |
+| `podSecurityContext` | non-root 1000:100, seccomp | Pod securityContext for head + worker (set `{}` for root images) |
+| `containerSecurityContext` | no privesc, drop ALL | Container securityContext for head + worker |
+| `networkPolicy.enabled` | `false` | Restrict ingress to Ray pods (see [Security](#security)) |
+| `networkPolicy.gatewayNamespaces` | `[envoy-gateway-system]` | Envoy Gateway namespace(s) — **verify per cluster** |
+| `networkPolicy.allowedClients` | `[]` | Cross-namespace clients allowed to reach the head |
+| `resourceQuota.enabled` | `false` | Namespace ResourceQuota (size for 2× a cluster) |
+| `limitRange.enabled` | `false` | Namespace LimitRange defaults |
 
 ### Serve Applications
 
diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt
index 4794381..ae0aeab 100644
--- a/chart/templates/NOTES.txt
+++ b/chart/templates/NOTES.txt
@@ -25,6 +25,35 @@ To access locally:
 
 {{- end }}
 
+=== Security ===
+
+{{- if .Values.networkPolicy.enabled }}
+
+NetworkPolicy is ENABLED. Ingress to Ray pods is restricted to:
+  - other Ray pods and the kuberay-operator (this namespace)
+  - gateway namespace(s): {{ join ", " .Values.networkPolicy.gatewayNamespaces }} -> head 8265/8000
+  {{- if .Values.networkPolicy.allowedClients }}
+  - allowed clients:
+    {{- range .Values.networkPolicy.allowedClients }}
+    - namespace {{ .namespace }} -> ports {{ join ", " (.ports | default (list 10001 8000 8265)) }}
+    {{- end }}
+  {{- else }}
+  - NO cross-namespace clients: ray.init()/job submission from other
+    namespaces (Jupyter, checkmaite, ...) will be BLOCKED until you add
+    networkPolicy.allowedClients entries.
+  {{- end }}
+
+Verify the gateway namespace matches your cluster (kubernetes.io/metadata.name):
+  kubectl get pods -A -o wide | grep -i envoy
+Your CNI must enforce NetworkPolicy (Calico, Cilium, ...); kindnet does not.
+{{- else }}
+
+WARNING: networkPolicy.enabled=false — Ray's client (10001), GCS (6379),
+and dashboard/job API (8265) ports have NO authentication; any pod in the
+cluster can execute code on this Ray cluster. Consider setting
+networkPolicy.enabled=true and allow-listing clients.
+{{- end }}
+
 === Deploying Models ===
 
 Option 1: Declarative (recommended for production)

From 44eb85f5cfd2a25c75e5b2277ff14973e9b69d8e Mon Sep 17 00:00:00 2001
From: Brandon Geraci <brandon.geraci@gmail.com>
Date: Wed, 5 Aug 2026 13:48:29 -0500
Subject: [PATCH 3/9] ci: netpol/quota/scheduling lint permutations,
 Calico-enforced NetworkPolicy kind job
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- lint: template permutations for networkPolicy (+allowedClients render
  assertions), resourceQuota+limitRange, orgCABundle+scheduling (proves
  the initContainer renders non-root alongside the pod runAsNonRoot
  default), and a default-render guard pinning securityContext /
  serviceAccount wiring on both pod specs.
- test: enable networkPolicy on the standalone job (apply-cleanly smoke
  only — kindnet doesn't enforce NetworkPolicy) and add a netpol-enforced
  job: kind with disableDefaultCNI + Calico, asserts both policies exist,
  an unlisted pod is denied at head:8265, an allow-listed namespace gets
  through, and RayService Ready structurally proves the kuberay-operator
  ingress rule.
- test-integration: netpol smoke on the full nebari stack (kindnet,
  non-enforcing — commented as such).
---
 .github/kind-calico.yaml                |   7 ++
 .github/workflows/lint.yaml             |  43 +++++++++
 .github/workflows/test-integration.yaml |   4 +
 .github/workflows/test.yaml             | 112 ++++++++++++++++++++++++
 4 files changed, 166 insertions(+)
 create mode 100644 .github/kind-calico.yaml

diff --git a/.github/kind-calico.yaml b/.github/kind-calico.yaml
new file mode 100644
index 0000000..f989315
--- /dev/null
+++ b/.github/kind-calico.yaml
@@ -0,0 +1,7 @@
+# kind config for the netpol-enforced CI job: disable kindnet (it does not
+# enforce NetworkPolicy) so Calico can be installed as the CNI.
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  disableDefaultCNI: true
+  podSubnet: 192.168.0.0/16   # Calico's default IP pool
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index 4ba8f14..4bc145d 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -40,3 +40,46 @@ jobs:
             --set nebariapp.dashboard.enabled=true \
             --set nebariapp.dashboard.hostname=dashboard.example.com \
             --set nebariapp.auth.enabled=true
+
+      - name: Template chart (NetworkPolicy enabled + allowed client)
+        run: |
+          helm template test chart/ \
+            --set networkPolicy.enabled=true \
+            --set networkPolicy.allowedClients[0].namespace=checkmaite \
+            --set networkPolicy.allowedClients[0].ports[0]=10001 \
+            | tee /tmp/netpol.yaml | grep -q "kind: NetworkPolicy"
+          count=$(grep -c "kind: NetworkPolicy" /tmp/netpol.yaml)
+          test "$count" -eq 2 || { echo "expected 2 NetworkPolicies, got $count"; exit 1; }
+          grep -q "kubernetes.io/metadata.name: checkmaite" /tmp/netpol.yaml
+
+      - name: Template chart (ResourceQuota + LimitRange enabled)
+        run: |
+          helm template test chart/ \
+            --set resourceQuota.enabled=true \
+            --set limitRange.enabled=true \
+            | tee /tmp/quota.yaml >/dev/null
+          grep -q "kind: ResourceQuota" /tmp/quota.yaml
+          grep -q "kind: LimitRange" /tmp/quota.yaml
+
+      - name: Template chart (orgCABundle + scheduling values)
+        run: |
+          helm template test chart/ \
+            --set orgCABundle.configMapName=org-ca \
+            --set head.nodeSelector.pool=gpu \
+            --set head.priorityClassName=high \
+            | tee /tmp/sched.yaml >/dev/null
+          # initContainer must render non-root alongside the pod-level
+          # runAsNonRoot default, or pods are rejected by the kubelet.
+          grep -q "name: build-ca-bundle" /tmp/sched.yaml
+          test "$(grep -c "runAsNonRoot: true" /tmp/sched.yaml)" -ge 4
+          grep -q "pool: gpu" /tmp/sched.yaml
+          grep -q "priorityClassName: high" /tmp/sched.yaml
+
+      - name: Default render regression guard (securityContext asserted)
+        run: |
+          helm template test chart/ | tee /tmp/default.yaml >/dev/null
+          # head pod + worker pod securityContext
+          rayservice=$(helm template test chart/ --show-only templates/rayservice.yaml)
+          test "$(echo "$rayservice" | grep -c "runAsNonRoot: true")" -eq 2
+          test "$(echo "$rayservice" | grep -c "serviceAccountName:")" -eq 2
+          test "$(echo "$rayservice" | grep -c "automountServiceAccountToken: false")" -eq 2
diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml
index 8ba30af..359cd5e 100644
--- a/.github/workflows/test-integration.yaml
+++ b/.github/workflows/test-integration.yaml
@@ -114,10 +114,14 @@ jobs:
           helm install kuberay-operator chart/charts/kuberay-operator-*.tgz \
             --wait --timeout 3m
 
+      # networkPolicy.enabled is smoke-only here: this kind cluster runs
+      # kindnet, which does not enforce NetworkPolicy. Enforcement is covered
+      # by the netpol-enforced job in test.yaml (Calico).
       - name: Deploy Ray Serve with NebariApp
         run: |
           helm install rayserve chart/ \
             --set kuberay-operator.enabled=false \
+            --set networkPolicy.enabled=true \
             --set nebariapp.enabled=true \
             --set nebariapp.hostname=rayserve.nebari.local \
             --set nebariapp.serve.enabled=true \
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index bc84c94..0848050 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -29,11 +29,16 @@ jobs:
           helm install kuberay-operator chart/charts/kuberay-operator-*.tgz \
             --wait --timeout 3m
 
+      # networkPolicy.enabled=true here only proves the policies apply cleanly
+      # and pods still reconcile — kind's default CNI (kindnet) does NOT
+      # enforce NetworkPolicy. Enforcement is proven by the netpol-enforced
+      # job below (Calico).
       - name: Install chart (standalone, no NebariApp)
         run: |
           helm install rayserve chart/ \
             --set kuberay-operator.enabled=false \
             --set nebariapp.enabled=false \
+            --set networkPolicy.enabled=true \
             --set head.resources.requests.cpu=500m \
             --set head.resources.requests.memory=512Mi \
             --set head.resources.limits.cpu=1 \
@@ -105,3 +110,110 @@ jobs:
           echo ""
           echo "=== Events ==="
           kubectl get events --sort-by=.lastTimestamp | tail -30 || true
+
+  # Proves the NetworkPolicies are actually ENFORCED, not just applied:
+  # kindnet ignores NetworkPolicy, so this job swaps in Calico. RayService
+  # reaching Ready also structurally proves the kuberay-operator ingress rule
+  # (operator -> head:8265) is correct — with a wrong rule the RayService
+  # never reconciles.
+  netpol-enforced:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create kind cluster (no default CNI)
+        uses: helm/kind-action@v1
+        with:
+          cluster_name: rayserve-netpol
+          config: .github/kind-calico.yaml
+
+      - name: Install Calico
+        run: |
+          kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/tigera-operator.yaml
+          kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/custom-resources.yaml
+          echo "Waiting for Calico..."
+          for i in $(seq 1 60); do
+            if kubectl get tigerastatus calico -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null | grep -q True; then
+              echo "Calico available"; break
+            fi
+            echo "  attempt $i/60..."; sleep 5
+          done
+          kubectl wait --for=condition=ready node --all --timeout=180s
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+
+      - name: Update chart dependencies
+        run: helm dependency update chart/
+
+      - name: Install KubeRay operator
+        run: |
+          helm install kuberay-operator chart/charts/kuberay-operator-*.tgz \
+            --wait --timeout 3m
+
+      - name: Install chart (NetworkPolicy enabled)
+        run: |
+          helm install rayserve chart/ \
+            --set kuberay-operator.enabled=false \
+            --set nebariapp.enabled=false \
+            --set networkPolicy.enabled=true \
+            --set networkPolicy.gatewayNamespaces[0]=fake-gateway \
+            --set networkPolicy.allowedClients[0].namespace=client-ns \
+            --set networkPolicy.allowedClients[0].ports[0]=8265 \
+            --set head.resources.requests.cpu=500m \
+            --set head.resources.requests.memory=512Mi \
+            --set head.resources.limits.cpu=1 \
+            --set head.resources.limits.memory=1Gi \
+            --set worker.replicas=1 \
+            --set worker.resources.requests.cpu=250m \
+            --set worker.resources.requests.memory=512Mi \
+            --set worker.resources.limits.cpu=500m \
+            --set worker.resources.limits.memory=1Gi \
+            --timeout 5m
+
+      - name: Wait for Ray pods (operator rule proven if Ready)
+        run: |
+          for i in $(seq 1 60); do
+            if kubectl get pod -l ray.io/node-type=head -o name 2>/dev/null | grep -q pod; then
+              break
+            fi
+            echo "  attempt $i/60..."; sleep 5
+          done
+          kubectl wait --for=condition=ready pod -l ray.io/node-type=head --timeout=300s
+          kubectl wait --for=condition=ready pod -l ray.io/node-type=worker --timeout=300s
+
+      - name: Assert policies exist
+        run: |
+          count=$(kubectl get networkpolicy -o name | wc -l)
+          test "$count" -eq 2 || { echo "expected 2 NetworkPolicies, got $count"; exit 1; }
+
+      - name: Denied — unlisted pod cannot reach the dashboard
+        run: |
+          set +e
+          kubectl run deny-test --restart=Never --image=curlimages/curl -- \
+            curl -m 5 -sf http://rayserve-nebari-rayserve-head-svc:8265/api/version
+          kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/deny-test --timeout=60s
+          if [ $? -eq 0 ]; then
+            echo "ERROR: unlisted pod reached the Ray dashboard — policy not enforced"
+            exit 1
+          fi
+          echo "Denied as expected"
+
+      - name: Allowed — allow-listed namespace reaches the dashboard
+        run: |
+          kubectl create namespace client-ns
+          kubectl run allow-test -n client-ns --restart=Never --image=curlimages/curl -- \
+            curl -m 15 --retry 3 -sf http://rayserve-nebari-rayserve-head-svc.default:8265/api/version
+          kubectl wait -n client-ns --for=jsonpath='{.status.phase}'=Succeeded pod/allow-test --timeout=90s
+          echo "Allowed as expected"
+
+      - name: Debug info on failure
+        if: failure()
+        run: |
+          kubectl get pods -A -o wide || true
+          kubectl get networkpolicy -o yaml || true
+          kubectl describe pod deny-test || true
+          kubectl -n client-ns describe pod allow-test || true
+          kubectl logs -l ray.io/node-type=head --tail=50 || true
+          kubectl get events --sort-by=.lastTimestamp | tail -30 || true

From 2ce7dd9085f7b24b716a1071c5618e482fa22fd0 Mon Sep 17 00:00:00 2001
From: Brandon Geraci <brandon.geraci@gmail.com>
Date: Wed, 5 Aug 2026 13:48:29 -0500
Subject: [PATCH 4/9] chore: bump chart to 0.5.0

---
 chart/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/chart/Chart.yaml b/chart/Chart.yaml
index ba07c8b..8f0c259 100644
--- a/chart/Chart.yaml
+++ b/chart/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: nebari-rayserve-pack
 description: A Nebari Software Pack for Ray Serve
 type: application
-version: 0.4.1
+version: 0.5.0
 appVersion: "2.43.0"
 dependencies:
   - name: kuberay-operator

From 794de048ef9e859dd2e84036542f09021e993b9d Mon Sep 17 00:00:00 2001
From: Brandon Geraci <brandon.geraci@gmail.com>
Date: Wed, 5 Aug 2026 13:58:08 -0500
Subject: [PATCH 5/9] docs: link mTLS follow-up issue #29

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8643fb8..e6cb342 100644
--- a/README.md
+++ b/README.md
@@ -228,7 +228,7 @@ Egress is deliberately **not** restricted — Ray workloads legitimately fetch m
 - Your CNI must enforce NetworkPolicy (Calico, Cilium, …). kind's default kindnet silently enforces nothing.
 - `podSelector` inside an `allowedClients` entry matches labels on pods in *that* namespace — with one release per namespace, `app.kubernetes.io/component: api` is unambiguous for checkmaite.
 - Extra ingress (e.g. Prometheus scraping) goes in `networkPolicy.extraIngress` as raw rules.
-- NetworkPolicy restricts *who can connect*; traffic is still plaintext and any allowed client is fully trusted by Ray. In-transit mTLS (`RAY_USE_TLS`) is a tracked follow-up.
+- NetworkPolicy restricts *who can connect*; traffic is still plaintext and any allowed client is fully trusted by Ray. In-transit mTLS (`RAY_USE_TLS`) is a tracked follow-up — see [#29](https://github.com/nebari-dev/rayserve-pack/issues/29).
 
 ### Pod hardening (on by default)
 

From 32cabe94ef95804bb6d31c5a6d8c903e16263cb1 Mon Sep 17 00:00:00 2001
From: Brandon Geraci <brandon.geraci@gmail.com>
Date: Wed, 5 Aug 2026 14:04:01 -0500
Subject: [PATCH 6/9] fix: stale pre-rename service names; keep NetworkPolicy
 out of port-forward jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two CI fixes surfaced by this PR's first run:

- The chart rename to nebari-rayserve-pack (#24) changed the stable
  service names to <release>-nebari-rayserve-pack-{head,serve}-svc, but
  the Test/Integration workflows, README, and dev Makefile still said
  <release>-nebari-rayserve-{head,serve}-svc — the standalone job's
  dashboard health check has been failing on main since 0.4.0. Fix all
  references.

- Recent kind versions enforce NetworkPolicy by default (kindnet ships
  kube-network-policies), so port-forward-based health checks break the
  moment networkPolicy.enabled=true — that traffic matches no allow rule.
  Keep the standalone and integration jobs netpol-OFF baselines and
  concentrate enforcement coverage in the netpol-enforced job, which uses
  in-cluster client pods. Also fix its Calico install racing CRD
  establishment (custom-resources.yaml applied before the Installation
  CRD existed).
---
 .github/workflows/test-integration.yaml | 11 ++++++-----
 .github/workflows/test.yaml             | 20 ++++++++++++--------
 README.md                               | 10 +++++-----
 dev/Makefile                            |  4 ++--
 4 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml
index 359cd5e..8d4528c 100644
--- a/.github/workflows/test-integration.yaml
+++ b/.github/workflows/test-integration.yaml
@@ -114,14 +114,15 @@ jobs:
           helm install kuberay-operator chart/charts/kuberay-operator-*.tgz \
             --wait --timeout 3m
 
-      # networkPolicy.enabled is smoke-only here: this kind cluster runs
-      # kindnet, which does not enforce NetworkPolicy. Enforcement is covered
-      # by the netpol-enforced job in test.yaml (Calico).
+      # networkPolicy stays OFF here: recent kind versions enforce
+      # NetworkPolicy via kube-network-policies, and this job's port-forward
+      # health checks don't match any allow rule. Enforcement coverage lives
+      # in test.yaml's netpol-enforced job; rendering with NebariApp+netpol
+      # combined is covered by the lint permutations.
       - name: Deploy Ray Serve with NebariApp
         run: |
           helm install rayserve chart/ \
             --set kuberay-operator.enabled=false \
-            --set networkPolicy.enabled=true \
             --set nebariapp.enabled=true \
             --set nebariapp.hostname=rayserve.nebari.local \
             --set nebariapp.serve.enabled=true \
@@ -165,7 +166,7 @@ jobs:
 
       - name: Health check Ray Dashboard via port-forward
         run: |
-          kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 &
+          kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 &
           sleep 5
           curl -sf http://localhost:8265/api/version
           kill %1
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 0848050..c9d34e9 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -29,16 +29,16 @@ jobs:
           helm install kuberay-operator chart/charts/kuberay-operator-*.tgz \
             --wait --timeout 3m
 
-      # networkPolicy.enabled=true here only proves the policies apply cleanly
-      # and pods still reconcile — kind's default CNI (kindnet) does NOT
-      # enforce NetworkPolicy. Enforcement is proven by the netpol-enforced
-      # job below (Calico).
+      # Baseline job: networkPolicy stays OFF here so the port-forward health
+      # check exercises the unrestricted path (recent kind versions enforce
+      # NetworkPolicy via kube-network-policies, which would block traffic
+      # that doesn't match a rule). NetworkPolicy coverage lives in the
+      # netpol-enforced job below.
       - name: Install chart (standalone, no NebariApp)
         run: |
           helm install rayserve chart/ \
             --set kuberay-operator.enabled=false \
             --set nebariapp.enabled=false \
-            --set networkPolicy.enabled=true \
             --set head.resources.requests.cpu=500m \
             --set head.resources.requests.memory=512Mi \
             --set head.resources.limits.cpu=1 \
@@ -83,7 +83,7 @@ jobs:
 
       - name: Health check Ray Dashboard
         run: |
-          kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 &
+          kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 &
           sleep 5
           curl -sf http://localhost:8265/api/version
           kill %1
@@ -131,6 +131,10 @@ jobs:
       - name: Install Calico
         run: |
           kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/tigera-operator.yaml
+          # custom-resources.yaml needs the operator's CRDs registered first —
+          # applying immediately races CRD establishment and fails with
+          # "no matches for kind Installation".
+          kubectl wait --for condition=established crd/installations.operator.tigera.io --timeout=120s
           kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/custom-resources.yaml
           echo "Waiting for Calico..."
           for i in $(seq 1 60); do
@@ -192,7 +196,7 @@ jobs:
         run: |
           set +e
           kubectl run deny-test --restart=Never --image=curlimages/curl -- \
-            curl -m 5 -sf http://rayserve-nebari-rayserve-head-svc:8265/api/version
+            curl -m 5 -sf http://rayserve-nebari-rayserve-pack-head-svc:8265/api/version
           kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/deny-test --timeout=60s
           if [ $? -eq 0 ]; then
             echo "ERROR: unlisted pod reached the Ray dashboard — policy not enforced"
@@ -204,7 +208,7 @@ jobs:
         run: |
           kubectl create namespace client-ns
           kubectl run allow-test -n client-ns --restart=Never --image=curlimages/curl -- \
-            curl -m 15 --retry 3 -sf http://rayserve-nebari-rayserve-head-svc.default:8265/api/version
+            curl -m 15 --retry 3 -sf http://rayserve-nebari-rayserve-pack-head-svc.default:8265/api/version
           kubectl wait -n client-ns --for=jsonpath='{.status.phase}'=Succeeded pod/allow-test --timeout=90s
           echo "Allowed as expected"
 
diff --git a/README.md b/README.md
index e6cb342..b257314 100644
--- a/README.md
+++ b/README.md
@@ -40,10 +40,10 @@ Access via port-forward:
 
 ```bash
 # Ray Dashboard
-kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 -n rayserve
+kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 -n rayserve
 
 # Ray Serve endpoint
-kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000 -n rayserve
+kubectl port-forward svc/rayserve-nebari-rayserve-pack-serve-svc 8000:8000 -n rayserve
 ```
 
 ### On a Nebari cluster (via ArgoCD)
@@ -135,7 +135,7 @@ from ray import serve
 import requests
 
 # Connect to the Ray cluster
-ray.init("ray://rayserve-nebari-rayserve-head-svc.rayserve.svc.cluster.local:10001")
+ray.init("ray://rayserve-nebari-rayserve-pack-head-svc.rayserve.svc.cluster.local:10001")
 
 # Deploy a model
 @serve.deployment
@@ -146,7 +146,7 @@ class Hello:
 serve.run(Hello.bind(), name="hello", route_prefix="/hello")
 
 # Run inference
-resp = requests.get("http://rayserve-nebari-rayserve-serve-svc.rayserve.svc.cluster.local:8000/hello")
+resp = requests.get("http://rayserve-nebari-rayserve-pack-serve-svc.rayserve.svc.cluster.local:8000/hello")
 print(resp.text)
 # Hello from Ray Serve!
 ```
@@ -418,7 +418,7 @@ The NebariApp may be pointing at a service that doesn't exist. Check the actual
 kubectl get svc -n rayserve
 ```
 
-The stable services are `<release>-nebari-rayserve-head-svc` and `<release>-nebari-rayserve-serve-svc`.
+The stable services are `<release>-nebari-rayserve-pack-head-svc` and `<release>-nebari-rayserve-pack-serve-svc`.
 
 ### Version mismatch connecting from Jupyter
 
diff --git a/dev/Makefile b/dev/Makefile
index 24702dd..207f5c3 100644
--- a/dev/Makefile
+++ b/dev/Makefile
@@ -144,8 +144,8 @@ up-standalone:
 	@echo "Ray Serve deployed (standalone, no NebariApp)."
 	@echo ""
 	@echo "Access via port-forward:"
-	@echo "  kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000"
-	@echo "  kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265"
+	@echo "  kubectl port-forward svc/rayserve-nebari-rayserve-pack-serve-svc 8000:8000"
+	@echo "  kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265"
 
 # --------------------------------------------------------------------------
 # update-hosts - update /etc/hosts with all NebariApp hostnames

From 2042fb483a4db65952dbfe863750c630a5a8d1a7 Mon Sep 17 00:00:00 2001
From: Brandon Geraci <brandon.geraci@gmail.com>
Date: Wed, 5 Aug 2026 14:14:22 -0500
Subject: [PATCH 7/9] fix: integration workflow asserted pre-rename NebariApp
 names

Same #24-rename fallout as the service names: the NebariApp resources are
rayserve-nebari-rayserve-pack{,-dashboard} since 0.4.0, but the integration
workflow (failing on main since then) and dev Makefile still waited on the
old names.
---
 .github/workflows/test-integration.yaml | 4 ++--
 dev/Makefile                            | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml
index 8d4528c..2fa4ba2 100644
--- a/.github/workflows/test-integration.yaml
+++ b/.github/workflows/test-integration.yaml
@@ -149,10 +149,10 @@ jobs:
           kubectl wait --for=condition=ready pod -l ray.io/node-type=head --timeout=300s
 
       - name: Verify NebariApp (serve)
-        run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve --timeout=300s
+        run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack --timeout=300s
 
       - name: Verify NebariApp (dashboard)
-        run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-dashboard --timeout=300s
+        run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack-dashboard --timeout=300s
 
       - name: Verify HTTPRoutes created
         run: |
diff --git a/dev/Makefile b/dev/Makefile
index 207f5c3..7a72bb6 100644
--- a/dev/Makefile
+++ b/dev/Makefile
@@ -122,7 +122,7 @@ up: cluster
 		--set nebariapp.dashboard.enabled=true \
 		--set nebariapp.dashboard.hostname=$(DASH_HOSTNAME) \
 		--wait --timeout 5m
-	kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve --timeout=180s
+	kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack --timeout=180s
 	CLUSTER_NAME=$(CLUSTER_NAME) $(OPERATOR_REPO)/dev/scripts/networking/update-hosts.sh
 	@echo ""
 	@echo "Ray Serve deployed with NebariApp."

From 14229a977b4e0df7286c3e7be2c12dfc716e213d Mon Sep 17 00:00:00 2001
From: Brandon Geraci <brandon.geraci@gmail.com>
Date: Wed, 5 Aug 2026 14:23:11 -0500
Subject: [PATCH 8/9] =?UTF-8?q?ci:=20retry=20the=20netpol=20allow-path=20t?=
 =?UTF-8?q?est=20=E2=80=94=20Calico=20lags=20new-namespace=20label=20match?=
 =?UTF-8?q?es?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The allow-listed-namespace curl ran once immediately after creating
client-ns; Calico's dataplane can take a few seconds to propagate the
namespaceSelector match for a brand-new namespace, making the one-shot
check flaky (passed run 1, failed run 2). Retry with fresh pods via
kubectl run --rm --attach, which propagates curl's exit code directly.
---
 .github/workflows/test.yaml | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index c9d34e9..1c5aeda 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -204,13 +204,24 @@ jobs:
           fi
           echo "Denied as expected"
 
+      # Retried with fresh pods: Calico's dataplane can lag a few seconds
+      # behind a newly created namespace before the namespaceSelector match
+      # takes effect, so a single one-shot curl is flaky.
       - name: Allowed — allow-listed namespace reaches the dashboard
         run: |
           kubectl create namespace client-ns
-          kubectl run allow-test -n client-ns --restart=Never --image=curlimages/curl -- \
-            curl -m 15 --retry 3 -sf http://rayserve-nebari-rayserve-pack-head-svc.default:8265/api/version
-          kubectl wait -n client-ns --for=jsonpath='{.status.phase}'=Succeeded pod/allow-test --timeout=90s
-          echo "Allowed as expected"
+          for i in $(seq 1 6); do
+            if kubectl run allow-test-$i -n client-ns --restart=Never --rm --attach \
+                 --image=curlimages/curl -- \
+                 curl -m 10 -sf http://rayserve-nebari-rayserve-pack-head-svc.default:8265/api/version; then
+              echo "Allowed as expected (attempt $i)"
+              exit 0
+            fi
+            echo "attempt $i failed, retrying..."
+            sleep 10
+          done
+          echo "ERROR: allow-listed namespace never reached the Ray dashboard"
+          exit 1
 
       - name: Debug info on failure
         if: failure()

From 94422d7ed29b3a22459cf070e22fcdf8eac71954 Mon Sep 17 00:00:00 2001
From: Brandon Geraci <brandon.geraci@gmail.com>
Date: Wed, 5 Aug 2026 14:38:09 -0500
Subject: [PATCH 9/9] ci: carry install-time overrides through the auth-upgrade
 step

The upgrade step only re-set the nebariapp flags, silently reverting
head/worker resources and worker.replicas to chart defaults. That spec
change triggers a RayService zero-downtime cluster replacement (old + new
clusters concurrently) that doesn't reliably fit the runner, so
helm --wait timed out intermittently. The auth flip is not supposed to
touch the Ray cluster; repeating the overrides makes rayClusterConfig a
no-op on upgrade.
---
 .github/workflows/test-integration.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml
index 2fa4ba2..db79544 100644
--- a/.github/workflows/test-integration.yaml
+++ b/.github/workflows/test-integration.yaml
@@ -174,6 +174,11 @@ jobs:
       # ============================================================
       # Test: Upgrade to enable auth
       # ============================================================
+      # Must repeat the install step's resource/replica overrides: omitting
+      # them reverts the RayService to chart-default sizing, which triggers a
+      # zero-downtime cluster replacement (old + new clusters running at
+      # once) that doesn't reliably fit the runner and times out --wait. The
+      # auth flip should not touch the Ray cluster at all.
       - name: Upgrade with auth enabled
         run: |
           helm upgrade rayserve chart/ \
@@ -184,6 +189,11 @@ jobs:
             --set nebariapp.dashboard.enabled=true \
             --set nebariapp.dashboard.hostname=ray-dashboard.nebari.local \
             --set nebariapp.auth.enabled=true \
+            --set head.resources.requests.cpu=500m \
+            --set head.resources.requests.memory=512Mi \
+            --set head.resources.limits.cpu=1 \
+            --set head.resources.limits.memory=1Gi \
+            --set worker.replicas=0 \
             --wait --timeout 5m
 
       - name: Verify SecurityPolicies created