diff --git a/.github/kind-calico.yaml b/.github/kind-calico.yaml new file mode 100644 index 0000000..f989315 --- /dev/null +++ b/.github/kind-calico.yaml @@ -0,0 +1,7 @@ +# kind config for the netpol-enforced CI job: disable kindnet (it does not +# enforce NetworkPolicy) so Calico can be installed as the CNI. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + disableDefaultCNI: true + podSubnet: 192.168.0.0/16 # Calico's default IP pool diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 4ba8f14..4bc145d 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -40,3 +40,46 @@ jobs: --set nebariapp.dashboard.enabled=true \ --set nebariapp.dashboard.hostname=dashboard.example.com \ --set nebariapp.auth.enabled=true + + - name: Template chart (NetworkPolicy enabled + allowed client) + run: | + helm template test chart/ \ + --set networkPolicy.enabled=true \ + --set networkPolicy.allowedClients[0].namespace=checkmaite \ + --set networkPolicy.allowedClients[0].ports[0]=10001 \ + | tee /tmp/netpol.yaml | grep -q "kind: NetworkPolicy" + count=$(grep -c "kind: NetworkPolicy" /tmp/netpol.yaml) + test "$count" -eq 2 || { echo "expected 2 NetworkPolicies, got $count"; exit 1; } + grep -q "kubernetes.io/metadata.name: checkmaite" /tmp/netpol.yaml + + - name: Template chart (ResourceQuota + LimitRange enabled) + run: | + helm template test chart/ \ + --set resourceQuota.enabled=true \ + --set limitRange.enabled=true \ + | tee /tmp/quota.yaml >/dev/null + grep -q "kind: ResourceQuota" /tmp/quota.yaml + grep -q "kind: LimitRange" /tmp/quota.yaml + + - name: Template chart (orgCABundle + scheduling values) + run: | + helm template test chart/ \ + --set orgCABundle.configMapName=org-ca \ + --set head.nodeSelector.pool=gpu \ + --set head.priorityClassName=high \ + | tee /tmp/sched.yaml >/dev/null + # initContainer must render non-root alongside the pod-level + # runAsNonRoot default, or pods are rejected by the kubelet. + grep -q "name: build-ca-bundle" /tmp/sched.yaml + test "$(grep -c "runAsNonRoot: true" /tmp/sched.yaml)" -ge 4 + grep -q "pool: gpu" /tmp/sched.yaml + grep -q "priorityClassName: high" /tmp/sched.yaml + + - name: Default render regression guard (securityContext asserted) + run: | + helm template test chart/ | tee /tmp/default.yaml >/dev/null + # head pod + worker pod securityContext + rayservice=$(helm template test chart/ --show-only templates/rayservice.yaml) + test "$(echo "$rayservice" | grep -c "runAsNonRoot: true")" -eq 2 + test "$(echo "$rayservice" | grep -c "serviceAccountName:")" -eq 2 + test "$(echo "$rayservice" | grep -c "automountServiceAccountToken: false")" -eq 2 diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index 8ba30af..db79544 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -114,6 +114,11 @@ jobs: helm install kuberay-operator chart/charts/kuberay-operator-*.tgz \ --wait --timeout 3m + # networkPolicy stays OFF here: recent kind versions enforce + # NetworkPolicy via kube-network-policies, and this job's port-forward + # health checks don't match any allow rule. Enforcement coverage lives + # in test.yaml's netpol-enforced job; rendering with NebariApp+netpol + # combined is covered by the lint permutations. - name: Deploy Ray Serve with NebariApp run: | helm install rayserve chart/ \ @@ -144,10 +149,10 @@ jobs: kubectl wait --for=condition=ready pod -l ray.io/node-type=head --timeout=300s - name: Verify NebariApp (serve) - run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve --timeout=300s + run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack --timeout=300s - name: Verify NebariApp (dashboard) - run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-dashboard --timeout=300s + run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack-dashboard --timeout=300s - name: Verify HTTPRoutes created run: | @@ -161,7 +166,7 @@ jobs: - name: Health check Ray Dashboard via port-forward run: | - kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 & + kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 & sleep 5 curl -sf http://localhost:8265/api/version kill %1 @@ -169,6 +174,11 @@ jobs: # ============================================================ # Test: Upgrade to enable auth # ============================================================ + # Must repeat the install step's resource/replica overrides: omitting + # them reverts the RayService to chart-default sizing, which triggers a + # zero-downtime cluster replacement (old + new clusters running at + # once) that doesn't reliably fit the runner and times out --wait. The + # auth flip should not touch the Ray cluster at all. - name: Upgrade with auth enabled run: | helm upgrade rayserve chart/ \ @@ -179,6 +189,11 @@ jobs: --set nebariapp.dashboard.enabled=true \ --set nebariapp.dashboard.hostname=ray-dashboard.nebari.local \ --set nebariapp.auth.enabled=true \ + --set head.resources.requests.cpu=500m \ + --set head.resources.requests.memory=512Mi \ + --set head.resources.limits.cpu=1 \ + --set head.resources.limits.memory=1Gi \ + --set worker.replicas=0 \ --wait --timeout 5m - name: Verify SecurityPolicies created diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bc84c94..1c5aeda 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -29,6 +29,11 @@ jobs: helm install kuberay-operator chart/charts/kuberay-operator-*.tgz \ --wait --timeout 3m + # Baseline job: networkPolicy stays OFF here so the port-forward health + # check exercises the unrestricted path (recent kind versions enforce + # NetworkPolicy via kube-network-policies, which would block traffic + # that doesn't match a rule). NetworkPolicy coverage lives in the + # netpol-enforced job below. - name: Install chart (standalone, no NebariApp) run: | helm install rayserve chart/ \ @@ -78,7 +83,7 @@ jobs: - name: Health check Ray Dashboard run: | - kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 & + kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 & sleep 5 curl -sf http://localhost:8265/api/version kill %1 @@ -105,3 +110,125 @@ jobs: echo "" echo "=== Events ===" kubectl get events --sort-by=.lastTimestamp | tail -30 || true + + # Proves the NetworkPolicies are actually ENFORCED, not just applied: + # kindnet ignores NetworkPolicy, so this job swaps in Calico. RayService + # reaching Ready also structurally proves the kuberay-operator ingress rule + # (operator -> head:8265) is correct — with a wrong rule the RayService + # never reconciles. + netpol-enforced: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + + - name: Create kind cluster (no default CNI) + uses: helm/kind-action@v1 + with: + cluster_name: rayserve-netpol + config: .github/kind-calico.yaml + + - name: Install Calico + run: | + kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/tigera-operator.yaml + # custom-resources.yaml needs the operator's CRDs registered first — + # applying immediately races CRD establishment and fails with + # "no matches for kind Installation". + kubectl wait --for condition=established crd/installations.operator.tigera.io --timeout=120s + kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/custom-resources.yaml + echo "Waiting for Calico..." + for i in $(seq 1 60); do + if kubectl get tigerastatus calico -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null | grep -q True; then + echo "Calico available"; break + fi + echo " attempt $i/60..."; sleep 5 + done + kubectl wait --for=condition=ready node --all --timeout=180s + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Update chart dependencies + run: helm dependency update chart/ + + - name: Install KubeRay operator + run: | + helm install kuberay-operator chart/charts/kuberay-operator-*.tgz \ + --wait --timeout 3m + + - name: Install chart (NetworkPolicy enabled) + run: | + helm install rayserve chart/ \ + --set kuberay-operator.enabled=false \ + --set nebariapp.enabled=false \ + --set networkPolicy.enabled=true \ + --set networkPolicy.gatewayNamespaces[0]=fake-gateway \ + --set networkPolicy.allowedClients[0].namespace=client-ns \ + --set networkPolicy.allowedClients[0].ports[0]=8265 \ + --set head.resources.requests.cpu=500m \ + --set head.resources.requests.memory=512Mi \ + --set head.resources.limits.cpu=1 \ + --set head.resources.limits.memory=1Gi \ + --set worker.replicas=1 \ + --set worker.resources.requests.cpu=250m \ + --set worker.resources.requests.memory=512Mi \ + --set worker.resources.limits.cpu=500m \ + --set worker.resources.limits.memory=1Gi \ + --timeout 5m + + - name: Wait for Ray pods (operator rule proven if Ready) + run: | + for i in $(seq 1 60); do + if kubectl get pod -l ray.io/node-type=head -o name 2>/dev/null | grep -q pod; then + break + fi + echo " attempt $i/60..."; sleep 5 + done + kubectl wait --for=condition=ready pod -l ray.io/node-type=head --timeout=300s + kubectl wait --for=condition=ready pod -l ray.io/node-type=worker --timeout=300s + + - name: Assert policies exist + run: | + count=$(kubectl get networkpolicy -o name | wc -l) + test "$count" -eq 2 || { echo "expected 2 NetworkPolicies, got $count"; exit 1; } + + - name: Denied — unlisted pod cannot reach the dashboard + run: | + set +e + kubectl run deny-test --restart=Never --image=curlimages/curl -- \ + curl -m 5 -sf http://rayserve-nebari-rayserve-pack-head-svc:8265/api/version + kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/deny-test --timeout=60s + if [ $? -eq 0 ]; then + echo "ERROR: unlisted pod reached the Ray dashboard — policy not enforced" + exit 1 + fi + echo "Denied as expected" + + # Retried with fresh pods: Calico's dataplane can lag a few seconds + # behind a newly created namespace before the namespaceSelector match + # takes effect, so a single one-shot curl is flaky. + - name: Allowed — allow-listed namespace reaches the dashboard + run: | + kubectl create namespace client-ns + for i in $(seq 1 6); do + if kubectl run allow-test-$i -n client-ns --restart=Never --rm --attach \ + --image=curlimages/curl -- \ + curl -m 10 -sf http://rayserve-nebari-rayserve-pack-head-svc.default:8265/api/version; then + echo "Allowed as expected (attempt $i)" + exit 0 + fi + echo "attempt $i failed, retrying..." + sleep 10 + done + echo "ERROR: allow-listed namespace never reached the Ray dashboard" + exit 1 + + - name: Debug info on failure + if: failure() + run: | + kubectl get pods -A -o wide || true + kubectl get networkpolicy -o yaml || true + kubectl describe pod deny-test || true + kubectl -n client-ns describe pod allow-test || true + kubectl logs -l ray.io/node-type=head --tail=50 || true + kubectl get events --sort-by=.lastTimestamp | tail -30 || true diff --git a/README.md b/README.md index a74e09c..b257314 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,10 @@ Access via port-forward: ```bash # Ray Dashboard -kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 -n rayserve +kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 -n rayserve # Ray Serve endpoint -kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000 -n rayserve +kubectl port-forward svc/rayserve-nebari-rayserve-pack-serve-svc 8000:8000 -n rayserve ``` ### On a Nebari cluster (via ArgoCD) @@ -123,7 +123,7 @@ spec: - `managedNamespaceMetadata` with `nebari.dev/managed: "true"` is required for the nebari-operator to manage NebariApp resources - `redirectURI` must be `/oauth2/callback` (Envoy Gateway rejects `/`) - Set `serve.enabled: false` to keep the serve endpoint internal-only (recommended — notebooks access it via cluster DNS) -- The `/spec/rayClusterConfig` ignore rule combined with `RespectIgnoreDifferences=true` makes ArgoCD stop managing everything under `rayClusterConfig`. If you enable [`orgCABundle`](#organization-ca-bundle-injection), the CA injection lives under that path and will **silently not apply** — see the ArgoCD footgun warning in that section before turning it on. +- The `/spec/rayClusterConfig` ignore rule combined with `RespectIgnoreDifferences=true` makes ArgoCD stop managing everything under `rayClusterConfig`. If you enable [`orgCABundle`](#organization-ca-bundle-injection), the CA injection lives under that path and will **silently not apply** — see the ArgoCD footgun warning in that section before turning it on. The same applies to every pod-spec hardening field this chart injects (`securityContext`, `serviceAccountName`, `automountServiceAccountToken`, `nodeSelector`, `affinity`, `priorityClassName`) — see [Security](#security). ## Connecting from Jupyter @@ -135,7 +135,7 @@ from ray import serve import requests # Connect to the Ray cluster -ray.init("ray://rayserve-nebari-rayserve-head-svc.rayserve.svc.cluster.local:10001") +ray.init("ray://rayserve-nebari-rayserve-pack-head-svc.rayserve.svc.cluster.local:10001") # Deploy a model @serve.deployment @@ -146,7 +146,7 @@ class Hello: serve.run(Hello.bind(), name="hello", route_prefix="/hello") # Run inference -resp = requests.get("http://rayserve-nebari-rayserve-serve-svc.rayserve.svc.cluster.local:8000/hello") +resp = requests.get("http://rayserve-nebari-rayserve-pack-serve-svc.rayserve.svc.cluster.local:8000/hello") print(resp.text) # Hello from Ray Serve! ``` @@ -187,6 +187,66 @@ serveApplications: The RayService controller handles deployment, health monitoring, and zero-downtime upgrades automatically. +## Security + +Ray has **no native authentication** on the client port (10001), GCS (6379), or dashboard/job API (8265): any pod that can reach those ports can execute arbitrary code on the cluster with the cluster's privileges. Keycloak OIDC (via NebariApp) only protects the *external* hostnames — in-cluster traffic bypasses the gateway entirely. This chart therefore treats **NetworkPolicy as the in-cluster authorization boundary**, plus pod hardening and namespace resource isolation. + +### NetworkPolicy (opt-in) + +```yaml +networkPolicy: + enabled: true + # MUST match your cluster's Envoy Gateway data-plane namespace(s) — + # verify with: kubectl get pods -A -o wide | grep -i envoy + gatewayNamespaces: + - envoy-gateway-system + allowedClients: + # checkmaite api — its ray job backend uses ray.init(ray://...:10001) only + - namespace: checkmaite + podSelector: + matchLabels: + app.kubernetes.io/component: api + ports: [10001] + # Jupyter notebooks (data-science-pack): interactive ray client, serve + # endpoint, and the job-submission SDK + - namespace: jupyter + ports: [10001, 8000, 8265] +``` + +What gets rendered (two Ingress-only policies, selecting only Ray pods): + +| Traffic | Allowed by | +|---------|-----------| +| Ray pod ↔ Ray pod (all ports — inter-node traffic uses ephemeral ports) | `-ray-cluster` policy | +| kuberay-operator → 8265/52365/8000 (required for RayService reconciliation) | `-ray-cluster` policy | +| Gateway namespaces → head 8265 (dashboard) + 8000 (serve) | `-ray-head` policy | +| `allowedClients` entries → head, their listed ports (default `[10001, 8000, 8265]`) | `-ray-head` policy | +| Everything else → any Ray pod | **denied** | + +Egress is deliberately **not** restricted — Ray workloads legitimately fetch models, datasets, and packages (the reason [`orgCABundle`](#organization-ca-bundle-injection) exists). Notes: + +- Your CNI must enforce NetworkPolicy (Calico, Cilium, …). kind's default kindnet silently enforces nothing. +- `podSelector` inside an `allowedClients` entry matches labels on pods in *that* namespace — with one release per namespace, `app.kubernetes.io/component: api` is unambiguous for checkmaite. +- Extra ingress (e.g. Prometheus scraping) goes in `networkPolicy.extraIngress` as raw rules. +- NetworkPolicy restricts *who can connect*; traffic is still plaintext and any allowed client is fully trusted by Ray. In-transit mTLS (`RAY_USE_TLS`) is a tracked follow-up — see [#29](https://github.com/nebari-dev/rayserve-pack/issues/29). + +### Pod hardening (on by default) + +`podSecurityContext` / `containerSecurityContext` default to non-root uid 1000 / gid 100, seccomp `RuntimeDefault`, no privilege escalation, all capabilities dropped — exactly what the stock `rayproject/ray` image already does at runtime, so default installs behave identically, but the kubelet now enforces it and the pods satisfy the **restricted** [Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/) (labeling the namespace, e.g. `pod-security.kubernetes.io/enforce: restricted`, is up to your GitOps layer; Ray's `/dev/shm` Memory-medium emptyDir is restricted-compatible). Running a custom image that needs root? Set `podSecurityContext: {}` and `containerSecurityContext: {}`. + +Ray pods also get a dedicated ServiceAccount with `automountServiceAccountToken: false` — Ray never talks to the Kubernetes API, so a compromised job finds no K8s credentials. + +### Resource isolation (opt-in) + +Deploy one RayService per team namespace and cap it: `resourceQuota.enabled=true` + `limitRange.enabled=true`. **Size the quota for two full clusters** — RayService zero-downtime upgrades run old + new RayClusters simultaneously; a tight quota wedges upgrades. Pin to dedicated node pools with the new `head.nodeSelector` / `worker.nodeSelector` / `affinity` / `priorityClassName` values (GPU tolerations are still auto-injected). + +> **⚠️ ArgoCD footgun (again):** every pod-spec field above (`securityContext`, `serviceAccountName`, `automountServiceAccountToken`, `nodeSelector`, `affinity`, `priorityClassName`) lives under `/spec/rayClusterConfig`, the exact path the example ArgoCD `Application` tells ArgoCD to stop managing. On such clusters these fields **silently never apply** while ArgoCD reports Synced/Healthy. The top-level NetworkPolicy / ServiceAccount / ResourceQuota / LimitRange resources are unaffected. Narrow the ignore rule as described in the [orgCABundle warning](#organization-ca-bundle-injection), then verify against the running pod, not the sync status: +> +> ```bash +> kubectl get pod -n rayserve -l ray.io/node-type=head \ +> -o jsonpath='{.items[0].spec.securityContext}' +> ``` + ## Chart Configuration Key values in `chart/values.yaml`: @@ -219,6 +279,23 @@ Key values in `chart/values.yaml`: | `worker.resources.requests.cpu` | `1` | Worker CPU request | | `worker.resources.requests.memory` | `2Gi` | Worker memory request | | `worker.runtimeClassName` | - | Runtime class for worker pods (e.g., `nvidia` for GPU) | +| `head.nodeSelector` / `worker.nodeSelector` | `{}` | Pin pods to a dedicated node pool | +| `head.affinity` / `worker.affinity` | `{}` | Pod affinity/anti-affinity | +| `head.priorityClassName` / `worker.priorityClassName` | `""` | PriorityClass for the pods | + +### Security + +| Value | Default | Description | +|-------|---------|-------------| +| `serviceAccount.create` | `true` | Dedicated ServiceAccount for Ray pods | +| `serviceAccount.automount` | `false` | Mount the SA token (Ray doesn't use the K8s API) | +| `podSecurityContext` | non-root 1000:100, seccomp | Pod securityContext for head + worker (set `{}` for root images) | +| `containerSecurityContext` | no privesc, drop ALL | Container securityContext for head + worker | +| `networkPolicy.enabled` | `false` | Restrict ingress to Ray pods (see [Security](#security)) | +| `networkPolicy.gatewayNamespaces` | `[envoy-gateway-system]` | Envoy Gateway namespace(s) — **verify per cluster** | +| `networkPolicy.allowedClients` | `[]` | Cross-namespace clients allowed to reach the head | +| `resourceQuota.enabled` | `false` | Namespace ResourceQuota (size for 2× a cluster) | +| `limitRange.enabled` | `false` | Namespace LimitRange defaults | ### Serve Applications @@ -341,7 +418,7 @@ The NebariApp may be pointing at a service that doesn't exist. Check the actual kubectl get svc -n rayserve ``` -The stable services are `-nebari-rayserve-head-svc` and `-nebari-rayserve-serve-svc`. +The stable services are `-nebari-rayserve-pack-head-svc` and `-nebari-rayserve-pack-serve-svc`. ### Version mismatch connecting from Jupyter diff --git a/chart/Chart.yaml b/chart/Chart.yaml index ba07c8b..8f0c259 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: nebari-rayserve-pack description: A Nebari Software Pack for Ray Serve type: application -version: 0.4.1 +version: 0.5.0 appVersion: "2.43.0" dependencies: - name: kuberay-operator diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index 4794381..ae0aeab 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -25,6 +25,35 @@ To access locally: {{- end }} +=== Security === + +{{- if .Values.networkPolicy.enabled }} + +NetworkPolicy is ENABLED. Ingress to Ray pods is restricted to: + - other Ray pods and the kuberay-operator (this namespace) + - gateway namespace(s): {{ join ", " .Values.networkPolicy.gatewayNamespaces }} -> head 8265/8000 + {{- if .Values.networkPolicy.allowedClients }} + - allowed clients: + {{- range .Values.networkPolicy.allowedClients }} + - namespace {{ .namespace }} -> ports {{ join ", " (.ports | default (list 10001 8000 8265)) }} + {{- end }} + {{- else }} + - NO cross-namespace clients: ray.init()/job submission from other + namespaces (Jupyter, checkmaite, ...) will be BLOCKED until you add + networkPolicy.allowedClients entries. + {{- end }} + +Verify the gateway namespace matches your cluster (kubernetes.io/metadata.name): + kubectl get pods -A -o wide | grep -i envoy +Your CNI must enforce NetworkPolicy (Calico, Cilium, ...); kindnet does not. +{{- else }} + +WARNING: networkPolicy.enabled=false — Ray's client (10001), GCS (6379), +and dashboard/job API (8265) ports have NO authentication; any pod in the +cluster can execute code on this Ray cluster. Consider setting +networkPolicy.enabled=true and allow-listing clients. +{{- end }} + === Deploying Models === Option 1: Declarative (recommended for production) diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index a918c40..d3b5967 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -62,6 +62,17 @@ Ray serve service name - RayService creates a service named -se {{- printf "%s-serve-svc" (include "nebari-rayserve.fullname" .) }} {{- end }} +{{/* +Name of the ServiceAccount used by the Ray head and worker pods. +*/}} +{{- define "nebari-rayserve.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "nebari-rayserve.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + {{/* Whether organization CA bundle injection is enabled. */}} @@ -80,6 +91,20 @@ specs so the SSL_CERT_FILE bundle exists before the main container starts. {{- if include "nebari-rayserve.orgCABundle.enabled" . -}} - name: build-ca-bundle image: {{ .Values.orgCABundle.initImage | quote }} + # Explicit non-root securityContext: the default podSecurityContext sets + # runAsNonRoot: true, which the kubelet enforces on initContainers too — + # alpine would otherwise start as root and be rejected. Reading the + # ConfigMap and the system trust store needs no privileges, and the pod's + # fsGroup makes the shared emptyDir group-writable. Deliberately NOT + # driven by .Values.containerSecurityContext: this container's needs are + # fixed and known. + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 100 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] command: - sh - -c diff --git a/chart/templates/limitrange.yaml b/chart/templates/limitrange.yaml new file mode 100644 index 0000000..446ca83 --- /dev/null +++ b/chart/templates/limitrange.yaml @@ -0,0 +1,11 @@ +{{- if .Values.limitRange.enabled }} +apiVersion: v1 +kind: LimitRange +metadata: + name: {{ include "nebari-rayserve.fullname" . }} + labels: + {{- include "nebari-rayserve.labels" . | nindent 4 }} +spec: + limits: + {{- toYaml .Values.limitRange.limits | nindent 4 }} +{{- end }} diff --git a/chart/templates/networkpolicy.yaml b/chart/templates/networkpolicy.yaml new file mode 100644 index 0000000..08a366d --- /dev/null +++ b/chart/templates/networkpolicy.yaml @@ -0,0 +1,97 @@ +{{- if .Values.networkPolicy.enabled }} +# Ray has no native authentication on the client port (10001), GCS (6379), +# or dashboard/job API (8265): anything that can reach those ports can run +# arbitrary code on the cluster. These policies make NetworkPolicy the +# authorization boundary for in-cluster access. +# +# Selectors deliberately use the KubeRay-injected labels ray.io/is-ray-node / +# ray.io/node-type and NOT ray.io/cluster — RayService generates RayCluster +# names with a random suffix, and during zero-downtime upgrades the old and +# new clusters coexist; these selectors cover both. +# +# Both policies are Ingress-only. Egress is deliberately unrestricted: Ray +# workloads legitimately fetch models, datasets, and packages (that is why +# orgCABundle exists). There is also no namespace-wide default-deny — the +# policies select only Ray pods, so the colocated kuberay-operator is +# untouched. +# +# Policy 1 — all Ray pods: intra-cluster traffic + operator control plane. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "nebari-rayserve.fullname" . }}-ray-cluster + labels: + {{- include "nebari-rayserve.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + ray.io/is-ray-node: "yes" + policyTypes: + - Ingress + ingress: + # Ray inter-node traffic (object manager, node manager, GCS, metrics) + # uses many ephemeral ports — allow-all between Ray pods rather than + # port-enumerated. + - from: + - podSelector: + matchLabels: + ray.io/is-ray-node: "yes" + # kuberay-operator (same namespace): dashboard API 8265 (serve config + # submission + RayService health checks), dashboard agent 52365, + # serve 8000. Without this the RayService never reconciles to Ready. + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: kuberay-operator + ports: + - port: 8265 + - port: 52365 + - port: 8000 +--- +# Policy 2 — head pod only: gateway + allow-listed cross-namespace clients. +# Unions with policy 1 (the head is also is-ray-node), so the head accepts +# intra-cluster + operator + gateway + clients; workers accept only +# intra-cluster + operator. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "nebari-rayserve.fullname" . }}-ray-head + labels: + {{- include "nebari-rayserve.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + ray.io/node-type: head + policyTypes: + - Ingress + ingress: + # Envoy Gateway → dashboard (8265) and serve (8000) via the stable + # Services. Namespaces are matched by the kubernetes.io/metadata.name + # auto-label; verify the actual gateway namespace per cluster. + - from: + {{- range .Values.networkPolicy.gatewayNamespaces }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ . }} + {{- end }} + ports: + - port: 8265 + - port: 8000 + {{- range .Values.networkPolicy.allowedClients }} + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ required "networkPolicy.allowedClients entries require a namespace" .namespace }} + {{- with .podSelector }} + podSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + {{- range (.ports | default (list 10001 8000 8265)) }} + - port: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.networkPolicy.extraIngress }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/templates/rayservice.yaml b/chart/templates/rayservice.yaml index b2475ee..91961d8 100644 --- a/chart/templates/rayservice.yaml +++ b/chart/templates/rayservice.yaml @@ -24,9 +24,26 @@ spec: dashboard-host: "0.0.0.0" template: spec: + serviceAccountName: {{ include "nebari-rayserve.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automount }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} {{- with .Values.head.runtimeClassName }} runtimeClassName: {{ . }} {{- end }} + {{- with .Values.head.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.head.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.head.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} {{- with (include "nebari-rayserve.orgCABundle.initContainers" .) }} initContainers: {{- . | nindent 12 }} @@ -42,6 +59,10 @@ spec: containers: - name: ray-head image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 16 }} + {{- end }} {{- $headEnv := concat (.Values.head.containerEnv | default list) (fromYamlArray (include "nebari-rayserve.orgCABundle.env" .)) }} {{- with $headEnv }} env: @@ -78,9 +99,26 @@ spec: rayStartParams: {} template: spec: + serviceAccountName: {{ include "nebari-rayserve.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automount }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 14 }} + {{- end }} {{- with .Values.worker.runtimeClassName }} runtimeClassName: {{ . }} {{- end }} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.worker.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} {{- with (include "nebari-rayserve.orgCABundle.initContainers" .) }} initContainers: {{- . | nindent 14 }} @@ -96,6 +134,10 @@ spec: containers: - name: ray-worker image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 18 }} + {{- end }} {{- $workerEnv := concat (.Values.worker.containerEnv | default list) (fromYamlArray (include "nebari-rayserve.orgCABundle.env" .)) }} {{- with $workerEnv }} env: diff --git a/chart/templates/resourcequota.yaml b/chart/templates/resourcequota.yaml new file mode 100644 index 0000000..1a22f6c --- /dev/null +++ b/chart/templates/resourcequota.yaml @@ -0,0 +1,14 @@ +{{- if .Values.resourceQuota.enabled }} +# Namespace resource cap for the Ray cluster. Size the hard limits for TWO +# full clusters: RayService zero-downtime upgrades run the old and new +# RayCluster simultaneously, and a tight quota wedges the upgrade. +apiVersion: v1 +kind: ResourceQuota +metadata: + name: {{ include "nebari-rayserve.fullname" . }} + labels: + {{- include "nebari-rayserve.labels" . | nindent 4 }} +spec: + hard: + {{- toYaml .Values.resourceQuota.hard | nindent 4 }} +{{- end }} diff --git a/chart/templates/serviceaccount.yaml b/chart/templates/serviceaccount.yaml new file mode 100644 index 0000000..06d0083 --- /dev/null +++ b/chart/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nebari-rayserve.serviceAccountName" . }} + labels: + {{- include "nebari-rayserve.labels" . | nindent 4 }} +# Ray never talks to the Kubernetes API; don't hand its pods a token. +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index b25c4d7..dccd1e6 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -122,6 +122,109 @@ orgCABundle: # Only needs `sh` and `cat` — defaults to a tiny Alpine image. initImage: "alpine:3.20" +# ============================================================================= +# Security +# ============================================================================= +# Dedicated ServiceAccount for the Ray head and worker pods. Ray itself never +# talks to the Kubernetes API, so the token is not mounted (automount: false) +# — a compromised Ray job gets no K8s credentials. Set automount: true only +# if your serve applications genuinely need the API. +serviceAccount: + create: true + # Defaults to the chart fullname. + name: "" + automount: false + +# Pod-level securityContext applied to BOTH head and worker pod specs. +# The defaults assert what the stock rayproject/ray image already does at +# runtime (user `ray`, uid 1000, gid 100), so default installs behave +# identically — but the kubelet now enforces it, and the pods satisfy the +# "restricted" Pod Security Standard. fsGroup keeps the orgCABundle shared +# emptyDir writable by the non-root initContainer. +# Running a custom image that needs root? Override with podSecurityContext: {}. +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 100 + fsGroup: 100 + seccompProfile: + type: RuntimeDefault + +# Container-level securityContext for the ray-head and ray-worker containers. +# readOnlyRootFilesystem is deliberately NOT set: Ray writes logs, object +# spilling, and runtime envs under /tmp/ray. +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + +# NetworkPolicy allow-listing for the Ray pods. Ray has NO native +# authentication on the client port (10001), GCS (6379), or dashboard/job +# API (8265) — any pod that can reach them can execute arbitrary code on the +# cluster. With enabled: true, ingress to Ray pods is restricted to: +# - other Ray pods in this namespace (inter-node traffic uses many +# ephemeral ports, so this is allow-all between Ray pods) +# - the kuberay-operator (dashboard API 8265, dashboard agent 52365, +# serve 8000 — required or RayService never reconciles) +# - the Envoy Gateway namespace(s) below → head 8265 + 8000 (the NebariApp +# path for dashboard/serve hostnames) +# - explicitly allow-listed cross-namespace clients (see allowedClients) +# Egress is deliberately NOT restricted: Ray workloads legitimately fetch +# models, datasets, and packages (that's why orgCABundle exists); the threat +# model here is inbound access to unauthenticated ports. +# NOTE: your CNI must enforce NetworkPolicy (Calico, Cilium, ...). kind's +# default kindnet does NOT. +networkPolicy: + enabled: false + # Namespace(s) hosting the Envoy Gateway data plane, matched via the + # kubernetes.io/metadata.name auto-label (k8s >= 1.21). MUST be verified + # per cluster: kubectl get pods -A -o wide | grep -i envoy + gatewayNamespaces: + - envoy-gateway-system + # Cross-namespace clients allowed to reach the Ray head. Each entry: + # namespace: (required — matched via metadata.name label) + # podSelector: {...} (optional — restricts to matching pods) + # ports: [10001, ...] (optional — defaults to [10001, 8000, 8265]) + # Examples: + # - namespace: checkmaite # checkmaite api: Ray client only + # podSelector: + # matchLabels: + # app.kubernetes.io/component: api + # ports: [10001] + # - namespace: jupyter # data-science-pack notebooks + # ports: [10001, 8000, 8265] + allowedClients: [] + # Extra raw ingress rules appended verbatim to the head policy (e.g. + # Prometheus scraping metrics from a monitoring namespace). + extraIngress: [] + +# Opt-in namespace ResourceQuota. Deploy one RayService per team namespace +# and cap the blast radius here. +# WARNING: size for TWO full clusters — RayService zero-downtime upgrades run +# the old and new RayCluster simultaneously; a tight quota wedges upgrades. +resourceQuota: + enabled: false + hard: + requests.cpu: "16" + requests.memory: 64Gi + limits.cpu: "32" + limits.memory: 128Gi + # requests.nvidia.com/gpu: "4" + +# Opt-in namespace LimitRange — default requests/limits for containers that +# don't set their own (head/worker and the kuberay-operator all set explicit +# resources, so this only catches ad-hoc pods in the namespace). +limitRange: + enabled: false + limits: + - type: Container + defaultRequest: + cpu: 250m + memory: 512Mi + default: + cpu: "1" + memory: 2Gi + # ============================================================================= # Ray Image # ============================================================================= @@ -136,6 +239,10 @@ image: # ============================================================================= head: # runtimeClassName: nvidia # Set for GPU-enabled head nodes + # Pin the head to a dedicated node pool for scheduling isolation. + nodeSelector: {} + affinity: {} + priorityClassName: "" containerEnv: [] resources: limits: @@ -166,6 +273,10 @@ worker: minReplicas: 1 maxReplicas: 1 # runtimeClassName: nvidia # Set for GPU-enabled worker nodes + # Pin workers to a dedicated node pool for scheduling isolation. + nodeSelector: {} + affinity: {} + priorityClassName: "" containerEnv: [] resources: limits: diff --git a/dev/Makefile b/dev/Makefile index 24702dd..7a72bb6 100644 --- a/dev/Makefile +++ b/dev/Makefile @@ -122,7 +122,7 @@ up: cluster --set nebariapp.dashboard.enabled=true \ --set nebariapp.dashboard.hostname=$(DASH_HOSTNAME) \ --wait --timeout 5m - kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve --timeout=180s + kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack --timeout=180s CLUSTER_NAME=$(CLUSTER_NAME) $(OPERATOR_REPO)/dev/scripts/networking/update-hosts.sh @echo "" @echo "Ray Serve deployed with NebariApp." @@ -144,8 +144,8 @@ up-standalone: @echo "Ray Serve deployed (standalone, no NebariApp)." @echo "" @echo "Access via port-forward:" - @echo " kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000" - @echo " kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265" + @echo " kubectl port-forward svc/rayserve-nebari-rayserve-pack-serve-svc 8000:8000" + @echo " kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265" # -------------------------------------------------------------------------- # update-hosts - update /etc/hosts with all NebariApp hostnames