From 1b7019b5e3189bf2bebacbf5b135943812530541 Mon Sep 17 00:00:00 2001 From: Carles Figuerola Date: Fri, 30 Oct 2026 15:14:03 -0500 Subject: [PATCH] fix(helm): Add extra values to widen deployment support Signed-off-by: Carles Figuerola --- charts/substrate-podcert/README.md | 4 + .../templates/pod-certificate-controller.yaml | 9 +- charts/substrate-podcert/values.yaml | 5 + charts/substrate/README.md | 16 ++ .../substrate/templates/ate-api-server.yaml | 28 +++ .../substrate/templates/ate-controller.yaml | 7 + .../substrate/templates/ate-idp-server.yaml | 7 + charts/substrate/templates/atelet.yaml | 21 +- charts/substrate/templates/atenet-egress.yaml | 7 + charts/substrate/templates/atenet-router.yaml | 6 + .../templates/k8s-credential-provider.yaml | 6 + charts/substrate/templates/rustfs.yaml | 50 ++++ .../tests/image_credential_provider_test.yaml | 69 +++++ charts/substrate/tests/pod_metadata_test.yaml | 235 ++++++++++++++++++ charts/substrate/tests/postgres_test.yaml | 37 +++ charts/substrate/values.yaml | 43 ++++ 16 files changed, 541 insertions(+), 9 deletions(-) create mode 100644 charts/substrate/tests/image_credential_provider_test.yaml create mode 100644 charts/substrate/tests/pod_metadata_test.yaml diff --git a/charts/substrate-podcert/README.md b/charts/substrate-podcert/README.md index 368142e2f5..25d90d8a2a 100644 --- a/charts/substrate-podcert/README.md +++ b/charts/substrate-podcert/README.md @@ -43,6 +43,10 @@ chart over resources still owned by the application release fails Helm ownership validation. Perform the release split during a planned installation update; do not delete the existing CA-pool Secrets. +The controller Deployment is named like its ServiceAccount (`podcertificate-controller`, +prefixed by other release names). `podLabels` and `podAnnotations` add labels and +annotations to its pod template. + Set `resources.requests` and `resources.limits` on this release to size the controller container. Defaults are `10m` CPU and `64Mi` memory requests, with `500m` CPU and `256Mi` memory limits. For example: diff --git a/charts/substrate-podcert/templates/pod-certificate-controller.yaml b/charts/substrate-podcert/templates/pod-certificate-controller.yaml index 80284bb991..422b9c2357 100644 --- a/charts/substrate-podcert/templates/pod-certificate-controller.yaml +++ b/charts/substrate-podcert/templates/pod-certificate-controller.yaml @@ -136,7 +136,7 @@ subjects: apiVersion: apps/v1 kind: Deployment metadata: - name: podcertificate-controller + name: {{ include "podcert.fullname" (list "podcertificate-controller" .) }} namespace: {{ .Release.Namespace }} labels: app: podcertificate-controller @@ -149,6 +149,13 @@ spec: metadata: labels: app: podcertificate-controller + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: {{- with include "podcert.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: diff --git a/charts/substrate-podcert/values.yaml b/charts/substrate-podcert/values.yaml index 30674c848f..ce71f46a09 100644 --- a/charts/substrate-podcert/values.yaml +++ b/charts/substrate-podcert/values.yaml @@ -43,3 +43,8 @@ image: registry: ghcr.io repository: kagent-dev/substrate tag: "" + +# Labels added to the controller pod template. +podLabels: {} +# Annotations added to the controller pod template. +podAnnotations: {} diff --git a/charts/substrate/README.md b/charts/substrate/README.md index f80beb3809..2955eaef0b 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -56,16 +56,28 @@ See `values.yaml` for the full set; the important keys: | `postgres.ownerConnectionStringSecretRef` | `substrate-postgres-owner` | Read the owner connection from a pre-created Secret | | `postgres.readWriteRole` | `substrate_readwrite` | Role assumed by read/write connections | | `postgres.ownerRole` | `substrate_owner` | Role assumed by owner connections | +| `postgres.readWriteRoleSecretRef` | `{}` | `{name, key}` of a Secret key holding the read/write role name, instead of `readWriteRole`; `key` defaults to `username` | +| `postgres.ownerRoleSecretRef` | `{}` | Same for the owner role, instead of `ownerRole` | | `preview` | `[]` | Preview gates for the API server, controller and workers, and atelet; set `[ExternalVolumes]` to enable external volume APIs and mounts | | `postgres.schema` | `substrate` | Store the Substrate tables in this PostgreSQL schema | | `postgres.clientCertificates.enabled` | `false` | Project separate owner and runtime login certificates into the API server | | `.resources` | see `values.yaml` | Requests and limits for each container, e.g. `ateApi.resources` and `atenetRouter.agentgateway.resources` | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `rustfs.storageClassName` | `""` | StorageClass of the RustFS data claim; empty uses the cluster default | +| `rustfs.strategy` | `{}` | RustFS Deployment strategy; set `{type: Recreate}` because the data volume is ReadWriteOnce | +| `rustfs.bucketInit.annotations` | `{}` | Annotations on the bucket-init Job, e.g. a GitOps hook so the immutable Job is replaced on change | +| `rustfs.bucketInit.dedicatedServiceAccount` | `false` | Run the bucket-init Job under its own ServiceAccount named like the Job | +| `podLabels` | `{}` | Labels added to every pod template, e.g. `sidecar.istio.io/inject: "false"` | +| `podAnnotations` | `{}` | Annotations added to every pod template, e.g. node placement hints | +| `commonLabels` | `{}` | Labels added to the RustFS PersistentVolumeClaim | | `ateApi.snapshotLocation` | `""` | Cluster-wide snapshot URI; defaults to `gs:///` with bundled RustFS | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.imageCredentialProviderConfig` | `""` | Host path to the kubelet credential provider config; set together with the bin directory | | `atelet.imageCredentialProviderConfigYAML` | `""` | Credential provider config as text, rendered into a ConfigMap, for nodes where that file cannot be authored or must differ from the kubelet's; mutually exclusive with the host path | | `atelet.imageCredentialProviderBinDir` | `""` | Host path to the kubelet credential provider binaries; both paths are mounted read-only | +| `atelet.extraVolumes` | `[]` | Extra volumes on the atelet pod, e.g. a `kubernetes.io/dockerconfigjson` Secret a plugin reads | +| `atelet.extraVolumeMounts` | `[]` | Extra volume mounts on the atelet container | +| `atelet.tolerations` | sandbox-class and GKE ComputeClass taints | Tolerations of the atelet pods, replaced as a whole; set to what the cluster's admission controllers produce if they rewrite DaemonSet tolerations | | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider | | `ateApi.actorJWTIssuer` | `https://..svc` | Issuer placed in actor JWTs; set a stable URL reachable by relying parties | | `objectStorePlugin.extraEnv` | `[]` | Object storage credentials/settings shared by both object-store plugins | @@ -203,6 +215,10 @@ certificates for the bundled development database. For an operator-managed database, the operator must provision the identities, schema, and grants, and create both connection Secrets in the release namespace. +When the provisioner generates and rotates the login and that login is itself the +role to assume, set `postgres.readWriteRoleSecretRef` and `postgres.ownerRoleSecretRef` +to the Secret key that holds the login name instead of fixed `readWriteRole` and +`ownerRole` values; the API server reads the role names from those keys at start. Substrate runs `SET ROLE` for each new connection. It rejects a login without the required membership. diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 8089720d3e..0b439b6484 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -77,7 +77,13 @@ spec: metadata: labels: app: ate-api-server + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} checksum/api-config: {{ .Values.ateApi.logLevel | sha256sum }} prometheus.io/scrape: "true" prometheus.io/port: "9090" @@ -161,8 +167,16 @@ spec: - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" - "--postgres-read-write-connection-string=@env" - "--postgres-owner-connection-string=@env" +{{- if .Values.postgres.readWriteRoleSecretRef.name }} + - "--postgres-read-write-role=$(ATE_API_POSTGRES_READ_WRITE_ROLE)" +{{- else }} - {{ printf "--postgres-read-write-role=%s" .Values.postgres.readWriteRole | quote }} +{{- end }} +{{- if .Values.postgres.ownerRoleSecretRef.name }} + - "--postgres-owner-role=$(ATE_API_POSTGRES_OWNER_ROLE)" +{{- else }} - {{ printf "--postgres-owner-role=%s" .Values.postgres.ownerRole | quote }} +{{- end }} - "--postgres-schema=@env" - "--actor-jwt-issuer=@env" - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" @@ -205,6 +219,20 @@ spec: secretKeyRef: name: {{ include "substrate.postgres.ownerSecretName" . | quote }} key: {{ get $ownerConnectionStringSecretRef "key" | default "ownerConnectionString" | quote }} +{{- with .Values.postgres.readWriteRoleSecretRef }}{{- if .name }} + - name: ATE_API_POSTGRES_READ_WRITE_ROLE + valueFrom: + secretKeyRef: + name: {{ .name | quote }} + key: {{ .key | default "username" | quote }} +{{- end }}{{- end }} +{{- with .Values.postgres.ownerRoleSecretRef }}{{- if .name }} + - name: ATE_API_POSTGRES_OWNER_ROLE + valueFrom: + secretKeyRef: + name: {{ .name | quote }} + key: {{ .key | default "username" | quote }} +{{- end }}{{- end }} {{- with include "substrate.otel.env" . }} {{- . | trim | nindent 8 }} {{- end }} diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 7ac38e2409..3c10d73348 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -79,6 +79,13 @@ spec: metadata: labels: app: ate-controller + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "substrate.fullname" (list "ate-controller" .) }} {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} diff --git a/charts/substrate/templates/ate-idp-server.yaml b/charts/substrate/templates/ate-idp-server.yaml index 62f09032c3..1733f2a091 100644 --- a/charts/substrate/templates/ate-idp-server.yaml +++ b/charts/substrate/templates/ate-idp-server.yaml @@ -39,6 +39,13 @@ spec: metadata: labels: app: ate-idp-server + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} serviceAccountName: {{ include "substrate.fullname" (list "ate-idp-server" .) }} diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 5abad9d1c5..651703e3dd 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -126,21 +126,20 @@ spec: metadata: labels: app: atelet + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} prometheus.io/scrape: "true" prometheus.io/port: "9090" spec: serviceAccountName: {{ include "substrate.fullname" (list "atelet" .) }} priorityClassName: {{ include "substrate.fullname" (list "ate-node-critical" .) }} - # Run on nodes reserved for any WorkerPool sandbox class. tolerations: - - key: ate.dev/sandboxClass - operator: Exists - effect: NoSchedule - # GKE reserves ComputeClass nodes with this taint. - - key: cloud.google.com/compute-class - operator: Exists - effect: NoSchedule +{{ toYaml .Values.atelet.tolerations | indent 6 }} {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} initContainers: - name: objectstore-plugin @@ -297,6 +296,9 @@ spec: subPath: config.yaml {{- end }} readOnly: true +{{- end }} +{{- with .Values.atelet.extraVolumeMounts }} +{{ toYaml . | indent 8 }} {{- end }} volumes: - name: objectstore-plugin @@ -358,3 +360,6 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem +{{- with .Values.atelet.extraVolumes }} +{{ toYaml . | indent 6 }} +{{- end }} diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 76cb93ad50..3c43df9553 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -132,6 +132,13 @@ spec: metadata: labels: app: atenet-egress + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "substrate.fullname" (list "atenet-egress" .) }} securityContext: diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 761386f4b6..76a34f51d3 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -177,7 +177,13 @@ spec: metadata: labels: app: atenet-router + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} prometheus.io/scrape: "true" prometheus.io/port: "15020" spec: diff --git a/charts/substrate/templates/k8s-credential-provider.yaml b/charts/substrate/templates/k8s-credential-provider.yaml index 718e60879c..838a880923 100644 --- a/charts/substrate/templates/k8s-credential-provider.yaml +++ b/charts/substrate/templates/k8s-credential-provider.yaml @@ -62,9 +62,15 @@ spec: template: metadata: annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} checksum/namespace-policy: {{ toJson .Values.credentialProvider.namespacePolicies | sha256sum }} labels: app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml index ab04cb1a20..4bb99bfee3 100644 --- a/charts/substrate/templates/rustfs.yaml +++ b/charts/substrate/templates/rustfs.yaml @@ -26,7 +26,14 @@ kind: PersistentVolumeClaim metadata: name: {{ include "substrate.fullname" (list "rustfs-data" .) }} namespace: {{ .Release.Namespace }} + {{- with .Values.commonLabels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} spec: + {{- with .Values.rustfs.storageClassName }} + storageClassName: {{ . | quote }} + {{- end }} accessModes: - ReadWriteOnce resources: @@ -56,6 +63,10 @@ metadata: name: {{ include "substrate.fullname" (list "rustfs" .) }} namespace: {{ .Release.Namespace }} spec: + {{- with .Values.rustfs.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} replicas: 1 selector: matchLabels: @@ -64,6 +75,13 @@ spec: metadata: labels: app: rustfs + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "substrate.fullname" (list "rustfs" .) }} securityContext: @@ -108,12 +126,31 @@ kind: Job metadata: name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} namespace: {{ .Release.Namespace }} + {{- with .Values.rustfs.bucketInit.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: backoffLimit: 10 template: + {{- if or .Values.podLabels .Values.podAnnotations }} + metadata: + {{- with .Values.podLabels }} + labels: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} spec: restartPolicy: OnFailure + {{- if .Values.rustfs.bucketInit.dedicatedServiceAccount }} + serviceAccountName: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} + {{- else }} serviceAccountName: {{ include "substrate.fullname" (list "rustfs" .) }} + {{- end }} {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: create-bucket @@ -176,3 +213,16 @@ spec: - protocol: TCP port: api {{- end }} +{{- if .Values.rustfs.bucketInit.dedicatedServiceAccount }} +--- +# A ServiceAccount named like the Job, for admission policies that require it. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} + namespace: {{ .Release.Namespace }} + {{- with .Values.rustfs.bucketInit.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/substrate/tests/image_credential_provider_test.yaml b/charts/substrate/tests/image_credential_provider_test.yaml new file mode 100644 index 0000000000..0d38d35dd6 --- /dev/null +++ b/charts/substrate/tests/image_credential_provider_test.yaml @@ -0,0 +1,69 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +suite: atelet image credential provider +templates: +- atelet.yaml +release: + name: custom + namespace: custom-system +tests: +- it: renders no credential provider wiring by default + documentSelector: + path: kind + value: DaemonSet + asserts: + - notContains: + path: spec.template.spec.containers[0].args + content: --image-credential-provider-config=/run/image-credential-provider/config.yaml + - notExists: + path: spec.template.spec.volumes[?(@.name == "image-credential-provider-bin")] + +- it: mounts the node's provider config and binaries from host paths + documentSelector: + path: kind + value: DaemonSet + set: + atelet: + imageCredentialProviderConfig: /etc/srv/kubernetes/cri_auth_config.yaml + imageCredentialProviderBinDir: /home/kubernetes/bin + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --image-credential-provider-bin-dir=/run/image-credential-provider/bin + - equal: + path: spec.template.spec.volumes[?(@.name == "image-credential-provider-bin")].hostPath.path + value: /home/kubernetes/bin + +- it: appends extraVolumes and extraVolumeMounts to atelet + documentSelector: + path: kind + value: DaemonSet + set: + atelet: + extraVolumes: + - name: registry-dockerconfig + secret: + secretName: registry-pull-secret + extraVolumeMounts: + - name: registry-dockerconfig + mountPath: /run/image-credential-provider/dockerconfig + readOnly: true + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name == "registry-dockerconfig")].secret.secretName + value: registry-pull-secret + - equal: + path: spec.template.spec.containers[0].volumeMounts[?(@.name == "registry-dockerconfig")].mountPath + value: /run/image-credential-provider/dockerconfig diff --git a/charts/substrate/tests/pod_metadata_test.yaml b/charts/substrate/tests/pod_metadata_test.yaml new file mode 100644 index 0000000000..89adb4cf0c --- /dev/null +++ b/charts/substrate/tests/pod_metadata_test.yaml @@ -0,0 +1,235 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +suite: Pod metadata and RustFS knobs +templates: +- ate-api-server.yaml +- ate-controller.yaml +- atelet.yaml +- rustfs.yaml +release: + name: custom + namespace: custom-system +tests: +- it: adds podLabels and podAnnotations to a Deployment pod template + template: ate-controller.yaml + documentSelector: + path: kind + value: Deployment + set: + podLabels: + sidecar.istio.io/inject: "false" + podAnnotations: + example.com/placement: quiet + asserts: + - equal: + path: spec.template.metadata.labels["sidecar.istio.io/inject"] + value: "false" + - equal: + path: spec.template.metadata.annotations["example.com/placement"] + value: quiet + +- it: merges podAnnotations with the annotations a template already sets + template: ate-api-server.yaml + documentSelector: + path: kind + value: Deployment + set: + podAnnotations: + example.com/placement: quiet + asserts: + - equal: + path: spec.template.metadata.annotations["example.com/placement"] + value: quiet + - equal: + path: spec.template.metadata.annotations["prometheus.io/scrape"] + value: "true" + +- it: adds podLabels and podAnnotations to the atelet DaemonSet + template: atelet.yaml + documentSelector: + path: kind + value: DaemonSet + set: + podLabels: + sidecar.istio.io/inject: "false" + podAnnotations: + example.com/placement: quiet + asserts: + - equal: + path: spec.template.metadata.labels["sidecar.istio.io/inject"] + value: "false" + - equal: + path: spec.template.metadata.annotations["example.com/placement"] + value: quiet + +- it: adds podLabels and podAnnotations to the bucket-init Job + template: rustfs.yaml + documentSelector: + path: kind + value: Job + set: + podLabels: + sidecar.istio.io/inject: "false" + podAnnotations: + example.com/placement: quiet + asserts: + - equal: + path: spec.template.metadata.labels["sidecar.istio.io/inject"] + value: "false" + - equal: + path: spec.template.metadata.annotations["example.com/placement"] + value: quiet + +- it: leaves the bucket-init Job pod template metadata out by default + template: rustfs.yaml + documentSelector: + path: kind + value: Job + asserts: + - notExists: + path: spec.template.metadata + +- it: labels the RustFS claim and selects its StorageClass + template: rustfs.yaml + documentSelector: + path: kind + value: PersistentVolumeClaim + set: + commonLabels: + app.kubernetes.io/name: substrate + rustfs: + storageClassName: fast + asserts: + - equal: + path: metadata.labels["app.kubernetes.io/name"] + value: substrate + - equal: + path: spec.storageClassName + value: fast + +- it: uses the cluster default StorageClass and no claim labels by default + template: rustfs.yaml + documentSelector: + path: kind + value: PersistentVolumeClaim + asserts: + - notExists: + path: spec.storageClassName + - notExists: + path: metadata.labels + +- it: applies the RustFS Deployment strategy + template: rustfs.yaml + documentSelector: + path: kind + value: Deployment + set: + rustfs: + strategy: + type: Recreate + asserts: + - equal: + path: spec.strategy.type + value: Recreate + +- it: runs the bucket-init Job under its own annotated ServiceAccount when dedicated + template: rustfs.yaml + documentSelector: + path: metadata.name + value: custom-rustfs-bucket-init + matchMany: true + set: + rustfs: + bucketInit: + dedicatedServiceAccount: true + annotations: + argocd.argoproj.io/hook: PostSync + asserts: + # The template renders one more document, the Job's ServiceAccount, than by default. + - hasDocuments: + count: 7 + - matchRegex: + path: kind + pattern: ^(Job|ServiceAccount)$ + - equal: + path: metadata.annotations["argocd.argoproj.io/hook"] + value: PostSync + +- it: renders no dedicated ServiceAccount by default + template: rustfs.yaml + documentSelector: + path: metadata.name + value: custom-rustfs-bucket-init + matchMany: true + asserts: + - hasDocuments: + count: 6 + - isKind: + of: Job + +- it: runs the bucket-init Job under the RustFS ServiceAccount by default + template: rustfs.yaml + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: spec.template.spec.serviceAccountName + value: custom-rustfs + +- it: runs the bucket-init Job under the dedicated ServiceAccount when enabled + template: rustfs.yaml + documentSelector: + path: kind + value: Job + set: + rustfs: + bucketInit: + dedicatedServiceAccount: true + asserts: + - equal: + path: spec.template.spec.serviceAccountName + value: custom-rustfs-bucket-init + +- it: tolerates sandbox-class and ComputeClass taints on atelet by default + template: atelet.yaml + documentSelector: + path: kind + value: DaemonSet + asserts: + - equal: + path: spec.template.spec.tolerations + value: + - key: ate.dev/sandboxClass + operator: Exists + effect: NoSchedule + - key: cloud.google.com/compute-class + operator: Exists + effect: NoSchedule + +- it: replaces atelet tolerations as a whole when overridden + template: atelet.yaml + documentSelector: + path: kind + value: DaemonSet + set: + atelet: + tolerations: + - operator: Exists + asserts: + - equal: + path: spec.template.spec.tolerations + value: + - operator: Exists diff --git a/charts/substrate/tests/postgres_test.yaml b/charts/substrate/tests/postgres_test.yaml index 3420ae38fc..e3fb1df550 100644 --- a/charts/substrate/tests/postgres_test.yaml +++ b/charts/substrate/tests/postgres_test.yaml @@ -101,3 +101,40 @@ tests: asserts: - failedTemplate: errorMessage: postgres.ownerConnectionStringSecretRef.name is required + +- it: reads the role names from Secret keys when the SecretRefs are set + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + readWriteRoleSecretRef: + name: substrate-postgres-readwrite + key: username + ownerRoleSecretRef: + name: substrate-postgres-owner + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-read-write-role=$(ATE_API_POSTGRES_READ_WRITE_ROLE) + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-owner-role=$(ATE_API_POSTGRES_OWNER_ROLE) + - notContains: + path: spec.template.spec.containers[0].args + content: --postgres-read-write-role=substrate_readwrite + - contains: + path: spec.template.spec.containers[0].env + content: + name: ATE_API_POSTGRES_READ_WRITE_ROLE + valueFrom: + secretKeyRef: + name: substrate-postgres-readwrite + key: username + - contains: + path: spec.template.spec.containers[0].env + content: + name: ATE_API_POSTGRES_OWNER_ROLE + valueFrom: + secretKeyRef: + name: substrate-postgres-owner + key: username diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 6cd8426931..ebcf81b2db 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -27,6 +27,14 @@ createNamespace: false # ExternalVolumes is required for external volume APIs and mounts. preview: [] +# Labels added to every pod template (all Deployments, the atelet DaemonSet and the RustFS +# bucket-init Job). +podLabels: {} +# Annotations added to every pod template. +podAnnotations: {} +# Labels added to the RustFS PersistentVolumeClaim. +commonLabels: {} + postgres: # Project database login certificates only when the connection strings use them. clientCertificates: @@ -42,11 +50,22 @@ postgres: # Roles assumed on each connection. Use distinct roles for installs sharing a database. readWriteRole: substrate_readwrite ownerRole: substrate_owner + # Read a role name from a Secret key ({name, key}; key defaults to "username") instead of + # the literal above, for databases whose login is generated and rotated by an external + # provisioner and is itself the role to assume. + readWriteRoleSecretRef: {} + ownerRoleSecretRef: {} schema: substrate rustfs: enabled: true storageSize: 1Gi + # StorageClass for the data PersistentVolumeClaim. Empty uses the cluster default class; + # the claim never binds on clusters without one. + storageClassName: "" + # Deployment strategy. The data volume is ReadWriteOnce, so a RollingUpdate cannot place + # the new pod while the old one holds it; set {type: Recreate} for in-place upgrades. + strategy: {} bucket: ate-snapshots accessKey: rustfsadmin secretKey: rustfsadmin @@ -59,6 +78,13 @@ rustfs: memory: 1Gi # The Job that creates the snapshot bucket. bucketInit: + # Annotations on the Job (and its ServiceAccount when dedicated). Job templates are + # immutable, so GitOps tools need a hint to replace the Job on change, for example + # argocd.argoproj.io/hook: PostSync. + annotations: {} + # Run the Job under its own ServiceAccount, named like the Job, instead of the RustFS + # ServiceAccount. + dedicatedServiceAccount: false resources: requests: cpu: 10m @@ -86,6 +112,23 @@ atelet: storageBackend: s3 extraArgs: [] extraEnv: [] + # Tolerations of the atelet pods, replaced as a whole when overridden. The + # defaults run atelet on nodes reserved for any WorkerPool sandbox class and + # on GKE ComputeClass nodes. Platforms whose admission controllers rewrite + # DaemonSet tolerations (e.g. to a single operator: Exists) can set the same + # value here so the rendered manifest matches what the cluster stores. + tolerations: + - key: ate.dev/sandboxClass + operator: Exists + effect: NoSchedule + - key: cloud.google.com/compute-class + operator: Exists + effect: NoSchedule + # Appended verbatim to the atelet pod's volumes and the atelet container's + # volumeMounts, e.g. a kubernetes.io/dockerconfigjson Secret a credential + # provider plugin reads. + extraVolumes: [] + extraVolumeMounts: [] resources: requests: cpu: 50m