Deploy — Kubernetes (Kustomize) #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Deploy the DeltaDatabase Kustomize stack to a Kubernetes cluster. | |
| # | |
| # Triggers: | |
| # • Automatically after "Docker Hub Publish" succeeds (workflow_run). | |
| # • Manually via the GitHub Actions UI (workflow_dispatch), optionally | |
| # specifying an image-tag prefix to deploy a specific release. | |
| # | |
| # Required repository secrets (Settings → Secrets and variables → Actions → Secrets): | |
| # KUBE_CONFIG Base64-encoded kubeconfig granting access to the target cluster. | |
| # Generate with: base64 -w0 ~/.kube/config | |
| # DELTA_MASTER_KEY Hex-encoded AES-256 master encryption key for DeltaDatabase. | |
| # Generate with: openssl rand -hex 32 | |
| # DELTA_ADMIN_KEY Admin API key for DeltaDatabase REST/gRPC endpoints. | |
| # Generate with: openssl rand -hex 24 | |
| # GRAFANA_ADMIN_PASSWORD Password for the Grafana admin user. | |
| # | |
| # Required repository variables (Settings → Secrets and variables → Actions → Variables): | |
| # DOCKERHUB_USERNAME Docker Hub username (shared with docker-publish.yml). | |
| # | |
| # What this workflow does: | |
| # 1. Validates that all required secrets are set. | |
| # 2. Creates or updates the three Kubernetes secrets | |
| # (delta-master-key, delta-admin-key, grafana-admin) from the GitHub secrets above. | |
| # 3. Ensures SeaweedFS (with its CSI driver) is installed in the cluster so that | |
| # the 'seaweedfs-storage' ReadWriteMany StorageClass is available. | |
| # 4. Optionally pins the image tags to a specific release version. | |
| # 5. Applies the full Kustomize overlay (kubectl apply -k deploy/kubernetes/kustomize), | |
| # which deploys: Namespace · SeaweedFS StorageClass · Shared PVC (ReadWriteMany) · | |
| # Main Worker · Processing Workers · memory-based HPA · Prometheus · Grafana | |
| # (with pre-built dashboard). | |
| # 6. Waits for all four Deployments to roll out successfully. | |
| name: Deploy — Kubernetes (Kustomize) | |
| on: | |
| # Automatically run after every successful Docker Hub Publish workflow. | |
| workflow_run: | |
| workflows: ["Docker Hub Publish"] | |
| types: [completed] | |
| # Allow operators to trigger a deploy manually and optionally pin a release tag. | |
| workflow_dispatch: | |
| inputs: | |
| image_tag: | |
| description: > | |
| Image-tag prefix to deploy. | |
| Use 'latest' for the rolling build, or a version such as 'v0.1.1-alpha' | |
| for a pinned release. Defaults to 'latest'. | |
| required: false | |
| default: "latest" | |
| allow_pvc_deletion: | |
| description: > | |
| Set to 'true' to allow automatic deletion and recreation of the | |
| delta-shared-pvc PersistentVolumeClaim when its storageClass does not | |
| match the manifest (e.g. migrating from nfs-client to seaweedfs-storage). | |
| WARNING: deleting the PVC destroys all data it holds. | |
| Back up any data you want to keep before enabling this option. | |
| When triggered automatically by workflow_run this check is skipped and | |
| the PVC is always migrated. | |
| required: false | |
| default: "false" | |
| jobs: | |
| deploy: | |
| name: Apply Kustomize stack | |
| runs-on: self-hosted | |
| # For workflow_run: only proceed when every build matrix job succeeded. | |
| # For workflow_dispatch: always proceed. | |
| if: >- | |
| github.event_name == 'workflow_dispatch' || | |
| github.event.workflow_run.conclusion == 'success' | |
| permissions: | |
| contents: read | |
| steps: | |
| # ── 1. Source code ──────────────────────────────────────────────────────── | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # ── 2. CLI tools ────────────────────────────────────────────────────────── | |
| - name: Set up kubectl | |
| uses: azure/setup-kubectl@v4 | |
| - name: Ensure Caddy is installed | |
| run: | | |
| if command -v caddy &>/dev/null; then | |
| echo "Caddy $(caddy version) already installed — skipping installation." | |
| exit 0 | |
| fi | |
| echo "Caddy not found — installing via the official apt repository..." | |
| sudo apt-get install -y debian-keyring debian-archive-keyring apt-transport-https curl | |
| curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | \ | |
| sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg | |
| curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | \ | |
| sudo tee /etc/apt/sources.list.d/caddy-stable.list | |
| sudo apt-get update | |
| sudo apt-get install -y caddy | |
| echo "Caddy $(caddy version) installed." | |
| - name: Install kustomize | |
| env: | |
| KUSTOMIZE_VERSION: "v5.4.3" | |
| run: | | |
| # Skip installation if kustomize is already present (e.g. cached on runner). | |
| if command -v kustomize &>/dev/null; then | |
| echo "kustomize $(kustomize version --short) already installed — skipping." | |
| exit 0 | |
| fi | |
| OS="linux" | |
| ARCH="amd64" | |
| URL="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_${OS}_${ARCH}.tar.gz" | |
| curl -sL "${URL}" | tar xz -C /tmp | |
| mkdir -p "${HOME}/.local/bin" | |
| mv /tmp/kustomize "${HOME}/.local/bin/kustomize" | |
| echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" | |
| - name: Install Helm | |
| env: | |
| HELM_VERSION: "v4.1.1" | |
| run: | | |
| # Skip installation if Helm is already present (e.g. cached on runner). | |
| if command -v helm &>/dev/null; then | |
| echo "Helm $(helm version --short) already installed — skipping." | |
| exit 0 | |
| fi | |
| curl -sL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | \ | |
| DESIRED_VERSION="${HELM_VERSION}" bash | |
| # ── 3. Cluster authentication ───────────────────────────────────────────── | |
| - name: Validate required secrets | |
| run: | | |
| missing=() | |
| [[ -z "${{ secrets.KUBE_CONFIG }}" ]] && missing+=(KUBE_CONFIG) | |
| [[ -z "${{ secrets.DELTA_MASTER_KEY }}" ]] && missing+=(DELTA_MASTER_KEY) | |
| [[ -z "${{ secrets.DELTA_ADMIN_KEY }}" ]] && missing+=(DELTA_ADMIN_KEY) | |
| [[ -z "${{ secrets.GRAFANA_ADMIN_PASSWORD }}" ]] && missing+=(GRAFANA_ADMIN_PASSWORD) | |
| if [[ ${#missing[@]} -gt 0 ]]; then | |
| echo "::error::The following required secrets are not set: ${missing[*]}" | |
| echo "::error::See docs/usage/deployment.md#github-actions-deploy for setup instructions." | |
| exit 1 | |
| fi | |
| - name: Configure kubeconfig | |
| run: | | |
| mkdir -p "${HOME}/.kube" | |
| echo "${{ secrets.KUBE_CONFIG }}" | base64 -d > "${HOME}/.kube/config" | |
| chmod 600 "${HOME}/.kube/config" | |
| - name: Verify cluster connectivity | |
| run: kubectl cluster-info --request-timeout=10s | |
| # ── 4. Compute image-tag prefix ─────────────────────────────────────────── | |
| - name: Compute image-tag prefix | |
| id: tag | |
| run: | | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| PREFIX="${{ github.event.inputs.image_tag }}" | |
| else | |
| # workflow_run: head_branch is the tag name for tag-push triggers, | |
| # or the branch name (e.g. "main") for branch-push triggers. | |
| REF="${{ github.event.workflow_run.head_branch }}" | |
| if [[ "$REF" =~ ^v[0-9] ]]; then | |
| PREFIX="$REF" | |
| else | |
| PREFIX="latest" | |
| fi | |
| fi | |
| echo "prefix=${PREFIX}" >> "${GITHUB_OUTPUT}" | |
| echo "Deploying images with tag prefix: ${PREFIX}" | |
| # ── 5. Ensure SeaweedFS is installed ───────────────────────────────────── | |
| # | |
| # DeltaDatabase uses a ReadWriteMany PVC backed by the SeaweedFS CSI driver | |
| # (StorageClass 'seaweedfs-storage'). This step checks whether SeaweedFS is | |
| # already present in the cluster; if not, it installs it via Helm so that the | |
| # StorageClass is available before the Kustomize overlay is applied. | |
| # | |
| # IMPORTANT: The check uses `helm status` (not `kubectl get storageclass`) | |
| # because the 'seaweedfs-storage' StorageClass is created by the Kustomize | |
| # overlay (Step 10), not by Helm. Checking the StorageClass would falsely | |
| # indicate that SeaweedFS is installed after any previous Kustomize apply, | |
| # even when the SeaweedFS Helm release (and its CSI provisioner) has since | |
| # been removed — leaving the PVC stuck in Pending indefinitely. | |
| # | |
| # Note on encryption: DeltaDatabase Processing Workers encrypt all JSON blobs | |
| # before writing them to the shared filesystem (AES-256, key managed by the | |
| # Main Worker). SeaweedFS therefore acts as a plain shared block store — no | |
| # additional at-rest encryption is configured at the SeaweedFS layer. | |
| - name: Ensure SeaweedFS CSI driver is installed | |
| run: | | |
| # Detect SeaweedFS via its Helm release, not via the StorageClass. | |
| # The 'seaweedfs-storage' StorageClass is created by the Kustomize overlay | |
| # in Step 10, so it would persist on the cluster even after SeaweedFS | |
| # itself is uninstalled, making it an unreliable indicator of whether the | |
| # CSI provisioner is actually running. | |
| if helm status seaweedfs --namespace seaweedfs &>/dev/null; then | |
| echo "SeaweedFS Helm release already present — skipping installation." | |
| exit 0 | |
| fi | |
| echo "SeaweedFS Helm release not found — installing SeaweedFS via Helm..." | |
| helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm | |
| helm repo update | |
| # Deploy SeaweedFS with filer and CSI driver enabled into its own namespace. | |
| # Persistence is explicitly enabled so volume data survives pod restarts. | |
| # Resource limits keep SeaweedFS from starving other workloads on the node. | |
| # --wait ensures the CSI driver (and therefore the StorageClass) is ready | |
| # before this step exits. | |
| # helm upgrade --install is used so this step is idempotent: it installs | |
| # SeaweedFS on a fresh cluster and upgrades it if a partial install exists. | |
| helm upgrade --install seaweedfs seaweedfs/seaweedfs \ | |
| --namespace seaweedfs \ | |
| --create-namespace \ | |
| --set filer.enabled=true \ | |
| --set csi.enabled=true \ | |
| --set global.persistence.enabled=true \ | |
| --set master.resources.requests.cpu=100m \ | |
| --set master.resources.requests.memory=128Mi \ | |
| --set master.resources.limits.cpu=500m \ | |
| --set master.resources.limits.memory=256Mi \ | |
| --set volume.resources.requests.cpu=100m \ | |
| --set volume.resources.requests.memory=256Mi \ | |
| --set volume.resources.limits.cpu=1000m \ | |
| --set volume.resources.limits.memory=512Mi \ | |
| --set filer.resources.requests.cpu=100m \ | |
| --set filer.resources.requests.memory=128Mi \ | |
| --set filer.resources.limits.cpu=500m \ | |
| --set filer.resources.limits.memory=256Mi \ | |
| --wait \ | |
| --timeout=300s | |
| # Wait for the SeaweedFS CSI controller pod to be ready so that the | |
| # 'seaweedfs-csi-driver' provisioner is registered with the API server | |
| # before the Kustomize overlay creates the PVC. Without this extra wait | |
| # there is a race window where the PVC is created but no provisioner has | |
| # yet claimed it, leaving it stuck in Pending. | |
| # | |
| # Two label selectors are tried in sequence to accommodate different | |
| # versions of the SeaweedFS Helm chart which may use either label. | |
| echo "Waiting for SeaweedFS CSI controller to be ready..." | |
| if kubectl wait --for=condition=ready pod \ | |
| --selector=app.kubernetes.io/component=csi-controller \ | |
| --namespace=seaweedfs \ | |
| --timeout=120s 2>/dev/null; then | |
| echo "CSI controller ready (matched app.kubernetes.io/component=csi-controller)." | |
| elif kubectl wait --for=condition=ready pod \ | |
| --selector=app=seaweedfs-csi-controller \ | |
| --namespace=seaweedfs \ | |
| --timeout=120s 2>/dev/null; then | |
| echo "CSI controller ready (matched app=seaweedfs-csi-controller)." | |
| else | |
| echo "::warning::Could not confirm CSI controller readiness via pod selector — proceeding anyway." | |
| echo "If the PVC stays Pending, check: kubectl get pods -n seaweedfs" | |
| fi | |
| echo "SeaweedFS installed. Available storage classes:" | |
| kubectl get storageclass | |
| # ── 6. Ensure namespace exists ──────────────────────────────────────────── | |
| - name: Ensure namespace | |
| run: | | |
| kubectl create namespace deltadatabase \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| # ── 7. Sync Kubernetes secrets from GitHub secrets ──────────────────────── | |
| # | |
| # We create or update all three secrets before applying Kustomize so that | |
| # every Secret reference in the manifests resolves correctly on first apply. | |
| # Using --dry-run=client -o yaml | kubectl apply makes this idempotent. | |
| - name: Sync Kubernetes secrets | |
| env: | |
| DELTA_MASTER_KEY: ${{ secrets.DELTA_MASTER_KEY }} | |
| DELTA_ADMIN_KEY: ${{ secrets.DELTA_ADMIN_KEY }} | |
| GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} | |
| run: | | |
| # delta-master-key — AES-256 encryption key used by all DeltaDatabase workers. | |
| kubectl -n deltadatabase create secret generic delta-master-key \ | |
| --from-literal=master-key="${DELTA_MASTER_KEY}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| # delta-admin-key — admin API key for the DeltaDatabase REST / gRPC API. | |
| kubectl -n deltadatabase create secret generic delta-admin-key \ | |
| --from-literal=admin-key="${DELTA_ADMIN_KEY}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| # grafana-admin — Grafana web-UI admin credentials. | |
| kubectl -n deltadatabase create secret generic grafana-admin \ | |
| --from-literal=admin-password="${GRAFANA_ADMIN_PASSWORD}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| # ── 8. Pin image tags (version deployments only) ────────────────────────── | |
| - name: Pin image tags in Kustomize overlay | |
| working-directory: deploy/kubernetes/kustomize | |
| run: | | |
| PREFIX="${{ steps.tag.outputs.prefix }}" | |
| IMAGE="${{ vars.DOCKERHUB_USERNAME }}/deltadatabase" | |
| # For the rolling 'latest' builds the manifests already reference | |
| # the correct :latest-main / :latest-proc tags — nothing to change. | |
| if [[ "$PREFIX" == "latest" ]]; then | |
| echo "Using default 'latest' image tags — no override needed." | |
| exit 0 | |
| fi | |
| # For pinned releases, override both image references so kustomize | |
| # builds manifests that reference the exact version that was just built. | |
| kustomize edit set image \ | |
| "${IMAGE}:latest-main=${IMAGE}:${PREFIX}-main" \ | |
| "${IMAGE}:latest-proc=${IMAGE}:${PREFIX}-proc" | |
| echo "Image tags pinned to: ${PREFIX}-main / ${PREFIX}-proc" | |
| # ── 9. Migrate shared PVC if storageClass changed ──────────────────────── | |
| # | |
| # Kubernetes PVC specs are immutable after the claim is bound (except for | |
| # resources.requests and volumeAttributesClassName). When the desired | |
| # storageClass changes (e.g. from nfs-client to seaweedfs-storage), kubectl | |
| # apply returns a "spec is immutable" error and the deploy fails. | |
| # | |
| # This step detects that situation and deletes the existing PVC so that the | |
| # subsequent Kustomize apply can recreate it with the correct storageClass. | |
| # NOTE: deleting a PVC removes the data it holds. Back up any data you want | |
| # to preserve before running this workflow with PVC migration enabled. | |
| - name: Migrate shared PVC if storageClass changed | |
| run: | | |
| DESIRED_SC="seaweedfs-storage" | |
| # Query the current storageClass. Distinguish "not found" (safe to | |
| # continue) from other errors (authentication failure, network issues, | |
| # etc.) which must abort the deploy. | |
| SC_OUTPUT=$(kubectl -n deltadatabase get pvc delta-shared-pvc \ | |
| -o jsonpath='{.spec.storageClassName}' 2>/tmp/pvc_query_error) \ | |
| && QUERY_EXIT=0 || QUERY_EXIT=$? | |
| if [[ $QUERY_EXIT -ne 0 ]]; then | |
| if grep -qi "not found" /tmp/pvc_query_error 2>/dev/null; then | |
| echo "PVC delta-shared-pvc does not exist yet — will be created by Kustomize apply." | |
| exit 0 | |
| else | |
| echo "::error::Failed to query PVC delta-shared-pvc: $(cat /tmp/pvc_query_error)" | |
| exit 1 | |
| fi | |
| fi | |
| CURRENT_SC="$SC_OUTPUT" | |
| if [[ "$CURRENT_SC" == "$DESIRED_SC" ]]; then | |
| echo "PVC delta-shared-pvc already uses storageClass '${DESIRED_SC}' — no migration needed." | |
| exit 0 | |
| fi | |
| # storageClass mismatch: deletion required. | |
| echo "PVC delta-shared-pvc has storageClass '${CURRENT_SC}' but manifest requires '${DESIRED_SC}'." | |
| # For manual workflow_dispatch runs, require the operator to explicitly | |
| # acknowledge the data loss by setting allow_pvc_deletion=true. | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" && \ | |
| "${{ github.event.inputs.allow_pvc_deletion }}" != "true" ]]; then | |
| echo "::error::PVC storageClass migration requires data deletion." | |
| echo "::error::Re-run this workflow with 'allow_pvc_deletion' set to 'true' to proceed." | |
| echo "::error::WARNING: this will permanently destroy all data in delta-shared-pvc." | |
| exit 1 | |
| fi | |
| echo "Deleting PVC so it can be recreated with storageClass '${DESIRED_SC}'..." | |
| kubectl -n deltadatabase delete pvc delta-shared-pvc --wait=true | |
| echo "PVC deleted — it will be recreated by the Kustomize apply step." | |
| # ── 10. Apply the full Kustomize overlay ────────────────────────────────── | |
| - name: Apply Kustomize overlay | |
| run: | | |
| kubectl apply -k deploy/kubernetes/kustomize | |
| # ── 11. Wait for rollout ────────────────────────────────────────────────── | |
| - name: Wait for rollout — main-worker | |
| run: kubectl -n deltadatabase rollout status deployment/main-worker --timeout=300s | |
| - name: Wait for rollout — proc-worker | |
| run: kubectl -n deltadatabase rollout status deployment/proc-worker --timeout=300s | |
| - name: Wait for rollout — prometheus | |
| run: kubectl -n deltadatabase rollout status deployment/prometheus --timeout=300s | |
| - name: Wait for rollout — grafana | |
| run: kubectl -n deltadatabase rollout status deployment/grafana --timeout=300s | |
| # ── 13. Configure global Caddy reverse proxy ────────────────────────── | |
| # | |
| # Caddy is installed on the runner host (Step 2) and acts as a | |
| # global reverse proxy that forwards traffic from well-known host | |
| # ports to the NodePort services exposed by k3d: | |
| # | |
| # HOST_IP:1337 → k8s-node:30300 → Grafana pod :3000 | |
| # HOST_IP:42069 → k8s-node:30080 → Main Worker pod :8080 | |
| # | |
| # The node IP is resolved at deploy time from the running cluster so | |
| # that the Caddyfile always points at the correct k3d container IP. | |
| - name: Configure global Caddy reverse proxy | |
| run: | | |
| # Resolve the Kubernetes node IP (the k3d Docker container IP on the runner). | |
| K8S_NODE_IP=$(kubectl get nodes \ | |
| -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') | |
| if [[ -z "${K8S_NODE_IP}" ]]; then | |
| echo "::error::Could not resolve Kubernetes node IP. Ensure the cluster is reachable." | |
| exit 1 | |
| fi | |
| echo "Kubernetes node IP: ${K8S_NODE_IP}" | |
| export K8S_NODE_IP | |
| export GRAFANA_NODEPORT=30300 | |
| export MAINWORKER_NODEPORT=30080 | |
| export CADDY_GRAFANA_PORT=1337 | |
| export CADDY_MAINWORKER_PORT=42069 | |
| # Substitute environment variables into the Caddyfile template and | |
| # write the result to the system Caddy configuration path. | |
| envsubst < deploy/caddy/Caddyfile | sudo tee /etc/caddy/Caddyfile > /dev/null | |
| # Validate the Caddyfile syntax before reloading. | |
| sudo caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile | |
| # Reload Caddy if already running, otherwise enable and start it. | |
| # caddy reload sends the new config directly via the admin API, which | |
| # avoids a full service restart and returns as soon as the new config | |
| # is accepted. For a fresh start, --no-block prevents systemctl from | |
| # waiting for the service to reach its active state (which can hang | |
| # indefinitely when Caddy's sd_notify never fires, e.g. if the ports | |
| # are briefly occupied by a previous run). | |
| if sudo systemctl is-active --quiet caddy 2>/dev/null; then | |
| sudo caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile | |
| echo "Caddy reloaded with new configuration." | |
| else | |
| sudo systemctl enable caddy | |
| sudo systemctl start --no-block caddy | |
| echo "Caddy service started." | |
| fi | |
| # ── 14. Deployment summary ──────────────────────────────────────────────── | |
| - name: Print deployment summary | |
| run: | | |
| echo "===== DeltaDatabase deployment summary =====" | |
| kubectl -n deltadatabase get deployments,hpa,svc \ | |
| -o wide --no-headers 2>/dev/null || true | |
| echo "" | |
| echo "===== Running images =====" | |
| kubectl -n deltadatabase get pods \ | |
| -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .spec.containers[*]}{.image}{"\n"}{end}{end}' \ | |
| 2>/dev/null || true | |
| echo "" | |
| echo "===== Access URLs (via global Caddy reverse proxy) =====" | |
| HOST_IP=$(hostname -I | awk '{print $1}') | |
| echo " Grafana dashboard : http://${HOST_IP}:1337" | |
| echo " Main Worker UI : http://${HOST_IP}:42069" |