CredentialEngine · arielr-lt · Mar 6, 2026 · Mar 3, 2026 · Mar 3, 2026 · Mar 4, 2026
diff --git a/.github/workflows/update-es-ingress-whitelist.yaml b/.github/workflows/update-es-ingress-whitelist.yaml
@@ -0,0 +1,89 @@
+name: Update ES Ingress IP Whitelist
+
+on:
+  workflow_dispatch:
+    inputs:
+      ip:
+        description: "Your current IP address (without /32)"
+        type: string
+        required: true
+      label:
+        description: "Label for this IP (e.g. 'Ariel home')"
+        type: string
+        required: false
+        default: ""
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  AWS_REGION: us-east-1
+  EKS_CLUSTER: ce-registry-eks
+  NAMESPACE: credreg-sandbox
+  INGRESS_NAME: elasticsearch
+
+jobs:
+  update-whitelist:
+    if: ${{ github.repository_owner == 'CredentialEngine' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT }}:role/github-oidc-widget
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: v1.29.6
+
+      - name: Update kubeconfig
+        run: |
+          aws eks update-kubeconfig --name "${{ env.EKS_CLUSTER }}" --region "${{ env.AWS_REGION }}"
+
+      - name: Add IP to whitelist
+        run: |
+          NEW_IP="${{ inputs.ip }}/32"
+
+          CURRENT=$(kubectl get ingress "${{ env.INGRESS_NAME }}" \
+            -n "${{ env.NAMESPACE }}" \
+            -o jsonpath='{.metadata.annotations.nginx\.ingress\.kubernetes\.io/whitelist-source-range}')
+
+          echo "Current whitelist: $CURRENT"
+
+          if echo "$CURRENT" | grep -qF "$NEW_IP"; then
+            echo "IP $NEW_IP is already in the whitelist, nothing to do."
+            exit 0
+          fi
+
+          UPDATED="${CURRENT},${NEW_IP}"
+
+          kubectl annotate ingress "${{ env.INGRESS_NAME }}" \
+            -n "${{ env.NAMESPACE }}" \
+            --overwrite \
+            "nginx.ingress.kubernetes.io/whitelist-source-range=${UPDATED}"
+
+          echo "Updated whitelist: $UPDATED"
+          if [ -n "${{ inputs.label }}" ]; then
+            echo "Label: ${{ inputs.label }}"
+          fi
+
+      - name: Notify Slack
+        if: always()
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        run: |
+          if [ -z "${SLACK_WEBHOOK_URL}" ]; then
+            echo "SLACK_WEBHOOK_URL not set; skipping notification"
+            exit 0
+          fi
+          STATUS="${{ job.status }}"
+          EMOJI=✅; [ "$STATUS" = "failure" ] && EMOJI=❌
+          LABEL="${{ inputs.label }}"
+          IP="${{ inputs.ip }}/32"
+          MSG="$EMOJI ES ingress whitelist update ${STATUS}: ${IP}${LABEL:+ ($LABEL)} triggered by ${{ github.actor }} — ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          curl -sS -X POST -H 'Content-type: application/json' \
+            --data "$(jq -nc --arg text "$MSG" '{text:$text}')" \
+            "$SLACK_WEBHOOK_URL" || true
diff --git a/terraform/environments/eks/k8s-manifests-cluster/kubernetes-event-exporter.yaml b/terraform/environments/eks/k8s-manifests-cluster/kubernetes-event-exporter.yaml
@@ -0,0 +1,81 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: event-exporter
+  namespace: amazon-cloudwatch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: event-exporter
+rules:
+  - apiGroups: [""]
+    resources: ["events", "namespaces", "pods"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: event-exporter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: event-exporter
+subjects:
+  - kind: ServiceAccount
+    name: event-exporter
+    namespace: amazon-cloudwatch
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: event-exporter-config
+  namespace: amazon-cloudwatch
+data:
+  config.yaml: |
+    logLevel: warn
+    logFormat: json
+    route:
+      routes:
+        - match:
+            - receiver: stdout
+    receivers:
+      - name: stdout
+        stdout: {}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: event-exporter
+  namespace: amazon-cloudwatch
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: event-exporter
+  template:
+    metadata:
+      labels:
+        app: event-exporter
+    spec:
+      serviceAccountName: event-exporter
+      containers:
+        - name: event-exporter
+          image: ghcr.io/resmoio/kubernetes-event-exporter:v1.7
+          args:
+            - -conf=/data/config.yaml
+          volumeMounts:
+            - name: config
+              mountPath: /data
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
+      volumes:
+        - name: config
+          configMap:
+            name: event-exporter-config
diff --git a/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-exporter.yaml b/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-exporter.yaml
@@ -0,0 +1,59 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elasticsearch-exporter
+  namespace: credreg-sandbox
+  labels:
+    app: elasticsearch-exporter
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch-exporter
+  template:
+    metadata:
+      labels:
+        app: elasticsearch-exporter
+    spec:
+      nodeSelector:
+        env: sandbox
+      tolerations:
+        - key: "env"
+          operator: "Equal"
+          value: "sandbox"
+          effect: "NoSchedule"
+      containers:
+        - name: elasticsearch-exporter
+          image: prometheuscommunity/elasticsearch-exporter:v1.8.0
+          args:
+            - --es.uri=http://elasticsearch.credreg-sandbox.svc.cluster.local:9200
+            - --es.all
+            - --es.indices
+            - --es.shards
+          ports:
+            - containerPort: 9114
+              name: metrics
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch-exporter
+  namespace: credreg-sandbox
+  labels:
+    app: elasticsearch-exporter
+spec:
+  type: ClusterIP
+  selector:
+    app: elasticsearch-exporter
+  ports:
+    - name: metrics
+      port: 9114
+      targetPort: 9114
diff --git a/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-ingress.yaml b/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-ingress.yaml
@@ -8,11 +8,13 @@ metadata:
     nginx.ingress.kubernetes.io/auth-type: "basic"
     nginx.ingress.kubernetes.io/auth-secret: "es-basic-auth"
     nginx.ingress.kubernetes.io/auth-realm: "Authentication Required"
-    nginx.ingress.kubernetes.io/whitelist-source-range: "98.97.134.132/32,71.212.64.155/32,98.13.197.1/32,98.193.126.147/32"
+    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
+    nginx.ingress.kubernetes.io/whitelist-source-range: "192.140.91.9/32,98.97.134.132/32,71.212.64.155/32,98.13.197.1/32,98.193.126.147/32,128.203.139.2/32"
      # 71.212.64.155  – Rohit
      # 98.13.197.1  – Jenna
      # 98.193.126.147 – Mike P.
      # 98.97.134.132/32 - Ariel
+     # 128.203.139.2/32 - Azure's cluster
 spec:
   ingressClassName: nginx
   tls:

diff --git a/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-pdb.yaml b/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-pdb.yaml
@@ -0,0 +1,10 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: elasticsearch-pdb
+  namespace: credreg-sandbox
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
diff --git a/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml b/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml
@@ -17,14 +17,21 @@ spec:
         app: elasticsearch
     spec:
       subdomain: elasticsearch-discovery
-      priorityClassName: sandbox-medium
+      priorityClassName: prod-high
       nodeSelector:
         env: sandbox
       tolerations:
         - key: "env"
           operator: "Equal"
           value: "sandbox"
-          effect: "NoSchedule"       
+          effect: "NoSchedule"
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: elasticsearch
+              topologyKey: kubernetes.io/hostname
       securityContext:
         fsGroup: 1000
         runAsUser: 1000
@@ -37,16 +44,32 @@ spec:
             - containerPort: 9300
           resources:
             requests:
-              cpu: "256m"
+              cpu: "500m"
               memory: "2Gi"
             limits:
               cpu: "1000m"
               memory: "4Gi"
+          readinessProbe:
+            httpGet:
+              path: /_cluster/health?wait_for_status=yellow&timeout=5s
+              port: 9200
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 3
+          livenessProbe:
+            httpGet:
+              path: /_cluster/health
+              port: 9200
+            initialDelaySeconds: 120
+            periodSeconds: 20
+            failureThreshold: 5
           env:
             - name: ES_JAVA_OPTS
               value: "-Xms2g -Xmx2g"
             - name: cluster.name
               value: "elasticsearch"
+            - name: discovery.type
+              value: "multi-node"
             - name: xpack.security.enabled
               value: "false"
             - name: network.host

diff --git a/terraform/environments/eks/main.tf b/terraform/environments/eks/main.tf
@@ -118,15 +118,18 @@ module "eks" {
   app_namespace_prod          = var.app_namespace_prod
   app_service_account_prod    = var.app_service_account_prod
   # Env node group scaling
-  ng_staging_min_size     = var.ng_staging_min_size
-  ng_staging_desired_size = var.ng_staging_desired_size
-  ng_staging_max_size     = var.ng_staging_max_size
-  ng_sandbox_min_size     = var.ng_sandbox_min_size
-  ng_sandbox_desired_size = var.ng_sandbox_desired_size
-  ng_sandbox_max_size     = var.ng_sandbox_max_size
-  ng_prod_min_size        = var.ng_prod_min_size
-  ng_prod_desired_size    = var.ng_prod_desired_size
-  ng_prod_max_size        = var.ng_prod_max_size
+  ng_staging_min_size           = var.ng_staging_min_size
+  ng_staging_desired_size       = var.ng_staging_desired_size
+  ng_staging_max_size           = var.ng_staging_max_size
+  ng_sandbox_min_size           = var.ng_sandbox_min_size
+  ng_sandbox_desired_size       = var.ng_sandbox_desired_size
+  ng_sandbox_max_size           = var.ng_sandbox_max_size
+  ng_sandbox_large_min_size     = var.ng_sandbox_large_min_size
+  ng_sandbox_large_desired_size = var.ng_sandbox_large_desired_size
+  ng_sandbox_large_max_size     = var.ng_sandbox_large_max_size
+  ng_prod_min_size              = var.ng_prod_min_size
+  ng_prod_desired_size          = var.ng_prod_desired_size
+  ng_prod_max_size              = var.ng_prod_max_size
 }
 
 module "application_secret" {
@@ -231,9 +234,14 @@ module "cloudwatch_slack_forwarder" {
 
   log_filters = [
     {
-      name           = "es-warn-prod"
+      name           = "es-warn-sandbox"
       log_group_name = "/aws/containerinsights/ce-registry-eks/application"
-      filter_pattern = "\"WARN\" \"elasticsearch\" \"credreg-prod\""
+      filter_pattern = "\"WARN\" \"elasticsearch\" \"credreg-sandbox\""
+    },
+    {
+      name           = "k8s-backoff-prod-staging"
+      log_group_name = "/aws/containerinsights/ce-registry-eks/application"
+      filter_pattern = "{ ($.log_processed.reason = \"BackOff\") && (($.log_processed.involvedObject.namespace = \"credreg-prod\") || ($.log_processed.involvedObject.namespace = \"credreg-sandbox\")) }"
     },
   ]
 

diff --git a/terraform/environments/eks/terraform.tfvars b/terraform/environments/eks/terraform.tfvars
@@ -30,10 +30,13 @@ route53_hosted_zone_id = "Z1N75467P1FUL5"
 # Env node group scaling
 ng_staging_min_size     = 1
 ng_staging_desired_size = 1
-ng_staging_max_size     = 4
+ng_staging_max_size     = 6
 ng_sandbox_min_size     = 1
 ng_sandbox_desired_size = 1
 ng_sandbox_max_size     = 5
+ng_sandbox_large_min_size     = 1
+ng_sandbox_large_desired_size = 2
+ng_sandbox_large_max_size     = 4
 ng_prod_min_size        = 2
 ng_prod_desired_size    = 2
 ng_prod_max_size        = 8

diff --git a/terraform/environments/eks/variables.tf b/terraform/environments/eks/variables.tf
@@ -136,6 +136,21 @@ variable "ng_sandbox_max_size" {
   description = "Sandbox node group max size"
 }
 
+variable "ng_sandbox_large_min_size" {
+  type        = number
+  description = "Sandbox large (t3.large) node group min size"
+}
+
+variable "ng_sandbox_large_desired_size" {
+  type        = number
+  description = "Sandbox large (t3.large) node group desired size"
+}
+
+variable "ng_sandbox_large_max_size" {
+  type        = number
+  description = "Sandbox large (t3.large) node group max size"
+}
+
 variable "ng_prod_min_size" {
   type        = number
   description = "Production node group min size"