chore: prod to test backmerge

.github/configs/ct.yaml

@@ -6,7 +6,7 @@ target-branch: prod
 chart-dirs:
   - charts
 chart-repos: []
-validate-chart-schema: false
+validate-chart-schema: true
 validate-maintainers: false
 validate-yaml: true
 exclude-deprecated: true

.github/workflows/helm-dev-lint.yml

@@ -1,43 +1,60 @@
 # CI Workflow for Agent Helm chart
 name: CI
 
-# Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
     branches: [ "main" ]
   pull_request:
     branches: [ "main"]
-
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  helm-test:
+  helm-lint-and-validate:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      
+
       - name: Add dependency chart repos
         run: |
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo add opencost https://opencost.github.io/opencost-helm-chart
+          helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+          helm repo add bitnami https://charts.bitnami.com/bitnami
 
-      - uses: azure/setup-helm@v3
+      - uses: azure/setup-helm@v4
         with:
-          version: v3.10.0
+          version: v3.14.0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
-          check-latest: true
+          python-version: '3.12'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.3.1
+        uses: helm/chart-testing-action@v2.7.0
 
       - name: Run chart-testing (lint)
         run: ct lint --debug --config ./.github/configs/ct.yaml --lint-conf ./.github/configs/lintconf.yaml --check-version-increment=false
+
+      - name: Build chart dependencies
+        run: helm dependency build charts/nudgebee-agent
+
+      - name: Render templates
+        run: helm template nudgebee-agent charts/nudgebee-agent --namespace nudgebee-agent > /tmp/rendered.yaml
+
+      - name: Install kubeconform
+        run: |
+          curl -sSLo /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz
+          tar -xzf /tmp/kubeconform.tar.gz -C /usr/local/bin kubeconform
+
+      - name: Validate rendered templates
+        run: |
+          kubeconform \
+            -strict \
+            -summary \
+            -schema-location default \
+            -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \
+            -skip SecurityContextConstraints \
+            /tmp/rendered.yaml

.github/workflows/helm-prod-test.yml

@@ -1,48 +1,43 @@
 # CI Workflow for Agent Helm chart
 name: CI
 
-# Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
     branches: [ "prod" ]
   pull_request:
     branches: [ "prod"]
-
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  helm-test:
+  helm-lint-and-validate:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      
+
       - name: Add dependency chart repos
         run: |
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo add opencost https://opencost.github.io/opencost-helm-chart
+          helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+          helm repo add bitnami https://charts.bitnami.com/bitnami
 
-      - uses: azure/setup-helm@v3
+      - uses: azure/setup-helm@v4
         with:
-          version: v3.10.0
+          version: v3.14.0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
-          check-latest: true
+          python-version: '3.12'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.3.1
+        uses: helm/chart-testing-action@v2.7.0
 
       - name: List changed charts
         id: list-changed
         run: |
-          ## If executed with debug this won't work anymore.
           changed=$(ct --config ./.github/configs/ct.yaml list-changed)
           charts=$(echo "$changed" | tr '\n' ' ' | xargs)
           if [[ -n "$changed" ]]; then
@@ -52,3 +47,24 @@ jobs:
 
       - name: Run chart-testing (lint)
         run: ct lint --debug --config ./.github/configs/ct.yaml --lint-conf ./.github/configs/lintconf.yaml
+
+      - name: Build chart dependencies
+        run: helm dependency build charts/nudgebee-agent
+
+      - name: Render templates
+        run: helm template nudgebee-agent charts/nudgebee-agent --namespace nudgebee-agent > /tmp/rendered.yaml
+
+      - name: Install kubeconform
+        run: |
+          curl -sSLo /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz
+          tar -xzf /tmp/kubeconform.tar.gz -C /usr/local/bin kubeconform
+
+      - name: Validate rendered templates
+        run: |
+          kubeconform \
+            -strict \
+            -summary \
+            -schema-location default \
+            -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \
+            -skip SecurityContextConstraints \
+            /tmp/rendered.yaml

.github/workflows/release-rc.yml

@@ -81,7 +81,7 @@ jobs:
       - name: Update Helm Package for Nudgebee RC
         working-directory: ./charts/nudgebee-agent
         run: |
-          nudgebee_app_image=`aws ecr describe-images --repository-name nudgebee-agent --filter tagStatus=TAGGED --query 'sort_by(imageDetails[?imageTags], &imagePushedAt)[-1].imageTags[0]' --region us-east-1 --output text`
+          nudgebee_app_image=`aws ecr describe-images --repository-name nudgebee-agent --filter tagStatus=TAGGED --query 'sort_by(imageDetails[?imageTags], &imagePushedAt)[-1].imageTags[0]' --region us-east-1 --output text --no-paginate`
           echo "nudgebee_app_image: $nudgebee_app_image"
           yq -i ".runner.image.tag=\"$nudgebee_app_image\"" values.yaml
 

.github/workflows/release.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Update Helm Package for Nudgebee
         working-directory: ./charts/nudgebee-agent
         run: |
-          nudgebee_app_image=`aws ecr describe-images --repository-name nudgebee-agent --filter tagStatus=TAGGED --query 'sort_by(imageDetails[?imageTags], &imagePushedAt)[-1].imageTags[0]' --region us-east-1 --output text`
+          nudgebee_app_image=`aws ecr describe-images --repository-name nudgebee-agent --filter tagStatus=TAGGED --query 'sort_by(imageDetails[?imageTags], &imagePushedAt)[-1].imageTags[0]' --region us-east-1 --output text --no-paginate`
           echo "nudgebee_app_image: $nudgebee_app_image"
           yq -i ".runner.image.tag=\"$nudgebee_app_image\"" values.yaml
 

README.md

@@ -1,5 +1,3 @@
-[![CI](https://github.com/nudgebee/k8s-agent/actions/workflows/helm-test.yml/badge.svg?branch=main)](https://github.com/nudgebee/k8s-agent/actions/workflows/helm-test.yml) [![Release Charts](https://github.com/nudgebee/k8s-agent/actions/workflows/release.yml/badge.svg?branch=main)](https://github.com/nudgebee/k8s-agent/actions/workflows/release.yml)
-
 # NudgeBee Kubernetes agent
 Module to connect kubernetes to NudgeBee.
 

charts/nudgebee-agent/Chart.yaml

@@ -6,8 +6,8 @@ icon: https://nudgebee-documents.s3.amazonaws.com/images/Nudgebee-logo.png
 # these are set to the right value by .github/workflows/release.yaml
 # we use 0.0.1 as a placeholder for the version` because Helm wont allow `0.0.0` and we want to be able to run
 # `helm install` on development checkouts without updating this file. the version doesn't matter in that case anyway
-version: 0.0.115
-appVersion: 0.0.115
+version: 0.0.124
+appVersion: 0.0.124
 dependencies:
 - name: opencost
   version: 2.0.1

charts/nudgebee-agent/templates/daemonset.yaml

@@ -89,4 +89,35 @@ spec:
         - hostPath:
             path: /sys/kernel/debug
           name: debugfs
+{{- end }}
+{{- if (and .Values.nodeAgent.enabled .Values.nodeAgent.service.enabled) }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "nudgebee-agent.fullname" . }}-node-agent
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "80"
+    prometheus.io/path: /metrics
+    {{- with .Values.nodeAgent.service.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    {{- include "nudgebee-agent.labels" . | nindent 4 }}
+    component: node-agent
+    {{- with .Values.nodeAgent.service.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  clusterIP: None
+  selector:
+    {{- include "node-agent.selectorLabels" . | nindent 4 }}
+    component: node-agent
+  ports:
+  - name: http
+    port: 80
+    targetPort: http
+    protocol: TCP
 {{- end }}

charts/nudgebee-agent/templates/prometheus-alert-manager-config.yaml

@@ -14,8 +14,6 @@ spec:
     receiver: 'nudgebee-agent'
   receivers:
   - name: 'nudgebee-agent'
-    matchers:
-    - severity=~".*"
     webhookConfigs:
     - url: 'http://{{ include "nudgebee-agent.fullname" . }}-runner.{{ .Release.Namespace }}.svc/api/alerts'
       sendResolved: true

charts/nudgebee-agent/templates/prometheus-alert-rule.yaml

@@ -1,4 +1,13 @@
 {{- if (and (not (default false .Values.runner.victoria_metrics_enabled)) (default false .Values.alertmanager.create_nb_default_rules)) }}
+{{- /*
+Container-ID exclusions for the log-error and HTTP-failure rules.
+
+The log-error regex additionally excludes log-collection infrastructure
+(fluent-bit, loki, opensearch, etc.), which emits steady error-level
+log streams that are transport/ingest failures — not app-level signal.
+*/ -}}
+{{- $excludeLog := ".*(prometheus|grafana|kube-system|nudgebee-agent|containerd|kubelet|keda|actions-runner-system-1|fluent-bit|fluentd|loki|opensearch|elasticsearch|logstash|datadog|newrelic|otel|promtail).*" -}}
+{{- $excludeApi := ".*(prometheus|grafana|kube-system|nudgebee-agent|containerd|kubelet|keda|karpenter|actions-runner-system-1).*" -}}
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
@@ -13,48 +22,71 @@ spec:
       rules:
         - alert: KubeHpaMaxedOut
           expr: >-
-            kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} 
-              == 
+            kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
+              ==
             kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}
             > 1
           for: 15m
           labels:
             severity: warning
           annotations:
             description: >-
-              HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.horizontalpodautoscaler }}`}} 
+              HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.horizontalpodautoscaler }}`}}
               has been running at max replicas for longer than 15 minutes.
             summary: HPA is running at max replicas
+        # Fires when a container's error/critical log rate is >3x its historical
+        # baseline (1h window offset 1h — the offset keeps an ongoing spike from
+        # polluting the baseline and silencing sustained incidents). The 0.1/s
+        # floor is an activity floor (≥6 err/min) that avoids ratio blowups on
+        # idle containers. The baseline>0 guard prevents cold-start containers
+        # (no historical data) from firing on their first error burst.
         - alert: HighErrorCriticalLogs
-          expr: >-
-            increase(container_log_messages_total{level=~"error|critical",
-            container_id!~".*(prometheus|grafana|kube-system|nudgebee-agent|containerd|kubelet|keda|actions-runner-system-1).*"}[5m])
-            > 1
-          for: 5m
+          expr: |
+            rate(container_log_messages_total{level=~"error|critical", container_id!~"{{ $excludeLog }}"}[5m])
+              > 3 * rate(container_log_messages_total{level=~"error|critical", container_id!~"{{ $excludeLog }}"}[1h] offset 1h)
+            and
+            rate(container_log_messages_total{level=~"error|critical", container_id!~"{{ $excludeLog }}"}[5m]) > 0.1
+            and
+            rate(container_log_messages_total{level=~"error|critical", container_id!~"{{ $excludeLog }}"}[1h] offset 1h) > 0
+          for: 10m
           annotations:
             summary: "High error/critical logs - Sample: {{`{{ printf \"%.80s\" $labels.sample }}`}}"
-            description: "The total count of container log messages with error or critical level is higher for the past 5 minutes, grouped by container_id. Container ID: {{`{{ $labels.container_id }}`}} Log Sample: {{`{{ $labels.sample }}`}} Failure Count: {{`{{ printf \"%.0f\" $value }}`}}"
+            description: "Container log error rate is more than 3x its 1h baseline. Container ID: {{`{{ $labels.container_id }}`}} Log Sample: {{`{{ $labels.sample }}`}} Current rate (err/s): {{`{{ printf \"%.2f\" $value }}`}}"
           labels:
-            severity: critical
+            severity: warning
+        # Fires when >5% of a container's HTTP traffic is 5xx for 10m, with an
+        # activity floor of 0.1 req/s so ratios don't blow up on low-volume
+        # containers. 4xx is excluded — those are client errors (bad request,
+        # unauthorised, etc.) and don't indicate the application failing.
         - alert: ApplicationAPIFailures
-          expr: >-
-            increase(container_http_requests_total{container_id!~".*(prometheus|grafana|kube-system|nudgebee-agent|containerd|kubelet|keda|karpenter|actions-runner-system-1).*",
-            status=~"5..|4.."}[5m]) > 1
-          for: 5m
+          expr: |
+            (
+              sum by (container_id, method, path) (rate(container_http_requests_total{status=~"5..", container_id!~"{{ $excludeApi }}"}[5m]))
+              / on (container_id) group_left ()
+              sum by (container_id) (rate(container_http_requests_total{container_id!~"{{ $excludeApi }}"}[5m]))
+            ) > 0.05
+            and on (container_id)
+            sum by (container_id) (rate(container_http_requests_total{container_id!~"{{ $excludeApi }}"}[5m])) > 0.1
+          for: 10m
           annotations:
-            summary: "API Failures - {{`{{ $labels.method }}`}} {{`{{ printf \"%.50s\" $labels.path }}`}} ({{`{{ $labels.status }}`}})"
-            description: "Application reported API failure. Container ID: {{`{{ $labels.container_id }}`}} Request Path: {{`{{ $labels.path }}`}} Request Method: {{`{{ $labels.method }}`}} Failure Count: {{`{{ printf \"%.0f\" $value }}`}}"
+            summary: "High 5xx rate - {{`{{ $labels.method }}`}} {{`{{ printf \"%.50s\" $labels.path }}`}}"
+            description: "Aggregate 5xx rate across status codes exceeds 5% of total traffic. Container ID: {{`{{ $labels.container_id }}`}} Request Path: {{`{{ $labels.path }}`}} Request Method: {{`{{ $labels.method }}`}} 5xx Ratio: {{`{{ printf \"%.2f\" $value }}`}}"
           labels:
             severity: critical
+        # Fires when a pod has been in Terminating state for >30 minutes — that
+        # is 60x the Kubernetes default terminationGracePeriodSeconds (30s), well
+        # past any legitimate graceful termination. max_over_time smooths across
+        # kube-state-metrics scrape gaps so the alert stays firing instead of
+        # flapping on each missed scrape.
         - alert: KubePodStuckTerminating
-          expr: >-
-            count(kube_pod_deletion_timestamp) by (namespace, pod) *
-            count(kube_pod_status_reason{reason="NodeLost"} == 0) by
-            (namespace, pod) > 0
-          for: 5m
+          expr: |
+            (time() - max_over_time(kube_pod_deletion_timestamp[20m])) > 1800
+            unless on (namespace, pod)
+            max_over_time(kube_pod_status_reason{reason="NodeLost"}[20m]) == 1
+          for: 15m
           labels:
-            severity: critical
+            severity: warning
           annotations:
             summary: Pod stuck in terminating state
-            description: "Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} blocked in Terminating state."
+            description: "Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} blocked in Terminating state for over 30 minutes."
 {{- end }}