diff --git a/.github/configs/ct.yaml b/.github/configs/ct.yaml
index 6a3aebb8..cfa72bb0 100644
--- a/.github/configs/ct.yaml
+++ b/.github/configs/ct.yaml
@@ -6,7 +6,7 @@ target-branch: prod
 chart-dirs:
   - charts
 chart-repos: []
-validate-chart-schema: false
+validate-chart-schema: true
 validate-maintainers: false
 validate-yaml: true
 exclude-deprecated: true
diff --git a/.github/workflows/helm-dev-lint.yml b/.github/workflows/helm-dev-lint.yml
index 88a230ab..24bd8ae9 100644
--- a/.github/workflows/helm-dev-lint.yml
+++ b/.github/workflows/helm-dev-lint.yml
@@ -1,43 +1,60 @@
 # CI Workflow for Agent Helm chart
 name: CI
 
-# Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
     branches: [ "main" ]
   pull_request:
     branches: [ "main"]
-
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  helm-test:
+  helm-lint-and-validate:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      
+
       - name: Add dependency chart repos
         run: |
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo add opencost https://opencost.github.io/opencost-helm-chart
+          helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+          helm repo add bitnami https://charts.bitnami.com/bitnami
 
-      - uses: azure/setup-helm@v3
+      - uses: azure/setup-helm@v4
         with:
-          version: v3.10.0
+          version: v3.14.0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
-          check-latest: true
+          python-version: '3.12'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.3.1
+        uses: helm/chart-testing-action@v2.7.0
 
       - name: Run chart-testing (lint)
         run: ct lint --debug --config ./.github/configs/ct.yaml --lint-conf ./.github/configs/lintconf.yaml --check-version-increment=false
+
+      - name: Build chart dependencies
+        run: helm dependency build charts/nudgebee-agent
+
+      - name: Render templates
+        run: helm template nudgebee-agent charts/nudgebee-agent --namespace nudgebee-agent > /tmp/rendered.yaml
+
+      - name: Install kubeconform
+        run: |
+          curl -sSLo /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz
+          tar -xzf /tmp/kubeconform.tar.gz -C /usr/local/bin kubeconform
+
+      - name: Validate rendered templates
+        run: |
+          kubeconform \
+            -strict \
+            -summary \
+            -schema-location default \
+            -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \
+            -skip SecurityContextConstraints \
+            /tmp/rendered.yaml
diff --git a/.github/workflows/helm-prod-test.yml b/.github/workflows/helm-prod-test.yml
index c3ade027..28db426a 100644
--- a/.github/workflows/helm-prod-test.yml
+++ b/.github/workflows/helm-prod-test.yml
@@ -1,48 +1,43 @@
 # CI Workflow for Agent Helm chart
 name: CI
 
-# Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
     branches: [ "prod" ]
   pull_request:
     branches: [ "prod"]
-
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  helm-test:
+  helm-lint-and-validate:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      
+
       - name: Add dependency chart repos
         run: |
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo add opencost https://opencost.github.io/opencost-helm-chart
+          helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+          helm repo add bitnami https://charts.bitnami.com/bitnami
 
-      - uses: azure/setup-helm@v3
+      - uses: azure/setup-helm@v4
         with:
-          version: v3.10.0
+          version: v3.14.0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
-          check-latest: true
+          python-version: '3.12'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.3.1
+        uses: helm/chart-testing-action@v2.7.0
 
       - name: List changed charts
         id: list-changed
         run: |
-          ## If executed with debug this won't work anymore.
           changed=$(ct --config ./.github/configs/ct.yaml list-changed)
           charts=$(echo "$changed" | tr '\n' ' ' | xargs)
           if [[ -n "$changed" ]]; then
@@ -52,3 +47,24 @@ jobs:
 
       - name: Run chart-testing (lint)
         run: ct lint --debug --config ./.github/configs/ct.yaml --lint-conf ./.github/configs/lintconf.yaml
+
+      - name: Build chart dependencies
+        run: helm dependency build charts/nudgebee-agent
+
+      - name: Render templates
+        run: helm template nudgebee-agent charts/nudgebee-agent --namespace nudgebee-agent > /tmp/rendered.yaml
+
+      - name: Install kubeconform
+        run: |
+          curl -sSLo /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz
+          tar -xzf /tmp/kubeconform.tar.gz -C /usr/local/bin kubeconform
+
+      - name: Validate rendered templates
+        run: |
+          kubeconform \
+            -strict \
+            -summary \
+            -schema-location default \
+            -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \
+            -skip SecurityContextConstraints \
+            /tmp/rendered.yaml
diff --git a/.github/workflows/release-rc.yml b/.github/workflows/release-rc.yml
index b088d986..554b0d7a 100644
--- a/.github/workflows/release-rc.yml
+++ b/.github/workflows/release-rc.yml
@@ -81,7 +81,7 @@ jobs:
       - name: Update Helm Package for Nudgebee RC
         working-directory: ./charts/nudgebee-agent
         run: |
-          nudgebee_app_image=`aws ecr describe-images --repository-name nudgebee-agent --filter tagStatus=TAGGED --query 'sort_by(imageDetails[?imageTags], &imagePushedAt)[-1].imageTags[0]' --region us-east-1 --output text`
+          nudgebee_app_image=`aws ecr describe-images --repository-name nudgebee-agent --filter tagStatus=TAGGED --query 'sort_by(imageDetails[?imageTags], &imagePushedAt)[-1].imageTags[0]' --region us-east-1 --output text --no-paginate`
           echo "nudgebee_app_image: $nudgebee_app_image"
           yq -i ".runner.image.tag=\"$nudgebee_app_image\"" values.yaml
 
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6fe27918..b66219fb 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -37,7 +37,7 @@ jobs:
       - name: Update Helm Package for Nudgebee
         working-directory: ./charts/nudgebee-agent
         run: |
-          nudgebee_app_image=`aws ecr describe-images --repository-name nudgebee-agent --filter tagStatus=TAGGED --query 'sort_by(imageDetails[?imageTags], &imagePushedAt)[-1].imageTags[0]' --region us-east-1 --output text`
+          nudgebee_app_image=`aws ecr describe-images --repository-name nudgebee-agent --filter tagStatus=TAGGED --query 'sort_by(imageDetails[?imageTags], &imagePushedAt)[-1].imageTags[0]' --region us-east-1 --output text --no-paginate`
           echo "nudgebee_app_image: $nudgebee_app_image"
           yq -i ".runner.image.tag=\"$nudgebee_app_image\"" values.yaml
 
diff --git a/README.md b/README.md
index f09c8b25..a83479c5 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,3 @@
-[![CI](https://github.com/nudgebee/k8s-agent/actions/workflows/helm-test.yml/badge.svg?branch=main)](https://github.com/nudgebee/k8s-agent/actions/workflows/helm-test.yml) [![Release Charts](https://github.com/nudgebee/k8s-agent/actions/workflows/release.yml/badge.svg?branch=main)](https://github.com/nudgebee/k8s-agent/actions/workflows/release.yml)
-
 # NudgeBee Kubernetes agent
 Module to connect kubernetes to NudgeBee.
 
diff --git a/charts/nudgebee-agent/Chart.yaml b/charts/nudgebee-agent/Chart.yaml
index 0f4337ae..72049972 100644
--- a/charts/nudgebee-agent/Chart.yaml
+++ b/charts/nudgebee-agent/Chart.yaml
@@ -6,8 +6,8 @@ icon: https://nudgebee-documents.s3.amazonaws.com/images/Nudgebee-logo.png
 # these are set to the right value by .github/workflows/release.yaml
 # we use 0.0.1 as a placeholder for the version` because Helm wont allow `0.0.0` and we want to be able to run
 # `helm install` on development checkouts without updating this file. the version doesn't matter in that case anyway
-version: 0.0.115
-appVersion: 0.0.115
+version: 0.0.124
+appVersion: 0.0.124
 dependencies:
 - name: opencost
   version: 2.0.1
diff --git a/charts/nudgebee-agent/templates/daemonset.yaml b/charts/nudgebee-agent/templates/daemonset.yaml
index 38514645..877eb6ce 100644
--- a/charts/nudgebee-agent/templates/daemonset.yaml
+++ b/charts/nudgebee-agent/templates/daemonset.yaml
@@ -89,4 +89,35 @@ spec:
         - hostPath:
             path: /sys/kernel/debug
           name: debugfs
+{{- end }}
+{{- if (and .Values.nodeAgent.enabled .Values.nodeAgent.service.enabled) }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "nudgebee-agent.fullname" . }}-node-agent
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "80"
+    prometheus.io/path: /metrics
+    {{- with .Values.nodeAgent.service.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    {{- include "nudgebee-agent.labels" . | nindent 4 }}
+    component: node-agent
+    {{- with .Values.nodeAgent.service.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  clusterIP: None
+  selector:
+    {{- include "node-agent.selectorLabels" . | nindent 4 }}
+    component: node-agent
+  ports:
+  - name: http
+    port: 80
+    targetPort: http
+    protocol: TCP
 {{- end }}
\ No newline at end of file
diff --git a/charts/nudgebee-agent/templates/prometheus-alert-manager-config.yaml b/charts/nudgebee-agent/templates/prometheus-alert-manager-config.yaml
index c19fe644..3414dc22 100644
--- a/charts/nudgebee-agent/templates/prometheus-alert-manager-config.yaml
+++ b/charts/nudgebee-agent/templates/prometheus-alert-manager-config.yaml
@@ -14,8 +14,6 @@ spec:
     receiver: 'nudgebee-agent'
   receivers:
   - name: 'nudgebee-agent'
-    matchers:
-    - severity=~".*"
     webhookConfigs:
     - url: 'http://{{ include "nudgebee-agent.fullname" . }}-runner.{{ .Release.Namespace }}.svc/api/alerts'
       sendResolved: true
diff --git a/charts/nudgebee-agent/templates/prometheus-alert-rule.yaml b/charts/nudgebee-agent/templates/prometheus-alert-rule.yaml
index ca9397e9..dd7cc2cc 100644
--- a/charts/nudgebee-agent/templates/prometheus-alert-rule.yaml
+++ b/charts/nudgebee-agent/templates/prometheus-alert-rule.yaml
@@ -1,4 +1,13 @@
 {{- if (and (not (default false .Values.runner.victoria_metrics_enabled)) (default false .Values.alertmanager.create_nb_default_rules)) }}
+{{- /*
+Container-ID exclusions for the log-error and HTTP-failure rules.
+
+The log-error regex additionally excludes log-collection infrastructure
+(fluent-bit, loki, opensearch, etc.), which emits steady error-level
+log streams that are transport/ingest failures — not app-level signal.
+*/ -}}
+{{- $excludeLog := ".*(prometheus|grafana|kube-system|nudgebee-agent|containerd|kubelet|keda|actions-runner-system-1|fluent-bit|fluentd|loki|opensearch|elasticsearch|logstash|datadog|newrelic|otel|promtail).*" -}}
+{{- $excludeApi := ".*(prometheus|grafana|kube-system|nudgebee-agent|containerd|kubelet|keda|karpenter|actions-runner-system-1).*" -}}
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
@@ -13,8 +22,8 @@ spec:
       rules:
         - alert: KubeHpaMaxedOut
           expr: >-
-            kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} 
-              == 
+            kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
+              ==
             kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}
             > 1
           for: 15m
@@ -22,39 +31,62 @@ spec:
             severity: warning
           annotations:
             description: >-
-              HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.horizontalpodautoscaler }}`}} 
+              HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.horizontalpodautoscaler }}`}}
               has been running at max replicas for longer than 15 minutes.
             summary: HPA is running at max replicas
+        # Fires when a container's error/critical log rate is >3x its historical
+        # baseline (1h window offset 1h — the offset keeps an ongoing spike from
+        # polluting the baseline and silencing sustained incidents). The 0.1/s
+        # floor is an activity floor (≥6 err/min) that avoids ratio blowups on
+        # idle containers. The baseline>0 guard prevents cold-start containers
+        # (no historical data) from firing on their first error burst.
         - alert: HighErrorCriticalLogs
-          expr: >-
-            increase(container_log_messages_total{level=~"error|critical",
-            container_id!~".*(prometheus|grafana|kube-system|nudgebee-agent|containerd|kubelet|keda|actions-runner-system-1).*"}[5m])
-            > 1
-          for: 5m
+          expr: |
+            rate(container_log_messages_total{level=~"error|critical", container_id!~"{{ $excludeLog }}"}[5m])
+              > 3 * rate(container_log_messages_total{level=~"error|critical", container_id!~"{{ $excludeLog }}"}[1h] offset 1h)
+            and
+            rate(container_log_messages_total{level=~"error|critical", container_id!~"{{ $excludeLog }}"}[5m]) > 0.1
+            and
+            rate(container_log_messages_total{level=~"error|critical", container_id!~"{{ $excludeLog }}"}[1h] offset 1h) > 0
+          for: 10m
           annotations:
             summary: "High error/critical logs - Sample: {{`{{ printf \"%.80s\" $labels.sample }}`}}"
-            description: "The total count of container log messages with error or critical level is higher for the past 5 minutes, grouped by container_id. Container ID: {{`{{ $labels.container_id }}`}} Log Sample: {{`{{ $labels.sample }}`}} Failure Count: {{`{{ printf \"%.0f\" $value }}`}}"
+            description: "Container log error rate is more than 3x its 1h baseline. Container ID: {{`{{ $labels.container_id }}`}} Log Sample: {{`{{ $labels.sample }}`}} Current rate (err/s): {{`{{ printf \"%.2f\" $value }}`}}"
           labels:
-            severity: critical
+            severity: warning
+        # Fires when >5% of a container's HTTP traffic is 5xx for 10m, with an
+        # activity floor of 0.1 req/s so ratios don't blow up on low-volume
+        # containers. 4xx is excluded — those are client errors (bad request,
+        # unauthorised, etc.) and don't indicate the application failing.
         - alert: ApplicationAPIFailures
-          expr: >-
-            increase(container_http_requests_total{container_id!~".*(prometheus|grafana|kube-system|nudgebee-agent|containerd|kubelet|keda|karpenter|actions-runner-system-1).*",
-            status=~"5..|4.."}[5m]) > 1
-          for: 5m
+          expr: |
+            (
+              sum by (container_id, method, path) (rate(container_http_requests_total{status=~"5..", container_id!~"{{ $excludeApi }}"}[5m]))
+              / on (container_id) group_left ()
+              sum by (container_id) (rate(container_http_requests_total{container_id!~"{{ $excludeApi }}"}[5m]))
+            ) > 0.05
+            and on (container_id)
+            sum by (container_id) (rate(container_http_requests_total{container_id!~"{{ $excludeApi }}"}[5m])) > 0.1
+          for: 10m
           annotations:
-            summary: "API Failures - {{`{{ $labels.method }}`}} {{`{{ printf \"%.50s\" $labels.path }}`}} ({{`{{ $labels.status }}`}})"
-            description: "Application reported API failure. Container ID: {{`{{ $labels.container_id }}`}} Request Path: {{`{{ $labels.path }}`}} Request Method: {{`{{ $labels.method }}`}} Failure Count: {{`{{ printf \"%.0f\" $value }}`}}"
+            summary: "High 5xx rate - {{`{{ $labels.method }}`}} {{`{{ printf \"%.50s\" $labels.path }}`}}"
+            description: "Aggregate 5xx rate across status codes exceeds 5% of total traffic. Container ID: {{`{{ $labels.container_id }}`}} Request Path: {{`{{ $labels.path }}`}} Request Method: {{`{{ $labels.method }}`}} 5xx Ratio: {{`{{ printf \"%.2f\" $value }}`}}"
           labels:
             severity: critical
+        # Fires when a pod has been in Terminating state for >30 minutes — that
+        # is 60x the Kubernetes default terminationGracePeriodSeconds (30s), well
+        # past any legitimate graceful termination. max_over_time smooths across
+        # kube-state-metrics scrape gaps so the alert stays firing instead of
+        # flapping on each missed scrape.
         - alert: KubePodStuckTerminating
-          expr: >-
-            count(kube_pod_deletion_timestamp) by (namespace, pod) *
-            count(kube_pod_status_reason{reason="NodeLost"} == 0) by
-            (namespace, pod) > 0
-          for: 5m
+          expr: |
+            (time() - max_over_time(kube_pod_deletion_timestamp[20m])) > 1800
+            unless on (namespace, pod)
+            max_over_time(kube_pod_status_reason{reason="NodeLost"}[20m]) == 1
+          for: 15m
           labels:
-            severity: critical
+            severity: warning
           annotations:
             summary: Pod stuck in terminating state
-            description: "Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} blocked in Terminating state."
+            description: "Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} blocked in Terminating state for over 30 minutes."
 {{- end }}
diff --git a/charts/nudgebee-agent/values.yaml b/charts/nudgebee-agent/values.yaml
index ba0206ee..68b9ba64 100644
--- a/charts/nudgebee-agent/values.yaml
+++ b/charts/nudgebee-agent/values.yaml
@@ -265,6 +265,8 @@ builtinPlaybooks:
   - triggers:
       - on_pod_crash_loop:
           restart_reason: "CrashLoopBackOff"
+          rate_limit: 14400 # 4 hours, per workload
+          restart_count: 2
     actions:
       - report_crash_loop: {}
       - resource_events_enricher: {}
@@ -273,7 +275,9 @@ builtinPlaybooks:
           previous: true
       - impacted_services_enricher: {}
   - triggers:
-      - on_image_pull_backoff: {}
+      - on_image_pull_backoff:
+          rate_limit: 14400 # 4 hours, per workload
+          fire_delay: 120 # seconds, avoids false positives on startup
     actions:
       - image_pull_backoff_reporter: {}
       - resource_events_enricher: {}
@@ -281,7 +285,7 @@ builtinPlaybooks:
   # playbooks for non-prometheus based monitoring that use prometheus for enrichment
   - triggers:
       - on_pod_oom_killed:
-          rate_limit: 3600
+          rate_limit: 3600 # 1 hour, per workload
     actions:
       - pod_oom_killer_enricher: {}
       - logs_enricher:
@@ -636,7 +640,9 @@ runnerServiceAccount:
 runner:
   image:
     repository: registry.nudgebee.com/nudgebee-agent
-    tag: 2025-10-03T18-16-22_9f4528d40aae29f249adc0fcdfcb2c4901f63424
+    tag: |-
+      2026-04-29T06-02-05_5678ecd5b0955fbf7cadf4306e17d056903a12dc
+      2025-11-12T06-00-45_8ab57887ffd3ee0611c8b3e1a51413d1e0660445
   imagePullPolicy: IfNotPresent
   log_level: INFO
   resources:
@@ -653,14 +659,14 @@ runner:
   extraVolumes: []
   extraVolumeMounts: []
   image_registry: registry.nudgebee.com
-  krr_image_override: krr-public:2025-06-16T04-25-05_93361fa666f6eabe55ece9c9c918c080e38b73ab
+  krr_image_override: krr-public:2025-11-12T10-27-20_05e19006ab7b743e092267132d3b40e2c0b7b43c
   relay_address: wss://relay.nudgebee.com/register
   profiler_image_override: 2025-05-06T10-08-00_1a1779d4166a81b292375a950aeb49b86ed10b33
   kubepug_image_override: kubepug:2025-08-14T05-16-09_6da6481b63b699ab4c61e86e1c611a182319d3de
   nova_image_override: nova:2025-07-04T04-12-43_76ed8fffc46f03f798e9dd6a975890064807d6e2
-  clickhouse_enabled: true
+  clickhouse_enabled: false
   clickhouse_secret: ""
-  runbook_sidecar_image_tag: 2026-01-09T08-56-51_1bc8ed44579b442306416c9a2733c4ae4c124873
+  runbook_sidecar_image_tag: 2026-01-30T07-39-18_68169ef8bece3b4d3ccb6e80a12023d2ab305181-arm64
   victoria_metrics_enabled: false
   loki:
     url: ""
@@ -738,11 +744,15 @@ nodeAgent:
   podmonitor:
     enabled: true
     azuremanaged: false
+  service:
+    enabled: false
+    labels: {}
+    annotations: {}
   podAnnotations: {}
   image:
     repository: registry.nudgebee.com/nudgebee-node-agent
     pullPolicy: IfNotPresent
-    tag: 2026-01-12T03-44-08_8755b8391c96d6fefd4f4c6726d3796fe7bdcc5c
+    tag: 2026-04-29T06-15-05_7fd20886980a75b7b9c8ca561621a19e1ff0976f
   resources:
     requests:
       cpu: "100m"
@@ -827,38 +837,70 @@ opentelemetry-collector:
       enabled: false
   config:
     processors:
-      probabilistic_sampler:
-        sampling_percentage: 10
+      # Drop early under memory pressure rather than OOM-killing the collector.
+      # Must be the first processor in every pipeline.
+      memory_limiter:
+        check_interval: 1s
+        limit_percentage: 80
+        spike_limit_percentage: 25
       filter/drop_namespaces:
         error_mode: ignore
         traces:
           span:
             - attributes["k8s.namespace.name"] == "kube-system"
+      # Drop probe / scrape / health-check traffic. Regex covers /health, /healthz,
+      # /livez, /readyz, /ping, /metrics, /status and namespaced variants like
+      # /api/health, /healthcheck, /health/liveness across all three OTel attribute
+      # name conventions (http.route, http.target, url.path).
       filter/drop_health_check:
         error_mode: ignore
         traces:
           span:
-            - attributes["http.route"] == "/health"
-            - attributes["http.route"] == "/healthz"
-            - attributes["http.route"] == "/live"
-            - attributes["http.route"] == "/ready"
-            - attributes["http.route"] == "/metrics"
-            - attributes["http.target"] == "/health"
-            - attributes["http.target"] == "/healthz"
-            - attributes["http.target"] == "/live"
-            - attributes["http.target"] == "/ready"
-            - attributes["http.target"] == "/metrics"
-            - attributes["url.path"] == "/health"
-            - attributes["url.path"] == "/healthz"
-            - attributes["url.path"] == "/live"
-            - attributes["url.path"] == "/ready"
-            - attributes["url.path"] == "/metrics"
+            - 'IsMatch(attributes["http.route"], "^/(healthz?|livez?|readyz?|ping|metrics|status)(/.*)?$") == true'
+            - 'IsMatch(attributes["http.target"], "^/(healthz?|livez?|readyz?|ping|metrics|status)(/.*)?$") == true'
+            - 'IsMatch(attributes["url.path"], "^/(healthz?|livez?|readyz?|ping|metrics|status)(/.*)?$") == true'
+      # Drop spans with malformed timestamps (zero, or end before start). These
+      # come from broken upstream instrumentation and waste storage.
+      filter/drop_invalid_timestamps:
+        error_mode: ignore
+        traces:
+          span:
+            - end_time_unix_nano == 0
+            - start_time_unix_nano == 0
+            - end_time_unix_nano < start_time_unix_nano
+      # Trace-level sampling. Unlike probabilistic head sampling, this preserves
+      # 100% of error and slow traces — exactly the ones an SRE wants to see —
+      # while keeping a 10% sample of healthy traffic for baseline visibility.
+      # Memory cost: roughly num_traces * average-trace-size bytes (~50-100MB
+      # at 50k traces buffered).
+      # NOTE: tail sampling requires all spans of a trace to land on the SAME
+      # collector instance. This default ships a single-replica deployment which
+      # is correct out of the box. Customers scaling to >1 replica must add a
+      # consistent-hash load balancer (otel `loadbalancing` exporter) in front,
+      # otherwise sampling decisions are made on partial traces.
+      tail_sampling:
+        decision_wait: 10s
+        num_traces: 50000
+        expected_new_traces_per_sec: 200
+        policies:
+          - name: errors
+            type: status_code
+            status_code:
+              status_codes: [ERROR]
+          - name: slow
+            type: latency
+            latency:
+              threshold_ms: 1000
+          - name: healthy-sample
+            type: probabilistic
+            probabilistic:
+              sampling_percentage: 10
       batch:
         timeout: 5s
         send_batch_size: 25000
     exporters:
       clickhouse:
-        endpoint: "tcp://nudgebee-agent-clickhouse:9000?dial_timeout=10s&compress=lz4" # Default endpoint, can be overridden
+        endpoint: "http://nudgebee-agent-clickhouse:8123?dial_timeout=10s" # Default endpoint, can be overridden
         database: default
         ttl_days: 7
         username: default
@@ -874,16 +916,19 @@ opentelemetry-collector:
           max_elapsed_time: 300s
     service:
       pipelines:
+        # memory_limiter must be the first processor in every pipeline so it can
+        # back-pressure receivers under memory load before any expensive work
+        # (filters, sampling, batching) runs.
         traces:
-          processors: [filter/drop_namespaces, filter/drop_health_check, probabilistic_sampler, batch]
+          processors: [memory_limiter, filter/drop_namespaces, filter/drop_health_check, filter/drop_invalid_timestamps, tail_sampling, batch]
           exporters: [clickhouse]
           receivers: [otlp]
         logs:
-          processors: [batch]
+          processors: [memory_limiter, batch]
           exporters: [clickhouse]
           receivers: [otlp]
         metrics:
-          processors: [batch]
+          processors: [memory_limiter, batch]
           exporters: [clickhouse]
           receivers: [otlp]
 clickhouse:
@@ -961,4 +1006,4 @@ clickhouse:
             description: "Average query duration is above 1 second over last 5 minutes."
 alertmanager:
   create_nb_default_rules: true
-  create_nb_alert_config: true
+  create_nb_alert_config: false
diff --git a/installation.sh b/installation.sh
index c35458ad..b3fa5d1f 100644
--- a/installation.sh
+++ b/installation.sh
@@ -8,13 +8,19 @@ openshift_enable=""
 additional_secret=""
 prometheus_url=""
 opencost_service_url=""
-namespace="nudgebee-agent" 
+namespace="nudgebee-agent"
 agent_name="nudgebee-agent"
 env="prod"
 disable_node_agent=""
 values=""
 alert_manager_url=""
 prometheus_org_id=""
+relay_address=""
+collector_endpoint=""
+image_registry=""
+disable_opencost=""
+disable_otel=""
+disable_prometheus_stack=""
 
 # Help function
 usage() {
@@ -33,12 +39,18 @@ usage() {
   echo "  -f <values> values yaml"
   echo "  -m <alert_manager_url> Alert manager URL"
   echo "  -r <prometheus-org-id> Prometheus org id"
+  echo "  -w <relay_address>    WebSocket relay address (self-hosted)"
+  echo "  -c <collector_endpoint> Collector endpoint URL (self-hosted)"
+  echo "  -i <image_registry>   Image registry (air-gapped/on-prem)"
+  echo "  -x <disable_opencost> Disable OpenCost (true/false)"
+  echo "  -t <disable_otel>     Disable OpenTelemetry Collector & ClickHouse (true/false)"
+  echo "  -g <disable_prometheus_stack> Disable Prometheus stack (true/false)"
   echo "Example:"
   echo "  $0 -a my_auth_key -k my_k8s_context -o true -p http://prometheus:9090 -s my_secret"
   exit 1
 }
 
-while getopts ":a:k:o:p:s:n:z:h:e:d:f:m:r:" opt; do
+while getopts ":a:k:o:p:s:n:z:h:e:d:f:m:r:w:c:i:x:t:g:" opt; do
   case $opt in
     a)
       auth_key="$OPTARG"
@@ -73,6 +85,24 @@ while getopts ":a:k:o:p:s:n:z:h:e:d:f:m:r:" opt; do
     r)
       prometheus_org_id="$OPTARG"
       ;;
+    w)
+      relay_address="$OPTARG"
+      ;;
+    c)
+      collector_endpoint="$OPTARG"
+      ;;
+    i)
+      image_registry="$OPTARG"
+      ;;
+    x)
+      disable_opencost="$OPTARG"
+      ;;
+    t)
+      disable_otel="$OPTARG"
+      ;;
+    g)
+      disable_prometheus_stack="$OPTARG"
+      ;;
     h)
       usage
       ;;
@@ -153,44 +183,47 @@ if ! command -v helm &> /dev/null; then
     exit 1
 fi
 
-# Check for existing Prometheus
-if [ -z "$prometheus_url" ]; then
-    prometheus_selectors=(
-            "app=kube-prometheus-stack-prometheus"
-            "app=prometheus,component=server,release!=kubecost"
-            "app=prometheus-server"
-            "app=prometheus-operator-prometheus"
-    )
-    prometheus_url=$(getPrometheusURL "${prometheus_selectors[@]}")
-fi
-
+# Check for existing Prometheus (skip if prometheus stack is disabled)
 existingPrometheus=false
 grafana_command=""
-# Check if service_url is empty
-if [ -z "$prometheus_url" ]; then
-   echo "Prometheus not found..!"
-   read -p "Installing Prometheus using helm, do you want to continue? (yes/no): " install_prometheus
-    if [ "$install_prometheus" == "yes" ]; then
-        # Add Helm installation command here or instructions
-        helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
-        helm repo update
-        helm upgrade --install nudgebee-prometheus prometheus-community/kube-prometheus-stack \
-            -n $namespace --create-namespace \
-            --set nodeExporter.enabled=true \
-            --set nodeExporter.service.targetPort=9101 \
-            --set pushgateway.enabled=false \
-            --set alertmanager.enabled=true \
-            --set kubeStateMetrics.enabled=true \
-            --set grafana.enabled=true \
-            -f https://raw.githubusercontent.com/nudgebee/k8s-agent/main/extra-scrape-config.yaml
-        prometheus_url="http://nudgebee-prometheus-kube-p-prometheus:9090"  # Prometheus uses default port 9090
-        grafana_command=" --set runner.grafana.enabled=true --set runner.grafana.url=http://nudgebee-prometheus-grafana.${namespace}.svc --set runner.grafana.username=admin --set runner.grafana.password=admin "
+if [ "$disable_prometheus_stack" == "true" ]; then
+    echo "Prometheus stack disabled, skipping Prometheus discovery and installation."
+else
+    if [ -z "$prometheus_url" ]; then
+        prometheus_selectors=(
+                "app=kube-prometheus-stack-prometheus"
+                "app=prometheus,component=server,release!=kubecost"
+                "app=prometheus-server"
+                "app=prometheus-operator-prometheus"
+        )
+        prometheus_url=$(getPrometheusURL "${prometheus_selectors[@]}")
+    fi
+
+    # Check if service_url is empty
+    if [ -z "$prometheus_url" ]; then
+       echo "Prometheus not found..!"
+       read -p "Installing Prometheus using helm, do you want to continue? (yes/no): " install_prometheus
+        if [ "$install_prometheus" == "yes" ]; then
+            helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+            helm repo update
+            helm upgrade --install nudgebee-prometheus prometheus-community/kube-prometheus-stack \
+                -n $namespace --create-namespace \
+                --set nodeExporter.enabled=true \
+                --set nodeExporter.service.targetPort=9101 \
+                --set pushgateway.enabled=false \
+                --set alertmanager.enabled=true \
+                --set kubeStateMetrics.enabled=true \
+                --set grafana.enabled=true \
+                -f https://raw.githubusercontent.com/nudgebee/k8s-agent/main/extra-scrape-config.yaml
+            prometheus_url="http://nudgebee-prometheus-kube-p-prometheus:9090"
+            grafana_command=" --set runner.grafana.enabled=true --set runner.grafana.url=http://nudgebee-prometheus-grafana.${namespace}.svc --set runner.grafana.username=admin --set runner.grafana.password=admin "
+        else
+            echo "Prometheus installation not requested. Exiting."
+            exit 0
+        fi
     else
-        echo "Prometheus installation not requested. Exiting."
-        exit 0
+        existingPrometheus=true
     fi
-else
-    existingPrometheus=true
 fi
 
 echo "Discovered Prometheus URL: $prometheus_url"
@@ -228,8 +261,38 @@ if [ -n "$prometheus_org_id" ]; then
   prometheus_org_id_command=" --set globalConfig.prometheus_headers='X-Scope-OrgID: $prometheus_org_id' --set globalConfig.alertmanager_headers='X-Scope-OrgID: $prometheus_org_id' --set opencost.opencost.extraEnv.PROMETHEUS_HEADER_X_SCOPE_ORGID=$prometheus_org_id"
 fi
 
+relay_address_command=""
+if [ -n "$relay_address" ]; then
+  relay_address_command=" --set runner.relay_address=$relay_address"
+fi
+
+collector_endpoint_command=""
+if [ -n "$collector_endpoint" ]; then
+  collector_endpoint_command=" --set runner.nudgebee.endpoint=$collector_endpoint"
+fi
+
+image_registry_command=""
+if [ -n "$image_registry" ]; then
+  image_registry_command=" --set runner.image_registry=$image_registry"
+fi
+
+disable_opencost_command=""
+if [ "$disable_opencost" == "true" ]; then
+  disable_opencost_command=" --set opencost.enabled=false"
+fi
+
+disable_otel_command=""
+if [ "$disable_otel" == "true" ]; then
+  disable_otel_command=" --set opentelemetry-collector.enabled=false --set clickhouse.enabled=false"
+fi
+
+disable_prometheus_stack_command=""
+if [ "$disable_prometheus_stack" == "true" ]; then
+  disable_prometheus_stack_command=" --set enablePrometheusStack=false"
+fi
+
 # Use helm upgrade --install to either install or upgrade the Helm chart
-a="helm upgrade --install $agent_name nudgebee-agent/nudgebee-agent  --namespace $namespace --create-namespace --set runner.nudgebee.auth_secret_key=\"$auth_key\" --set globalConfig.prometheus_url=\"$prometheus_url\" --set opencost.opencost.prometheus.external.url=\"$prometheus_url\" $disable_node_agent_command $openshift_enable_command $addition_secret_command $values_command $grafana_command $alert_manager_url_command $prometheus_org_id_command"
+a="helm upgrade --install $agent_name nudgebee-agent/nudgebee-agent  --namespace $namespace --create-namespace --set runner.nudgebee.auth_secret_key=\"$auth_key\" --set globalConfig.prometheus_url=\"$prometheus_url\" --set opencost.opencost.prometheus.external.url=\"$prometheus_url\" $disable_node_agent_command $openshift_enable_command $addition_secret_command $values_command $grafana_command $alert_manager_url_command $prometheus_org_id_command $relay_address_command $collector_endpoint_command $image_registry_command $disable_opencost_command $disable_otel_command $disable_prometheus_stack_command"
 
 echo "Running command: $a"
 eval $a
@@ -250,7 +313,12 @@ if [ -z "$loki_url" ]; then
         # Add Helm installation command here or instructions
         helm repo add grafana https://grafana.github.io/helm-charts
         helm repo update
-        helm upgrade --install nudgebee-loki grafana/loki-stack -n $namespace --create-namespace --set loki.persistence.enabled=true --set loki.persistence.size=10Gi --set promtail.enabled=true
+        helm upgrade --install nudgebee-loki grafana/loki-stack \
+            -n "$namespace" --create-namespace \
+            --set loki.persistence.enabled=true \
+            --set loki.persistence.size=10Gi \
+            --set promtail.enabled=true \
+            --set loki.isDefault=false
         loki_url="http://nudgebee-loki:3100"
     else
         echo "Loki installation not requested. Node Agent will still be installed."