From 85b0688d1e5d379f97857ef3a8bf64bbe5c699dd Mon Sep 17 00:00:00 2001
From: Danylo Pilovets <danylo.p@sterl.in>
Date: Tue, 17 Mar 2026 09:56:45 +0200
Subject: [PATCH 1/2] fix: eliminate false positive fluentd alerts (inf
 division, internal plugins)

- Add emit_count > 0 guard to FluentdHighErrorRatio and FluentdHighRetryRatio to prevent inf from division by zero
- Add plugin_id!~"main.*" filter to FluentdOutputRetryWarning, FluentdOutputRetryErrors, FluentdHighErrorRatio, FluentdHighRetryRatio
- Add configurable excludePluginIdRegex parameter to values.yaml

Made-with: Cursor
---
 .../prometheus/prometheusrule-fluentd.yaml    | 20 +++++++++++++++----
 .../values.yaml                               |  3 +++
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/charts/banzaicloud-logging-operator-monitoring/templates/prometheus/prometheusrule-fluentd.yaml b/charts/banzaicloud-logging-operator-monitoring/templates/prometheus/prometheusrule-fluentd.yaml
index 0c826e1..070fe81 100644
--- a/charts/banzaicloud-logging-operator-monitoring/templates/prometheus/prometheusrule-fluentd.yaml
+++ b/charts/banzaicloud-logging-operator-monitoring/templates/prometheus/prometheusrule-fluentd.yaml
@@ -42,7 +42,7 @@ spec:
 
     {{- if .Values.prometheus.rule.alerts.fluentd.outputRetryWarning.enabled }}
     - alert: FluentdOutputRetryWarning
-      expr: rate(fluentd_output_status_retry_count[{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.rateTime }}]) > {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.threshold }}
+      expr: rate(fluentd_output_status_retry_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.rateTime }}]) > {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.threshold }}
       for: {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.for }}
       labels:
         severity: "{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.severity }}"
@@ -66,7 +66,13 @@ spec:
     {{- if $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.enabled }}
     - alert: FluentdHighErrorRatio
       expr: |
-        sum by (type, plugin_id)(rate(fluentd_output_status_num_errors[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}])) / sum by (type, plugin_id)(rate(fluentd_output_status_emit_count[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}])) > {{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.threshold }}
+        (
+          sum by (type, plugin_id)(rate(fluentd_output_status_num_errors{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}]))
+          /
+          sum by (type, plugin_id)(rate(fluentd_output_status_emit_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}]))
+        ) > {{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.threshold }}
+        and
+        sum by (type, plugin_id)(rate(fluentd_output_status_emit_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}])) > 0
       for: {{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.for }}
       labels:
         severity: "{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.severity }}"
@@ -78,7 +84,13 @@ spec:
     {{- if $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.enabled }}
     - alert: FluentdHighRetryRatio
       expr: |
-        sum by (type, plugin_id)(rate(fluentd_output_status_retry_count[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}])) / sum by (type, plugin_id)(rate(fluentd_output_status_emit_count[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}])) > {{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.threshold }}
+        (
+          sum by (type, plugin_id)(rate(fluentd_output_status_retry_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}]))
+          /
+          sum by (type, plugin_id)(rate(fluentd_output_status_emit_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}]))
+        ) > {{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.threshold }}
+        and
+        sum by (type, plugin_id)(rate(fluentd_output_status_emit_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}])) > 0
       for: {{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.for }}
       labels:
         severity: "{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.severity }}"
@@ -90,7 +102,7 @@ spec:
 
     {{- if $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.enabled }}
     - alert: FluentdOutputRetryErrors
-      expr: rate(fluentd_output_status_num_errors[{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.rateTime }}]) > {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.threshold }}
+      expr: rate(fluentd_output_status_num_errors{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.rateTime }}]) > {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.threshold }}
       for: {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.for }}
       labels:
         severity: "{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.severity }}"
diff --git a/charts/banzaicloud-logging-operator-monitoring/values.yaml b/charts/banzaicloud-logging-operator-monitoring/values.yaml
index 6ff5883..d82d846 100644
--- a/charts/banzaicloud-logging-operator-monitoring/values.yaml
+++ b/charts/banzaicloud-logging-operator-monitoring/values.yaml
@@ -28,6 +28,9 @@ prometheus:
 
 
       fluentd:
+        #Regex to exclude internal Fluentd plugins from alerts
+        excludePluginIdRegex: "main.*"
+
         serviceDown:
           enabled: true
           severity: critical

From 707a51370a39998a877295af3b662f6f57fe960f Mon Sep 17 00:00:00 2001
From: Danylo Pilovets <danylo.p@sterl.in>
Date: Tue, 17 Mar 2026 10:00:25 +0200
Subject: [PATCH 2/2] bump chart version

---
 charts/banzaicloud-logging-operator-monitoring/Chart.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/banzaicloud-logging-operator-monitoring/Chart.yaml b/charts/banzaicloud-logging-operator-monitoring/Chart.yaml
index 46cfca8..59c529a 100644
--- a/charts/banzaicloud-logging-operator-monitoring/Chart.yaml
+++ b/charts/banzaicloud-logging-operator-monitoring/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: banzaicloud-logging-operator-monitoring
 description: Monitor logging-operator
 type: application
-version: 2.3.2
-appVersion: 2.3.2
+version: 2.3.3
+appVersion: 2.3.3
 icon: https://raw.githubusercontent.com/SomeBlackMagic/helm-charts/master/charts/banzaicloud-logging-operator-monitoring/icon.png
 keywords:
   - banzaicloud