diff --git a/charts/banzaicloud-logging-operator-monitoring/Chart.yaml b/charts/banzaicloud-logging-operator-monitoring/Chart.yaml index 46cfca8..59c529a 100644 --- a/charts/banzaicloud-logging-operator-monitoring/Chart.yaml +++ b/charts/banzaicloud-logging-operator-monitoring/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: banzaicloud-logging-operator-monitoring description: Monitor logging-operator type: application -version: 2.3.2 -appVersion: 2.3.2 +version: 2.3.3 +appVersion: 2.3.3 icon: https://raw.githubusercontent.com/SomeBlackMagic/helm-charts/master/charts/banzaicloud-logging-operator-monitoring/icon.png keywords: - banzaicloud diff --git a/charts/banzaicloud-logging-operator-monitoring/templates/prometheus/prometheusrule-fluentd.yaml b/charts/banzaicloud-logging-operator-monitoring/templates/prometheus/prometheusrule-fluentd.yaml index 0c826e1..070fe81 100644 --- a/charts/banzaicloud-logging-operator-monitoring/templates/prometheus/prometheusrule-fluentd.yaml +++ b/charts/banzaicloud-logging-operator-monitoring/templates/prometheus/prometheusrule-fluentd.yaml @@ -42,7 +42,7 @@ spec: {{- if .Values.prometheus.rule.alerts.fluentd.outputRetryWarning.enabled }} - alert: FluentdOutputRetryWarning - expr: rate(fluentd_output_status_retry_count[{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.rateTime }}]) > {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.threshold }} + expr: rate(fluentd_output_status_retry_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.rateTime }}]) > {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.threshold }} for: {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.for }} labels: severity: "{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryWarning.severity }}" @@ -66,7 +66,13 @@ spec: {{- if $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.enabled }} - alert: FluentdHighErrorRatio expr: | - sum by (type, plugin_id)(rate(fluentd_output_status_num_errors[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}])) / sum by (type, plugin_id)(rate(fluentd_output_status_emit_count[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}])) > {{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.threshold }} + ( + sum by (type, plugin_id)(rate(fluentd_output_status_num_errors{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}])) + / + sum by (type, plugin_id)(rate(fluentd_output_status_emit_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}])) + ) > {{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.threshold }} + and + sum by (type, plugin_id)(rate(fluentd_output_status_emit_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.rateTime }}])) > 0 for: {{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.for }} labels: severity: "{{ $.Values.prometheus.rule.alerts.fluentd.highErrorRatio.severity }}" @@ -78,7 +84,13 @@ spec: {{- if $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.enabled }} - alert: FluentdHighRetryRatio expr: | - sum by (type, plugin_id)(rate(fluentd_output_status_retry_count[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}])) / sum by (type, plugin_id)(rate(fluentd_output_status_emit_count[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}])) > {{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.threshold }} + ( + sum by (type, plugin_id)(rate(fluentd_output_status_retry_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}])) + / + sum by (type, plugin_id)(rate(fluentd_output_status_emit_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}])) + ) > {{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.threshold }} + and + sum by (type, plugin_id)(rate(fluentd_output_status_emit_count{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.rateTime }}])) > 0 for: {{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.for }} labels: severity: "{{ $.Values.prometheus.rule.alerts.fluentd.highRetryRatio.severity }}" @@ -90,7 +102,7 @@ spec: {{- if $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.enabled }} - alert: FluentdOutputRetryErrors - expr: rate(fluentd_output_status_num_errors[{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.rateTime }}]) > {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.threshold }} + expr: rate(fluentd_output_status_num_errors{plugin_id!~"{{ $.Values.prometheus.rule.alerts.fluentd.excludePluginIdRegex }}"}[{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.rateTime }}]) > {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.threshold }} for: {{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.for }} labels: severity: "{{ $.Values.prometheus.rule.alerts.fluentd.outputRetryErrors.severity }}" diff --git a/charts/banzaicloud-logging-operator-monitoring/values.yaml b/charts/banzaicloud-logging-operator-monitoring/values.yaml index 6ff5883..d82d846 100644 --- a/charts/banzaicloud-logging-operator-monitoring/values.yaml +++ b/charts/banzaicloud-logging-operator-monitoring/values.yaml @@ -28,6 +28,9 @@ prometheus: fluentd: + #Regex to exclude internal Fluentd plugins from alerts + excludePluginIdRegex: "main.*" + serviceDown: enabled: true severity: critical