diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index b4b1d26d3..ae6b52a18 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -191,12 +191,16 @@ These environment variables are read directly (not through config file): { "exporters": { "httpExporterConfig": { - "url": "https://api.example.com/v1/runtimealerts", - "headers": { - "Authorization": "Bearer " - }, + "url": "https://api.example.com", + "headers": [ + {"key": "Authorization", "value": "Bearer "} + ], "timeoutSeconds": 5, - "method": "POST" + "method": "POST", + "maxAlertsPerMinute": 100, + "eventFieldFilter": { + "denyList": ["spec.processTree", "spec.cloudMetadata"] + } } } } @@ -204,10 +208,54 @@ These environment variables are read directly (not through config file): | Key | Type | Default | Description | |-----|------|---------|-------------| -| `exporters::httpExporterConfig::url` | string | - | HTTP endpoint URL | -| `exporters::httpExporterConfig::headers` | map | `{}` | HTTP headers | -| `exporters::httpExporterConfig::timeoutSeconds` | int | `5` | Request timeout | -| `exporters::httpExporterConfig::method` | string | `POST` | HTTP method | +| `exporters::httpExporterConfig::url` | string | - | HTTP endpoint URL (required) | +| `exporters::httpExporterConfig::path` | string | `/v1/runtimealerts` | Path appended to URL | +| `exporters::httpExporterConfig::headers` | []`{key, value}` | `[]` | HTTP request headers | +| `exporters::httpExporterConfig::queryParams` | []`{key, value}` | `[]` | URL query parameters; set value to `""` to read from env var | +| `exporters::httpExporterConfig::timeoutSeconds` | int | `5` | Request timeout in seconds | +| `exporters::httpExporterConfig::method` | string | `POST` | HTTP method (`POST` or `PUT`) | +| `exporters::httpExporterConfig::maxAlertsPerMinute` | int | `100` | Rate limit; sends a limit-reached alert when exceeded | +| `exporters::httpExporterConfig::eventFieldFilter` | object | - | Field-level filter applied to every outgoing payload (see below) | + +##### Event Field Filter + +Controls which fields are included in the JSON payload sent to the HTTP endpoint. Useful for reducing payload size or stripping sensitive data before it leaves the cluster. + +> **Scope:** the filter is applied at the HTTP transport layer and affects all payloads sent through this exporter, including runtime alerts, malware alerts, and FIM events. If you need different filtering per payload type, use separate exporter instances with different configs. + +```json +{ + "exporters": { + "httpExporterConfig": { + "url": "https://api.example.com", + "eventFieldFilter": { + "denyList": ["spec.processTree", "spec.cloudMetadata"] + } + } + } +} +``` + +| Key | Type | Description | +|-----|------|-------------| +| `eventFieldFilter::allowList` | []string | Keep only these fields; all others are dropped. Takes precedence over `denyList`. | +| `eventFieldFilter::denyList` | []string | Remove these fields; all others are kept. Ignored if `allowList` is set. | + +Both lists support dot notation for nested fields (e.g. `spec.processTree`) and fields inside slices (e.g. `spec.alerts.severity`). If neither list is set, the filter is disabled and the full payload is sent. + +**DenyList example** — strip process tree and cloud metadata to reduce payload size: +```json +"eventFieldFilter": { + "denyList": ["spec.processTree", "spec.cloudMetadata"] +} +``` + +**AllowList example** — send only the fields your backend needs: +```json +"eventFieldFilter": { + "allowList": ["kind", "apiVersion", "metadata", "spec.alerts"] +} +``` #### AlertManager Exporter @@ -395,7 +443,10 @@ These flags disable specific tracers (useful for debugging): "networkServiceEnabled": true, "exporters": { "httpExporterConfig": { - "url": "https://kubescape-backend.example.com/v1/runtimealerts" + "url": "https://kubescape-backend.example.com", + "eventFieldFilter": { + "denyList": ["spec.processTree", "spec.cloudMetadata"] + } } } } @@ -425,10 +476,13 @@ These flags disable specific tracers (useful for debugging): }, "exporters": { "httpExporterConfig": { - "url": "https://api.example.com/v1/runtimealerts", + "url": "https://api.example.com", "enableAlertBulking": true, "bulkMaxAlerts": 100, - "bulkTimeoutSeconds": 5 + "bulkTimeoutSeconds": 5, + "eventFieldFilter": { + "denyList": ["spec.processTree", "spec.cloudMetadata"] + } }, "alertManagerExporterUrls": [ "alertmanager.monitoring.svc.cluster.local:9093" @@ -475,7 +529,7 @@ These flags disable specific tracers (useful for debugging): }, "exporters": { "httpExporterConfig": { - "url": "https://api.example.com/v1/runtimealerts", + "url": "https://api.example.com", "enableAlertBulking": true, "bulkMaxAlerts": 200, "bulkTimeoutSeconds": 3, diff --git a/pkg/exporters/field_filter.go b/pkg/exporters/field_filter.go new file mode 100644 index 000000000..857eac22f --- /dev/null +++ b/pkg/exporters/field_filter.go @@ -0,0 +1,151 @@ +package exporters + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +// EventFieldFilterConfig configures field-level filtering for exported alert events. +// AllowList takes precedence over DenyList. Both support dot notation (e.g. "spec.processTree"). +type EventFieldFilterConfig struct { + AllowList []string `json:"allowList,omitempty" mapstructure:"allowList"` + DenyList []string `json:"denyList,omitempty" mapstructure:"denyList"` +} + +// EventFieldFilter applies allow/deny list filtering to JSON payloads. +type EventFieldFilter struct { + allowSet map[string]struct{} + denySet map[string]struct{} + useAllow bool +} + +// NewEventFieldFilter creates a filter from config. Returns nil if no fields are configured. +func NewEventFieldFilter(config *EventFieldFilterConfig) *EventFieldFilter { + if config == nil { + return nil + } + if len(config.AllowList) == 0 && len(config.DenyList) == 0 { + return nil + } + + f := &EventFieldFilter{} + if len(config.AllowList) > 0 { + f.useAllow = true + f.allowSet = make(map[string]struct{}, len(config.AllowList)) + for _, field := range config.AllowList { + f.allowSet[field] = struct{}{} + } + } else { + f.denySet = make(map[string]struct{}, len(config.DenyList)) + for _, field := range config.DenyList { + f.denySet[field] = struct{}{} + } + } + return f +} + +// FilterJSON applies the field filter to marshaled JSON bytes and returns filtered bytes. +func (f *EventFieldFilter) FilterJSON(data []byte) ([]byte, error) { + var m map[string]any + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + if err := dec.Decode(&m); err != nil { + return nil, fmt.Errorf("field filter: failed to unmarshal: %w", err) + } + + if f.useAllow { + m = applyAllowList(m, f.allowSet) + } else { + for key := range f.denySet { + parts := strings.SplitN(key, ".", 2) + if len(parts) == 1 { + delete(m, key) + } else { + removePath(m, parts[0], parts[1]) + } + } + } + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(m); err != nil { + return nil, fmt.Errorf("field filter: failed to marshal: %w", err) + } + return bytes.TrimRight(buf.Bytes(), "\n"), nil +} + +func applyAllowList(m map[string]any, allowSet map[string]struct{}) map[string]any { + groups := make(map[string]map[string]struct{}) + for path := range allowSet { + parts := strings.SplitN(path, ".", 2) + topKey := parts[0] + if _, ok := groups[topKey]; !ok { + groups[topKey] = make(map[string]struct{}) + } + if len(parts) > 1 { + groups[topKey][parts[1]] = struct{}{} + } + } + + result := make(map[string]any) + for topKey, subSet := range groups { + val, exists := m[topKey] + if !exists { + continue + } + if _, ok := allowSet[topKey]; ok { + result[topKey] = val + continue + } + if len(subSet) > 0 { + if nested, ok := val.(map[string]any); ok { + result[topKey] = applyAllowList(nested, subSet) + } else if slice, ok := val.([]any); ok { + newSlice := make([]any, 0, len(slice)) + for _, item := range slice { + if itemMap, ok := item.(map[string]any); ok { + filtered := applyAllowList(itemMap, subSet) + if len(filtered) > 0 { + newSlice = append(newSlice, filtered) + } + } else { + newSlice = append(newSlice, item) + } + } + result[topKey] = newSlice + } else { + // scalar value (string, number, bool) — keep it as-is + result[topKey] = val + } + } + } + return result +} + +func removePath(m map[string]any, topKey, rest string) { + val, exists := m[topKey] + if !exists { + return + } + parts := strings.SplitN(rest, ".", 2) + if nested, ok := val.(map[string]any); ok { + if len(parts) == 1 { + delete(nested, rest) + } else { + removePath(nested, parts[0], parts[1]) + } + } else if slice, ok := val.([]any); ok { + for _, item := range slice { + if itemMap, ok := item.(map[string]any); ok { + if len(parts) == 1 { + delete(itemMap, rest) + } else { + removePath(itemMap, parts[0], parts[1]) + } + } + } + } +} diff --git a/pkg/exporters/field_filter_test.go b/pkg/exporters/field_filter_test.go new file mode 100644 index 000000000..d9b3d816c --- /dev/null +++ b/pkg/exporters/field_filter_test.go @@ -0,0 +1,378 @@ +package exporters + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewEventFieldFilter_NilConfig(t *testing.T) { + f := NewEventFieldFilter(nil) + assert.Nil(t, f) +} + +func TestNewEventFieldFilter_EmptyConfig(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{}) + assert.Nil(t, f) +} + +func TestNewEventFieldFilter_WithAllowList(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{ + AllowList: []string{"message", "ruleID"}, + }) + assert.NotNil(t, f) + assert.True(t, f.useAllow) +} + +func TestNewEventFieldFilter_WithDenyList(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{ + DenyList: []string{"processTree"}, + }) + assert.NotNil(t, f) + assert.False(t, f.useAllow) +} + +func TestFilterJSON_AllowListNestedField(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{ + AllowList: []string{"kind", "spec.alerts"}, + }) + + input := map[string]any{ + "kind": "RuntimeAlerts", + "spec": map[string]any{ + "alerts": []any{"alert1"}, + "processTree": map[string]any{"pid": 1234}, + "cloudMetadata": map[string]any{"region": "us-east-1"}, + }, + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + err = json.Unmarshal(result, &output) + require.NoError(t, err) + + assert.Equal(t, "RuntimeAlerts", output["kind"]) + spec := output["spec"].(map[string]any) + assert.NotNil(t, spec["alerts"]) + assert.Nil(t, spec["processTree"]) + assert.Nil(t, spec["cloudMetadata"]) + assert.Nil(t, output["apiVersion"]) +} + +func TestFilterJSON_AllowList(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{ + AllowList: []string{"message", "ruleID"}, + }) + + input := map[string]any{ + "message": "Unexpected process launched", + "ruleID": "R0001", + "processTree": map[string]any{"pid": 1234, "comm": "bash"}, + "k8sDetails": map[string]any{"namespace": "default"}, + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + err = json.Unmarshal(result, &output) + require.NoError(t, err) + + assert.Equal(t, "Unexpected process launched", output["message"]) + assert.Equal(t, "R0001", output["ruleID"]) + assert.Nil(t, output["processTree"]) + assert.Nil(t, output["k8sDetails"]) +} + +func TestFilterJSON_DenyList(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{ + DenyList: []string{"processTree", "k8sDetails"}, + }) + + input := map[string]any{ + "message": "Unexpected process launched", + "ruleID": "R0001", + "processTree": map[string]any{"pid": 1234}, + "k8sDetails": map[string]any{"namespace": "default"}, + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + err = json.Unmarshal(result, &output) + require.NoError(t, err) + + assert.Equal(t, "Unexpected process launched", output["message"]) + assert.Equal(t, "R0001", output["ruleID"]) + assert.Nil(t, output["processTree"]) + assert.Nil(t, output["k8sDetails"]) +} + +func TestFilterJSON_DenyListNestedField(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{ + DenyList: []string{"spec.processTree"}, + }) + + input := map[string]any{ + "kind": "RuntimeAlerts", + "spec": map[string]any{ + "alerts": []interface{}{"alert1"}, + "processTree": map[string]any{"pid": 1234}, + }, + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + err = json.Unmarshal(result, &output) + require.NoError(t, err) + + assert.Equal(t, "RuntimeAlerts", output["kind"]) + spec := output["spec"].(map[string]any) + assert.NotNil(t, spec["alerts"]) + assert.Nil(t, spec["processTree"]) +} + +func TestFilterJSON_AllowListTakesPrecedence(t *testing.T) { + // "ruleID" is in BOTH lists. If allowList takes precedence, it must be kept + // (allowList wins, denyList is ignored entirely). If denyList were also applied, + // "ruleID" would be removed, proving precedence didn't hold. + f := NewEventFieldFilter(&EventFieldFilterConfig{ + AllowList: []string{"message", "ruleID"}, + DenyList: []string{"ruleID"}, + }) + + input := map[string]any{ + "message": "test", + "ruleID": "R0001", + "extra": "data", + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + err = json.Unmarshal(result, &output) + require.NoError(t, err) + + assert.Equal(t, "test", output["message"]) + assert.Equal(t, "R0001", output["ruleID"], "ruleID kept because allowList takes precedence over denyList") + assert.Nil(t, output["extra"], "extra dropped (not in allowList)") +} + +func TestFilterJSON_DenyListSliceField(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{ + DenyList: []string{"spec.alerts.ruleID"}, + }) + + input := map[string]any{ + "kind": "RuntimeAlerts", + "spec": map[string]any{ + "alerts": []any{ + map[string]any{"ruleID": "R0001", "message": "alert 1"}, + map[string]any{"ruleID": "R0002", "message": "alert 2"}, + }, + "processTree": map[string]any{"pid": 1234}, + }, + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + err = json.Unmarshal(result, &output) + require.NoError(t, err) + + spec := output["spec"].(map[string]any) + alerts := spec["alerts"].([]any) + require.Len(t, alerts, 2) + + // ruleID removed from each alert, message kept + for _, a := range alerts { + alert := a.(map[string]any) + assert.Nil(t, alert["ruleID"]) + assert.NotNil(t, alert["message"]) + } + // processTree unaffected + assert.NotNil(t, spec["processTree"]) +} + +func TestFilterJSON_AllowListSliceField(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{ + AllowList: []string{"kind", "spec.alerts.message"}, + }) + + input := map[string]any{ + "kind": "RuntimeAlerts", + "spec": map[string]any{ + "alerts": []any{ + map[string]any{"ruleID": "R0001", "message": "alert 1", "severity": 5}, + map[string]any{"ruleID": "R0002", "message": "alert 2", "severity": 3}, + }, + "processTree": map[string]any{"pid": 1234}, + "cloudMetadata": map[string]any{"region": "us-east-1"}, + }, + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + err = json.Unmarshal(result, &output) + require.NoError(t, err) + + assert.Equal(t, "RuntimeAlerts", output["kind"]) + spec := output["spec"].(map[string]any) + alerts := spec["alerts"].([]any) + require.Len(t, alerts, 2) + + // only message kept in each alert + for _, a := range alerts { + alert := a.(map[string]any) + assert.NotNil(t, alert["message"]) + assert.Nil(t, alert["ruleID"]) + assert.Nil(t, alert["severity"]) + } + // processTree and cloudMetadata dropped + assert.Nil(t, spec["processTree"]) + assert.Nil(t, spec["cloudMetadata"]) +} + +func TestFilterJSON_InvalidJSON_ReturnsError(t *testing.T) { + f := NewEventFieldFilter(&EventFieldFilterConfig{ + DenyList: []string{"spec.processTree"}, + }) + + _, err := f.FilterJSON([]byte(`{not valid json`)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "field filter: failed to unmarshal") +} + +func TestFilterJSON_AllowListSliceField_OmitsEmptyItems(t *testing.T) { + // When an allow list targets a sub-field inside a slice, items that + // have none of the allowed fields should be omitted from the output. + f := NewEventFieldFilter(&EventFieldFilterConfig{ + AllowList: []string{"kind", "spec.alerts.message"}, + }) + + input := map[string]any{ + "kind": "RuntimeAlerts", + "spec": map[string]any{ + "alerts": []any{ + map[string]any{"ruleID": "R0001", "message": "alert 1"}, + map[string]any{"ruleID": "R0002", "severity": 5}, // no "message" → empty after filter + map[string]any{"message": "alert 3", "severity": 3}, + }, + }, + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + err = json.Unmarshal(result, &output) + require.NoError(t, err) + + spec := output["spec"].(map[string]any) + alerts := spec["alerts"].([]any) + + // Second item had no "message", so it's omitted + require.Len(t, alerts, 2) + assert.Equal(t, "alert 1", alerts[0].(map[string]any)["message"]) + assert.Equal(t, "alert 3", alerts[1].(map[string]any)["message"]) +} + +func TestFilterJSON_NoHTMLEscaping(t *testing.T) { + // json.Marshal escapes <, >, & but our filter should not + f := NewEventFieldFilter(&EventFieldFilterConfig{ + DenyList: []string{"extra"}, + }) + + input := map[string]any{ + "message": "process > limit & value < threshold", + "extra": "removed", + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + // The output should contain literal <, >, & not escaped unicode + assert.Contains(t, string(result), `process > limit & value < threshold`) + assert.NotContains(t, string(result), `\u003c`) + assert.NotContains(t, string(result), `\u003e`) + assert.NotContains(t, string(result), `\u0026`) +} + +func TestFilterJSON_AllowList_ScalarLeaf_MapToScalar(t *testing.T) { + // Allow path ends at a scalar inside a nested map — must be kept, not silently dropped. + f := NewEventFieldFilter(&EventFieldFilterConfig{ + AllowList: []string{"spec.processTree.pid"}, + }) + + input := map[string]any{ + "spec": map[string]any{ + "processTree": map[string]any{ + "pid": 1234, + "comm": "sh", + }, + }, + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + require.NoError(t, json.Unmarshal(result, &output)) + + pt := output["spec"].(map[string]any)["processTree"].(map[string]any) + assert.NotNil(t, pt["pid"], "scalar pid kept") + assert.Nil(t, pt["comm"], "comm dropped") +} + +func TestFilterJSON_AllowList_ScalarLeaf_InsideSlice(t *testing.T) { + // Allow path ends at a scalar inside each slice element — must be kept. + f := NewEventFieldFilter(&EventFieldFilterConfig{ + AllowList: []string{"spec.alerts.severity"}, + }) + + input := map[string]any{ + "spec": map[string]any{ + "alerts": []any{ + map[string]any{"ruleID": "R0001", "message": "msg1", "severity": 5}, + map[string]any{"ruleID": "R0002", "message": "msg2", "severity": 7}, + }, + }, + } + data, _ := json.Marshal(input) + + result, err := f.FilterJSON(data) + require.NoError(t, err) + + var output map[string]any + require.NoError(t, json.Unmarshal(result, &output)) + + alerts := output["spec"].(map[string]any)["alerts"].([]any) + require.Len(t, alerts, 2) + for i, a := range alerts { + alert := a.(map[string]any) + assert.NotNil(t, alert["severity"], "alert[%d]: severity kept", i) + assert.Nil(t, alert["ruleID"], "alert[%d]: ruleID dropped", i) + assert.Nil(t, alert["message"], "alert[%d]: message dropped", i) + } +} diff --git a/pkg/exporters/http_exporter.go b/pkg/exporters/http_exporter.go index 416bfd912..f94d7af7a 100644 --- a/pkg/exporters/http_exporter.go +++ b/pkg/exporters/http_exporter.go @@ -60,6 +60,8 @@ type HTTPExporterConfig struct { BulkMaxRetries int `json:"bulkMaxRetries"` // Default: 3 BulkRetryBaseDelayMs int `json:"bulkRetryBaseDelayMs"` // Default: 1000ms BulkRetryMaxDelayMs int `json:"bulkRetryMaxDelayMs"` // Default: 30000ms + // Event field filter configuration + EventFieldFilter *EventFieldFilterConfig `json:"eventFieldFilter,omitempty"` } type HTTPExporter struct { @@ -73,6 +75,7 @@ type HTTPExporter struct { cloudMetadata *armotypes.CloudMetadata bulkManager *AlertBulkManager alertSourcePlatform armotypes.AlertSourcePlatform + fieldFilter *EventFieldFilter } type alertMetrics struct { @@ -112,6 +115,7 @@ func NewHTTPExporter(config HTTPExporterConfig, clusterName, nodeName string, cl alertMetrics: &alertMetrics{}, cloudMetadata: cloudMetadata, alertSourcePlatform: alertSourcePlatform, + fieldFilter: NewEventFieldFilter(config.EventFieldFilter), } // Initialize bulk manager if bulking is enabled @@ -402,6 +406,13 @@ func (e *HTTPExporter) sendHTTPRequest(ctx context.Context, payload interface{}) return fmt.Errorf("failed to marshal payload: %w", err) } + if e.fieldFilter != nil { + body, err = e.fieldFilter.FilterJSON(body) + if err != nil { + return fmt.Errorf("failed to apply field filter: %w", err) + } + } + var url string if e.config.Path != nil { url = fmt.Sprintf("%s%s", e.config.URL, *e.config.Path)