Fix aide-worker memory growth caused by cgroup page cache accumulation

AIDE scans the entire host filesystem, and the resulting kernel page
cache is charged to the container's cgroup. Without reclamation, reported
memory grows toward the resource limit after each scan cycle.

Use cgroup v2 memory.reclaim to evict file-backed page cache after each
AIDE scan and database initialization. This reduced aide-worker memory
from ~570 MiB to ~11 MiB in testing on OCP 4.18.22.

Use raw syscalls (syscall.Open/Write/Close) for memory.reclaim instead
of os.OpenFile, because Go's runtime registers fds with its epoll poller
and the cgroup v2 file's poll support causes the goroutine to hang
waiting for write-readiness that never arrives.

Additional fixes:
- Close leaked file descriptor in getNonEmptyFile when file is empty
- Pre-compile regex patterns used in log parsing
- Handle AlreadyExists on ConfigMap creation to avoid unnecessary retries
- Call runtime.GC and debug.FreeOSMemory after scan to return heap to OS
- Update outdated GODEBUG comment (madvdontneed=1 is default since Go 1.16)

Author: Vincent056

cmd/manager/daemon_util.go

@@ -16,6 +16,7 @@ limitations under the License.
 package manager
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io"
@@ -25,8 +26,11 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"runtime"
+	"runtime/debug"
 	"sort"
 	"strings"
+	"syscall"
 	"time"
 
 	backoff "github.com/cenkalti/backoff/v4"
@@ -38,6 +42,70 @@ import (
 	"github.com/openshift/file-integrity-operator/pkg/common"
 )
 
+// reclaimCgroupPageCache asks the kernel to reclaim file-backed (page cache)
+// memory charged to this container's cgroup. AIDE scans the entire host
+// filesystem, and the resulting page cache pages are charged to the container's
+// cgroup, causing reported memory to grow toward the limit over scan cycles.
+//
+// We use raw syscalls instead of os.OpenFile because Go's runtime registers
+// opened fds with its epoll-based poller. The cgroup v2 memory.reclaim file
+// supports poll (via cgroup_file_poll), so Go treats it as a pollable fd and
+// waits for write-readiness before issuing the write. That readiness event
+// never arrives, hanging the goroutine permanently.
+func reclaimCgroupPageCache() {
+	cgroupPath, err := getOwnCgroupPath()
+	if err != nil {
+		LOG("could not determine own cgroup path (page cache not reclaimed): %v", err)
+		return
+	}
+
+	reclaimFile := path.Join(cgroupPath, "memory.reclaim")
+	fd, err := syscall.Open(reclaimFile, syscall.O_WRONLY, 0)
+	if err != nil {
+		LOG("memory.reclaim not available at %s (page cache not reclaimed): %v", reclaimFile, err)
+		return
+	}
+
+	_, err = syscall.Write(fd, []byte("500M"))
+	closeErr := syscall.Close(fd)
+	if err != nil && err != syscall.EAGAIN {
+		LOG("memory.reclaim write returned (non-fatal): %v", err)
+	}
+	if closeErr != nil {
+		LOG("memory.reclaim close error: %v", closeErr)
+	}
+	LOG("reclaimed cgroup page cache after AIDE scan")
+}
+
+// getOwnCgroupPath reads /proc/self/cgroup (cgroup v2 unified format) and
+// returns the sysfs path for this process's cgroup.
+func getOwnCgroupPath() (string, error) {
+	f, err := os.Open("/proc/self/cgroup")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		// cgroup v2: "0::<path>"
+		parts := strings.SplitN(line, ":", 3)
+		if len(parts) == 3 && parts[0] == "0" {
+			return "/sys/fs/cgroup" + parts[2], nil
+		}
+	}
+	return "", fmt.Errorf("no cgroup v2 entry found in /proc/self/cgroup")
+}
+
+// releaseMemoryAfterScan forces the Go GC to run and returns freed memory to
+// the OS. Combined with reclaimCgroupPageCache, this minimizes the container's
+// memory footprint between scan cycles.
+func releaseMemoryAfterScan() {
+	runtime.GC()
+	debug.FreeOSMemory()
+}
+
 func aideReadDBPath(c *daemonConfig) string {
 	return path.Join(c.FileDir, aideReadDBFileName)
 }
@@ -345,6 +413,9 @@ func getNonEmptyFile(filename string) *os.File {
 		return file
 	}
 
+	if err := file.Close(); err != nil {
+		LOG("warning: error closing empty/unreadable file %s: %v", cleanFileName, err)
+	}
 	return nil
 }
 

cmd/manager/logcollector_util.go

@@ -30,12 +30,19 @@ import (
 	"github.com/spf13/cobra"
 
 	corev1 "k8s.io/api/core/v1"
+	kerr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	"github.com/openshift/file-integrity-operator/pkg/common"
 )
 
+var (
+	reFilesAdded   = regexp.MustCompile(`\s+Added entries:\s+(?P<num_added>\d+)`)
+	reFilesChanged = regexp.MustCompile(`\s+Changed entries:\s+(?P<num_changed>\d+)`)
+	reFilesRemoved = regexp.MustCompile(`\s+Removed entries:\s+(?P<num_removed>\d+)`)
+)
+
 const (
 	crdGroup      = "fileintegrity.openshift.io"
 	crdAPIVersion = "v1alpha1"
@@ -59,8 +66,7 @@ func getValidStringArg(cmd *cobra.Command, name string) string {
 	return val
 }
 
-func matchFileChangeRegex(contents string, regex string) string {
-	re := regexp.MustCompile(regex)
+func matchFileChangeRegex(contents string, re *regexp.Regexp) string {
 	match := re.FindStringSubmatch(contents)
 	if len(match) < 2 {
 		return "0"
@@ -70,9 +76,9 @@ func matchFileChangeRegex(contents string, regex string) string {
 }
 
 func annotateFileChangeSummary(contents string, annotations map[string]string) {
-	annotations[common.IntegrityLogFilesAddedAnnotation] = matchFileChangeRegex(contents, `\s+Added entries:\s+(?P<num_added>\d+)`)
-	annotations[common.IntegrityLogFilesChangedAnnotation] = matchFileChangeRegex(contents, `\s+Changed entries:\s+(?P<num_changed>\d+)`)
-	annotations[common.IntegrityLogFilesRemovedAnnotation] = matchFileChangeRegex(contents, `\s+Removed entries:\s+(?P<num_removed>\d+)`)
+	annotations[common.IntegrityLogFilesAddedAnnotation] = matchFileChangeRegex(contents, reFilesAdded)
+	annotations[common.IntegrityLogFilesChangedAnnotation] = matchFileChangeRegex(contents, reFilesChanged)
+	annotations[common.IntegrityLogFilesRemovedAnnotation] = matchFileChangeRegex(contents, reFilesRemoved)
 	DBG("added %s changed %s removed %s",
 		annotations[common.IntegrityLogFilesAddedAnnotation],
 		annotations[common.IntegrityLogFilesChangedAnnotation],
@@ -209,6 +215,12 @@ func reportOK(ctx context.Context, conf *daemonConfig, rt *daemonRuntime) error
 		fi := rt.GetFileIntegrityInstance()
 		confMap := newInformationalConfigMap(fi, conf.LogCollectorConfigMapName, conf.LogCollectorNode, nil)
 		_, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		if kerr.IsAlreadyExists(err) {
+			if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil {
+				return delErr
+			}
+			_, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		}
 		return err
 	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries))
 }
@@ -222,6 +234,12 @@ func reportError(ctx context.Context, msg string, conf *daemonConfig, rt *daemon
 		}
 		confMap := newInformationalConfigMap(fi, conf.LogCollectorConfigMapName, conf.LogCollectorNode, annotations)
 		_, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		if kerr.IsAlreadyExists(err) {
+			if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil {
+				return delErr
+			}
+			_, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		}
 		return err
 	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries))
 }
@@ -232,6 +250,12 @@ func uploadLog(ctx context.Context, contents, compressedContents []byte, conf *d
 		fi := rt.GetFileIntegrityInstance()
 		confMap := newLogConfigMap(fi, conf.LogCollectorConfigMapName, common.IntegrityLogContentKey, conf.LogCollectorNode, contents, compressedContents)
 		_, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		if kerr.IsAlreadyExists(err) {
+			if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil {
+				return delErr
+			}
+			_, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		}
 		return err
 	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries))
 }

cmd/manager/loops.go

@@ -72,6 +72,12 @@ func aideLoop(ctx context.Context, rt *daemonRuntime, conf *daemonConfig, errCha
 				// All done. Send the result.
 				rt.SendResult(aideResult)
 				rt.UnlockAideFiles("aideLoop")
+
+				// AIDE reads the entire host filesystem, and the resulting page
+				// cache is charged to this container's cgroup. Reclaim it so
+				// reported memory drops back to the daemon's actual working set.
+				reclaimCgroupPageCache()
+				releaseMemoryAfterScan()
 			}
 			time.Sleep(time.Second * time.Duration(conf.Interval))
 		}
@@ -215,6 +221,8 @@ func handleAIDEInit(ctx context.Context, rt *daemonRuntime, conf *daemonConfig,
 	}
 
 	LOG("initialization finished")
+	reclaimCgroupPageCache()
+	releaseMemoryAfterScan()
 	return nil
 }
 

pkg/controller/fileintegrity/fileintegrity_controller.go

@@ -909,7 +909,8 @@ func aideDaemonset(dsName string, fi *v1alpha1.FileIntegrity, operatorImage stri
 									},
 								},
 								{
-									// Needed for friendlier memory reporting as long as we are on golang < 1.16
+									// MADV_DONTNEED is already the default on Go >= 1.16. Kept for
+									// documentation; harmless no-op on current toolchain (Go 1.22).
 									Name:  "GODEBUG",
 									Value: "madvdontneed=1",
 								},